diff --git a/apps/labrinth/.sqlx/query-0f3d943e4fc48a94363b77c8a7d36eb1dd626e77331d8278c406df952691be4c.json b/apps/labrinth/.sqlx/query-0f3d943e4fc48a94363b77c8a7d36eb1dd626e77331d8278c406df952691be4c.json new file mode 100644 index 0000000000..4bc87e73a5 --- /dev/null +++ b/apps/labrinth/.sqlx/query-0f3d943e4fc48a94363b77c8a7d36eb1dd626e77331d8278c406df952691be4c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM payouts_values_notifications WHERE notified = FALSE AND user_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0f3d943e4fc48a94363b77c8a7d36eb1dd626e77331d8278c406df952691be4c" +} diff --git a/apps/labrinth/.sqlx/query-17f415f1140df5b3dd42a161c0f77b2475edb8041bc2b9701d51cf9cbfd69ba1.json b/apps/labrinth/.sqlx/query-17f415f1140df5b3dd42a161c0f77b2475edb8041bc2b9701d51cf9cbfd69ba1.json new file mode 100644 index 0000000000..444398a900 --- /dev/null +++ b/apps/labrinth/.sqlx/query-17f415f1140df5b3dd42a161c0f77b2475edb8041bc2b9701d51cf9cbfd69ba1.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tselect\n\t\t\tf.url,\n\t\t\tf.version_id as \"version_id: DBVersionId\",\n\t\t\tv.mod_id as \"project_id: DBProjectId\"\n\t\tfrom files f\n\t\tinner join versions v on v.id = f.version_id\n\t\twhere f.id = $1\n\t\t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "url", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "version_id: DBVersionId", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "project_id: DBProjectId", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "17f415f1140df5b3dd42a161c0f77b2475edb8041bc2b9701d51cf9cbfd69ba1" +} diff --git a/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json b/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json new file mode 100644 index 0000000000..921f7f92d9 --- /dev/null +++ b/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n id,\n status AS \"status: PayoutStatus\"\n FROM payouts\n ORDER BY id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "status: PayoutStatus", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286" +} diff --git a/apps/labrinth/.sqlx/query-20cff8fdf7971e91c9d473b9a4663ce02ca16781e32232ae0fa7a0af1973d3a4.json b/apps/labrinth/.sqlx/query-20cff8fdf7971e91c9d473b9a4663ce02ca16781e32232ae0fa7a0af1973d3a4.json new file mode 100644 index 0000000000..3c99ff3fed --- /dev/null +++ b/apps/labrinth/.sqlx/query-20cff8fdf7971e91c9d473b9a4663ce02ca16781e32232ae0fa7a0af1973d3a4.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM payouts_values_notifications WHERE notified = FALSE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "20cff8fdf7971e91c9d473b9a4663ce02ca16781e32232ae0fa7a0af1973d3a4" +} diff --git a/apps/labrinth/.sqlx/query-5ed1e727901573ee92683f206f3321fa4e192c75caa5a62a2790b5136e9a7dde.json b/apps/labrinth/.sqlx/query-5ed1e727901573ee92683f206f3321fa4e192c75caa5a62a2790b5136e9a7dde.json new file mode 100644 index 0000000000..f164ddcf96 --- /dev/null +++ b/apps/labrinth/.sqlx/query-5ed1e727901573ee92683f206f3321fa4e192c75caa5a62a2790b5136e9a7dde.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tdelete from attributions_exemptions\n\t\twhere version_id = $1\n\t\t", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5ed1e727901573ee92683f206f3321fa4e192c75caa5a62a2790b5136e9a7dde" +} diff --git a/apps/labrinth/.sqlx/query-6678cd4c51771cfaae2be8021ba66908ea41a06ba858dc5b523aef6aae27b850.json b/apps/labrinth/.sqlx/query-6678cd4c51771cfaae2be8021ba66908ea41a06ba858dc5b523aef6aae27b850.json new file mode 100644 index 0000000000..b4c2e5a56e --- /dev/null +++ b/apps/labrinth/.sqlx/query-6678cd4c51771cfaae2be8021ba66908ea41a06ba858dc5b523aef6aae27b850.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO payouts_values_notifications (date_available, user_id, notified)\n VALUES ($1, $2, FALSE)\n ON CONFLICT (date_available, user_id) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Timestamptz", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6678cd4c51771cfaae2be8021ba66908ea41a06ba858dc5b523aef6aae27b850" +} diff --git a/apps/labrinth/.sqlx/query-686994a5bfc061b4c6e1ca594fae9d5d3fb18974de6605a650f666d440dfe684.json b/apps/labrinth/.sqlx/query-686994a5bfc061b4c6e1ca594fae9d5d3fb18974de6605a650f666d440dfe684.json new file mode 100644 index 0000000000..7756621451 --- /dev/null +++ b/apps/labrinth/.sqlx/query-686994a5bfc061b4c6e1ca594fae9d5d3fb18974de6605a650f666d440dfe684.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n select\n (\n select count(*)\n from project_attribution_groups\n where project_id = $1\n ) as \"groups!\",\n (\n select count(*)\n from project_attribution_files paf\n inner join project_attribution_groups pag on pag.id = paf.group_id\n where pag.project_id = $1\n ) as \"files!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "groups!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "files!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "686994a5bfc061b4c6e1ca594fae9d5d3fb18974de6605a650f666d440dfe684" +} diff --git a/apps/labrinth/.sqlx/query-69a1cb4b7f1115a990d1fc4805d58541fc78e910111c09ba3d50a12d9ca4a9f8.json b/apps/labrinth/.sqlx/query-69a1cb4b7f1115a990d1fc4805d58541fc78e910111c09ba3d50a12d9ca4a9f8.json new file mode 100644 index 0000000000..fc7d2ac98d --- /dev/null +++ b/apps/labrinth/.sqlx/query-69a1cb4b7f1115a990d1fc4805d58541fc78e910111c09ba3d50a12d9ca4a9f8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO payouts_values (user_id, mod_id, amount, created, date_available)\n VALUES ($1, NULL, $2, NOW(), $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Numeric", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "69a1cb4b7f1115a990d1fc4805d58541fc78e910111c09ba3d50a12d9ca4a9f8" +} diff --git a/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json b/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json new file mode 100644 index 0000000000..89bd8147dc --- /dev/null +++ b/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT status AS \"status: PayoutStatus\" FROM payouts WHERE id = 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status: PayoutStatus", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3" +} diff --git a/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json b/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json new file mode 100644 index 0000000000..469c30168a --- /dev/null +++ b/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n VALUES ($1, $2, $3, $4, $5, 10.0, NOW())\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02" +} diff --git a/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json b/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json new file mode 100644 index 0000000000..52e020ebf2 --- /dev/null +++ b/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n VALUES ($1, $2, NULL, $3, $4, 10.00, NOW())\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606" +} diff --git a/apps/labrinth/.sqlx/query-fd5c773a61d35bcd71503ec4d5f86e8917cfab9679d5064074681663ba467e41.json b/apps/labrinth/.sqlx/query-fd5c773a61d35bcd71503ec4d5f86e8917cfab9679d5064074681663ba467e41.json new file mode 100644 index 0000000000..d3e3520bcc --- /dev/null +++ b/apps/labrinth/.sqlx/query-fd5c773a61d35bcd71503ec4d5f86e8917cfab9679d5064074681663ba467e41.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM notifications WHERE user_id = $1 AND body->>'type' = 'payout_available'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "fd5c773a61d35bcd71503ec4d5f86e8917cfab9679d5064074681663ba467e41" +} diff --git a/apps/labrinth/src/queue/file_scan.rs b/apps/labrinth/src/queue/file_scan.rs index 33052134c8..aff64fe477 100644 --- a/apps/labrinth/src/queue/file_scan.rs +++ b/apps/labrinth/src/queue/file_scan.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use chrono::Utc; use eyre::{Result, eyre}; use hex::ToHex; +use serde::Serialize; use sha1::Digest; use tokio::task::{spawn, spawn_blocking}; use tracing::{Instrument, info, info_span, warn}; @@ -42,6 +43,16 @@ struct PendingFileScan { project_id: DBProjectId, } +#[derive(Debug, Default, Serialize, utoipa::ToSchema)] +pub struct FileScanSummary { + /// Number of attribution groups newly created by the scan. + pub new_attribution_groups: u64, + /// Number of attribution files newly created by the scan. + pub new_attribution_files: u64, + /// Override file paths found and scanned in the file archive. + pub scanned_file_names: Vec, +} + /// Attribution enforcement is version/project-scoped, not file-hash-scoped. /// /// Versions or projects listed in `attributions_exemptions` predate this @@ -277,7 +288,7 @@ pub async fn scan_file( project_id: DBProjectId, file_id: DBFileId, file_url: &str, -) -> Result<()> { +) -> Result { let result = scan_file_inner(txn, redis, file_host, project_id, file_id, file_url) .await; @@ -296,7 +307,7 @@ async fn scan_file_inner( project_id: DBProjectId, file_id: DBFileId, file_url: &str, -) -> Result<()> { +) -> Result { let overrides = extract_override_files_from_storage(file_host, file_id, file_url) .await @@ -304,7 +315,16 @@ async fn scan_file_inner( eyre!("extracting overrides for file {file_id:?}") })?; + let scanned_file_names = + overrides.iter().map(|file| file.path.clone()).collect(); + let mut summary = FileScanSummary { + scanned_file_names, + ..Default::default() + }; + if !overrides.is_empty() { + let before = count_project_attributions(project_id, txn).await?; + let resolved = resolve_overrides(&overrides, redis, txn) .await .wrap_err_with(|| { @@ -318,15 +338,58 @@ async fn scan_file_inner( .wrap_err_with(|| { eyre!("persisting attribution results for file {file_id:?}") })?; + + let after = count_project_attributions(project_id, txn).await?; + summary.new_attribution_groups = + after.groups.saturating_sub(before.groups); + summary.new_attribution_files = + after.files.saturating_sub(before.files); + log_marked_override_projects(&resolved); } - Ok(()) + Ok(summary) +} + +struct ProjectAttributionCounts { + groups: u64, + files: u64, +} + +async fn count_project_attributions( + project_id: DBProjectId, + txn: &mut PgTransaction<'_>, +) -> Result { + let row = sqlx::query!( + r#" + select + ( + select count(*) + from project_attribution_groups + where project_id = $1 + ) as "groups!", + ( + select count(*) + from project_attribution_files paf + inner join project_attribution_groups pag on pag.id = paf.group_id + where pag.project_id = $1 + ) as "files!" + "#, + project_id as DBProjectId, + ) + .fetch_one(&mut *txn) + .await + .wrap_err("counting project attributions")?; + + Ok(ProjectAttributionCounts { + groups: row.groups as u64, + files: row.files as u64, + }) } -fn file_scan_result(result: &Result<()>) -> FileScanResult<'static> { +fn file_scan_result(result: &Result) -> FileScanResult<'static> { match result { - Ok(()) => Ok(()), + Ok(_) => Ok(()), Err(err) => Err(ApiError { error: "internal_error", description: format!("{err:#}"), @@ -480,6 +543,7 @@ const OVERRIDE_PREFIXES: &[&str] = &[ ]; fn should_scan(name: &str) -> bool { + let name = name.to_lowercase(); let should_skip = name.starts_with("mods/.connector/") || name.starts_with(".sable/natives/") || name.starts_with("local/crash_assistant/") @@ -491,8 +555,10 @@ fn should_scan(name: &str) -> bool { || name.starts_with("essential/") || name.ends_with(".rpo") || name.ends_with(".txt"); - let is_archive = name.contains(".jar") || name.contains(".zip"); - + let is_archive = name.ends_with(".jar") + || name.ends_with(".zip") + || name.ends_with(".jar.disabled") + || name.ends_with(".zip.disabled"); is_archive && !should_skip } diff --git a/apps/labrinth/src/routes/internal/attribution.rs b/apps/labrinth/src/routes/internal/attribution.rs index 7a0f37afcb..f141f96e98 100644 --- a/apps/labrinth/src/routes/internal/attribution.rs +++ b/apps/labrinth/src/routes/internal/attribution.rs @@ -3,17 +3,18 @@ use chrono::{DateTime, Utc}; use eyre::eyre; use serde::{Deserialize, Serialize}; -use crate::auth::get_user_from_headers; +use crate::auth::{check_is_moderator_from_headers, get_user_from_headers}; use crate::database::PgPool; use crate::database::models::{ - DBOrganization, DBTeamMember, DBVersion, + DBFileId, DBOrganization, DBTeamMember, DBVersion, ids::{ DBAttributionGroupId, DBProjectId, DBVersionId, generate_attribution_group_id, }, }; use crate::database::redis::RedisPool; -use crate::models::ids::{ProjectId, VersionId}; +use crate::file_hosting::FileHost; +use crate::models::ids::{FileId, ProjectId, VersionId}; use crate::models::pats::Scopes; use crate::models::projects::{ AttributionModerationStatusKind, AttributionResolution, @@ -21,6 +22,7 @@ use crate::models::projects::{ }; use crate::models::teams::ProjectPermissions; use crate::models::users::User; +use crate::queue::file_scan::{FileScanSummary, scan_file}; use crate::queue::moderation::ApprovalType; use crate::queue::session::AuthQueue; use crate::routes::ApiError; @@ -30,6 +32,7 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) { cfg.service(list) .service(update_group) .service(scan) + .service(force_scan_file) .service(assign) .service(split); } @@ -201,6 +204,80 @@ async fn scan( })) } +#[utoipa::path] +#[post("/file/{file_id}/scan")] +async fn force_scan_file( + req: HttpRequest, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, + file_host: web::Data, + path: web::Path, +) -> Result, ApiError> { + check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::PROJECT_READ, + ) + .await?; + + let file_id: DBFileId = path.into_inner().into(); + let file = sqlx::query!( + r#" + select + f.url, + f.version_id as "version_id: DBVersionId", + v.mod_id as "project_id: DBProjectId" + from files f + inner join versions v on v.id = f.version_id + where f.id = $1 + "#, + file_id as DBFileId, + ) + .fetch_optional(pool.as_ref()) + .await + .wrap_internal_err("failed to fetch attribution scan file")? + .ok_or(ApiError::NotFound)?; + + let mut transaction = pool.begin().await.wrap_internal_err( + "failed to begin attribution file scan transaction", + )?; + + sqlx::query!( + r#" + delete from attributions_exemptions + where version_id = $1 + "#, + file.version_id as DBVersionId, + ) + .execute(&mut transaction) + .await + .wrap_internal_err("failed to remove attribution scan exemption")?; + + let scan_summary = scan_file( + &mut transaction, + redis.as_ref(), + &**file_host, + file.project_id, + file_id, + &file.url, + ) + .await + .wrap_internal_err("failed to scan file for attributions")?; + + transaction.commit().await.wrap_internal_err( + "failed to commit attribution file scan transaction", + )?; + + DBVersion::clear_cache_ids(&[file.version_id], redis.as_ref()) + .await + .wrap_internal_err("failed to clear version cache")?; + + Ok(web::Json(scan_summary)) +} + #[utoipa::path] #[get("/{project_id}")] async fn list(