From 9490a9a34a672cdcf51feda24b5298bee76b30c9 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:22:07 +0100 Subject: [PATCH 01/42] Ignore some nags inside code blocks --- Cargo.toml | 128 +++++++++++++++++++-------------------- apps/labrinth/Cargo.toml | 1 + 2 files changed, 65 insertions(+), 64 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 30ad1af6b7..79bbd5591f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,22 +1,22 @@ [workspace] resolver = "2" members = [ - "apps/app", - "apps/app-playground", - "apps/daedalus_client", - "apps/labrinth", - "packages/app-lib", - "packages/ariadne", - "packages/component-derive", - "packages/daedalus", - "packages/modrinth-content-management", - "packages/modrinth-log", - "packages/modrinth-maxmind", - "packages/modrinth-util", - "packages/neverbounce", - "packages/path-util", - "packages/serde-binhum", - "packages/xredis", + "apps/app", + "apps/app-playground", + "apps/daedalus_client", + "apps/labrinth", + "packages/app-lib", + "packages/ariadne", + "packages/component-derive", + "packages/daedalus", + "packages/modrinth-content-management", + "packages/modrinth-log", + "packages/modrinth-maxmind", + "packages/modrinth-util", + "packages/neverbounce", + "packages/path-util", + "packages/serde-binhum", + "packages/xredis", ] [workspace.package] @@ -39,16 +39,16 @@ ariadne = { path = "packages/ariadne" } async-minecraft-ping = { path = "packages/async-minecraft-ping" } async-recursion = "1.1.1" async-stripe = { version = "0.41.0", default-features = false, features = [ - "runtime-tokio-hyper-rustls", + "runtime-tokio-hyper-rustls", ] } async-trait = "0.1.89" async-tungstenite = { version = "0.31.0", default-features = false, features = ["futures-03-sink"] } async-walkdir = "2.1.0" async_zip = "0.0.18" aws-sdk-s3 = { version = "=1.122.0", default-features = false, features = [ - "default-https-client", - "rt-tokio", - "rustls", + "default-https-client", + "rt-tokio", + "rustls", ] } base64 = "0.22.1" bitflags = "2.9.4" @@ -99,10 +99,10 @@ hmac = "0.12.1" httpdate = "1.0.3" hyper = "1.7.0" hyper-rustls = { version = "0.27.7", default-features = false, features = [ - "aws-lc-rs", - "http1", - "native-tokio", - "tls12", + "aws-lc-rs", + "http1", + "native-tokio", + "tls12", ] } hyper-util = "0.1.17" iana-time-zone = "0.1.64" @@ -114,22 +114,22 @@ jemalloc_pprof = "0.8.1" json-patch = { version = "4.1.0", default-features = false } json5 = "1.3.1" lettre = { version = "0.11.19", default-features = false, features = [ - "aws-lc-rs", - "builder", - "hostname", - "pool", - "rustls", - "rustls-native-certs", - "smtp-transport", - "tokio1", - "tokio1-rustls", + "aws-lc-rs", + "builder", + "hostname", + "pool", + "rustls", + "rustls-native-certs", + "smtp-transport", + "tokio1", + "tokio1-rustls", ] } linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ - "checked-decode", - "safe-decode", - "safe-encode", - "std", + "checked-decode", + "safe-decode", + "safe-encode", + "std", ] } maxminddb = "0.26.0" md5 = "0.8.0" @@ -159,8 +159,8 @@ pulldown-cmark = { version = "0.13.4", default-features = false } quartz_nbt = "0.2.9" quick-xml = "0.38.3" quote = { version = "1.0" } -rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 -rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 +rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 +rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 rdkafka = { version = "0.36.2", features = ["cmake-build"] } redis = "1.4.1" regex = "1.12.2" @@ -169,9 +169,9 @@ rgb = "0.8.52" rust_decimal = { version = "1.39.0", features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = "0.1.14" rust-s3 = { version = "0.37.0", default-features = false, features = [ - "fail-on-err", - "tags", - "tokio-rustls-tls", + "fail-on-err", + "tags", + "tokio-rustls-tls", ] } rustls = "0.23.32" rustrict = { version = "0.7.39", default-features = false, features = ["censor"] } @@ -180,12 +180,12 @@ same-file = "1.0.6" scalar_api_reference = { version = "0.2.2", default-features = false } secrecy = "0.10.3" sentry = { version = "0.45.0", default-features = false, features = [ - "backtrace", - "contexts", - "debug-images", - "panic", - "reqwest", - "rustls", + "backtrace", + "contexts", + "debug-images", + "panic", + "reqwest", + "rustls", ] } serde = "1.0.228" serde-binhum = { path = "packages/serde-binhum" } @@ -216,8 +216,8 @@ tauri-plugin-opener = "2.5.0" tauri-plugin-os = "2.3.1" tauri-plugin-single-instance = "2.3.4" tauri-plugin-updater = { git = "https://github.com/modrinth/plugins-workspace", rev = "0d30f2aa28ec668ce187d527da1c475da3c01cbc", default-features = false, features = [ - "rustls-tls", - "zip", + "rustls-tls", + "zip", ] } tauri-plugin-window-state = "2.4.0" tempfile = "3.23.0" @@ -247,21 +247,21 @@ validator = "0.20.0" webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" webp = { version = "0.3.1", default-features = false } -webview2-com = "0.38.0" # Should be updated in lockstep with wry +webview2-com = "0.38.0" # Should be updated in lockstep with wry whatlang = "0.18.0" whoami = "1.6.1" -windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 -windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 +windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 +windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 winreg = "0.55.0" woothee = "0.13.0" xredis = { path = "packages/xredis" } yaserde = "0.12.0" zbus = "5.11.0" zip = { version = "6.0.0", default-features = false, features = [ - "bzip2", - "deflate", - "deflate64", - "zstd", + "bzip2", + "deflate", + "deflate64", + "zstd", ] } zxcvbn = "3.1.0" @@ -319,15 +319,15 @@ opt-level = 3 # Optimize for speed and reduce size on release builds [profile.release] -opt-level = "s" # Optimize for binary size -strip = true # Remove debug symbols -lto = true # Enables link to optimizations -panic = "abort" # Strip expensive panic clean-up logic +opt-level = "s" # Optimize for binary size +strip = true # Remove debug symbols +lto = true # Enables link to optimizations +panic = "abort" # Strip expensive panic clean-up logic # Specific profile for labrinth production builds [profile.release-labrinth] inherits = "release" opt-level = 2 -strip = false # Keep debug symbols for Sentry -lto = "thin" # Enable LTO but keep compile times reasonable -panic = "unwind" # Don't exit the whole app on panic in production +strip = false # Keep debug symbols for Sentry +lto = "thin" # Enable LTO but keep compile times reasonable +panic = "unwind" # Don't exit the whole app on panic in production diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index f2325ec3ba..6eb09144f3 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -82,6 +82,7 @@ paste = { workspace = true } path-util = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } +pulldown-cmark = { workspace = true } psl = { workspace = true } pulldown-cmark = { workspace = true } quick-xml = { workspace = true } From af438af9706e27705bb1a3d5967c532382fb7db8 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 12:56:13 -0600 Subject: [PATCH 02/42] fmt --- apps/labrinth/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 6eb09144f3..f2325ec3ba 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -82,7 +82,6 @@ paste = { workspace = true } path-util = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } -pulldown-cmark = { workspace = true } psl = { workspace = true } pulldown-cmark = { workspace = true } quick-xml = { workspace = true } From ecc1de76742ebe60afd34bfba84ecd5a4963731d Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 10:55:38 -0600 Subject: [PATCH 03/42] fix: ensure all heading levels are validated --- .../validate/project/description/markdown.rs | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/apps/labrinth/src/validate/project/description/markdown.rs b/apps/labrinth/src/validate/project/description/markdown.rs index a102140974..5b6ccb21c2 100644 --- a/apps/labrinth/src/validate/project/description/markdown.rs +++ b/apps/labrinth/src/validate/project/description/markdown.rs @@ -6,10 +6,10 @@ use regex::Regex; use unicode_segmentation::UnicodeSegmentation; static HTML_HEADER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?is)]*>(.*?)").unwrap() + Regex::new(r"(?is)]*>(.*?)").unwrap() }); static ADJACENT_HTML_HEADERS: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)\s*\s* = LazyLock::new(|| { Regex::new(r"(?is)\s*(?:]*>\s*)*$").unwrap() }); @@ -101,7 +101,6 @@ impl<'a> DescriptionMarkdown<'a> { let markdown_headers = self .headings .iter() - .filter(|heading| is_primary_heading(heading.level)) .filter(|heading| header_is_long(&heading.text)) .count(); let without_code = self.replace_code(" "); @@ -130,8 +129,7 @@ impl<'a> DescriptionMarkdown<'a> { let [previous, current] = headings else { return false; }; - is_primary_heading(previous.level) - && previous.level == current.level + previous.level == current.level && self.markdown[previous.range.end..current.range.start] .trim() .is_empty() @@ -180,13 +178,6 @@ fn header_is_long(header: &str) -> bool { rendered.graphemes(true).count() > 80 } -fn is_primary_heading(level: HeadingLevel) -> bool { - matches!( - level, - HeadingLevel::H1 | HeadingLevel::H2 | HeadingLevel::H3 - ) -} - #[cfg(test)] mod tests { use super::DescriptionMarkdown; @@ -250,4 +241,38 @@ image: "![](/missing-alt.png)" .has_adjacent_same_level_headers() ); } + + #[test] + fn all_heading_levels_are_validated() { + for level in 1..=6 { + let prefix = "#".repeat(level); + let long_text = "heading ".repeat(12); + for (long_header, adjacent_headers, separated_headers) in [ + ( + format!("{prefix} {long_text}"), + format!("{prefix} First\n\n{prefix} Second"), + format!("{prefix} First\n\n```yaml\n# comment\n```\n\n{prefix} Second"), + ), + ( + format!("{long_text}"), + format!("First\n\nSecond"), + format!("First\n\n```yaml\n# comment\n```\n\nSecond"), + ), + ] { + let markdown = DescriptionMarkdown::parse(&long_header); + assert_eq!(markdown.long_header_count(), 1, "{long_header}"); + assert!(markdown.ends_with_header(), "{long_header}"); + assert!( + DescriptionMarkdown::parse(&adjacent_headers) + .has_adjacent_same_level_headers(), + "{adjacent_headers}" + ); + assert!( + !DescriptionMarkdown::parse(&separated_headers) + .has_adjacent_same_level_headers(), + "{separated_headers}" + ); + } + } + } } From 68ea4583f67099dd43420dfd9cb0ec7eb49dbb8f Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 11:07:23 -0600 Subject: [PATCH 04/42] feat: add new description matches summary with 80% similarity validator --- .../src/validate/project/description.rs | 11 ++- apps/labrinth/src/validate/project/mod.rs | 1 + apps/labrinth/src/validate/project/summary.rs | 48 +---------- apps/labrinth/src/validate/project/text.rs | 83 +++++++++++++++++++ apps/labrinth/tests/project.rs | 54 ++++++++++++ .../api-client/src/modules/labrinth/types.ts | 1 + .../moderation/src/data/nags/description.ts | 13 +++ 7 files changed, 165 insertions(+), 46 deletions(-) diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index 27dab4c4e3..269aea48d2 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -8,13 +8,14 @@ use super::text::{ extract_description_text, find_banned_description_link, has_image_without_alt_text, has_sufficient_english_blocks, js_string_length, non_standard_text_ratio, normalize_project_field_text, - profanity_matches, project_requires_english, + profanity_matches, project_requires_english, project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; const MIN_DESCRIPTION_CHARS: usize = 125; +const MAX_DESCRIPTION_SUMMARY_SIMILARITY: f64 = 0.8; const MAX_PROFANITY_COUNT: usize = 2; const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; @@ -91,6 +92,14 @@ pub(super) fn validate(project: &Project) -> Vec { ); } } + if project_text_similarity(&normalized_text, &project.summary) + >= MAX_DESCRIPTION_SUMMARY_SIMILARITY + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionMatchesSummary, + ProjectNagSeverity::Required, + )); + } if has_spam { nags.push(ProjectNag::new( ProjectNagKind::ProjectDescriptionSpam, diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index 308870b7a7..28ea0df273 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -68,6 +68,7 @@ pub enum ProjectNagKind { ProjectDescriptionProfanity, ProjectDescriptionNonStandardText, ProjectDescriptionNonEnglish, + ProjectDescriptionMatchesSummary, AddDescription, DescriptionTooShort, ProjectDescriptionSpam, diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 518bed6dfc..43ac090a06 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -3,7 +3,8 @@ use serde_json::json; use super::text::{ ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, has_summary_formatting, is_likely_english_summary, js_string_length, - normalize_project_field_text, profanity_matches, project_requires_english, + normalize_project_field_text, profanity_matches, project_requires_english, + project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; @@ -65,7 +66,7 @@ pub(super) fn validate(project: &Project) -> Vec { if !summary.is_empty() && !contains_link && !project.name.is_empty() - && summary_name_similarity(summary, &project.name) + && project_text_similarity(summary, &project.name) >= MAX_SUMMARY_NAME_SIMILARITY { nags.push(ProjectNag::new( @@ -135,46 +136,3 @@ fn requires_language_nag( && !contains_link && !has_spam } - -fn summary_name_similarity(summary: &str, name: &str) -> f64 { - let summary = normalized_for_similarity(summary); - let name = normalized_for_similarity(name); - let longest_length = summary.len().max(name.len()); - if longest_length == 0 { - return 0.0; - } - - 1.0 - levenshtein_distance(&summary, &name) as f64 / longest_length as f64 -} - -fn normalized_for_similarity(text: &str) -> Vec { - normalize_project_field_text(text) - .to_lowercase() - .chars() - .filter(|character| !character.is_whitespace()) - .collect() -} - -fn levenshtein_distance(left: &[char], right: &[char]) -> usize { - if left.len() > right.len() { - return levenshtein_distance(right, left); - } - - let mut previous_row = (0..=left.len()).collect::>(); - for (right_index, right_character) in right.iter().enumerate() { - let mut current_row = Vec::with_capacity(left.len() + 1); - current_row.push(right_index + 1); - for (left_index, left_character) in left.iter().enumerate() { - current_row.push( - (current_row[left_index] + 1) - .min(previous_row[left_index + 1] + 1) - .min( - previous_row[left_index] - + usize::from(left_character != right_character), - ), - ); - } - previous_row = current_row; - } - previous_row[left.len()] -} diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index c51df4d3d6..a4efd930ec 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::LazyLock; use linkify::{LinkFinder, LinkKind}; +use pulldown_cmark::{Event, Parser, Tag}; use regex::Regex; use unicode_normalization::UnicodeNormalization; use unicode_segmentation::UnicodeSegmentation; @@ -129,6 +130,49 @@ pub(super) fn normalize_project_field_text(text: &str) -> String { text.trim().nfc().collect() } +pub(super) fn project_text_similarity(left: &str, right: &str) -> f64 { + let left = normalized_for_similarity(left); + let right = normalized_for_similarity(right); + let longest_length = left.len().max(right.len()); + if longest_length == 0 { + return 0.0; + } + + 1.0 - levenshtein_distance(&left, &right) as f64 / longest_length as f64 +} + +fn normalized_for_similarity(text: &str) -> Vec { + normalize_project_field_text(text) + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() +} + +fn levenshtein_distance(left: &[char], right: &[char]) -> usize { + if left.len() > right.len() { + return levenshtein_distance(right, left); + } + + let mut previous_row = (0..=left.len()).collect::>(); + for (right_index, right_character) in right.iter().enumerate() { + let mut current_row = Vec::with_capacity(left.len() + 1); + current_row.push(right_index + 1); + for (left_index, left_character) in left.iter().enumerate() { + current_row.push( + (current_row[left_index] + 1) + .min(previous_row[left_index + 1] + 1) + .min( + previous_row[left_index] + + usize::from(left_character != right_character), + ), + ); + } + previous_row = current_row; + } + previous_row[left.len()] +} + pub(super) fn js_string_length(text: &str) -> usize { text.encode_utf16().count() } @@ -838,6 +882,8 @@ pub(super) fn find_link_or_ip(text: &str) -> Option { pub(super) fn has_summary_formatting(summary: &str) -> bool { has_paired_html_formatting(summary) || MARKDOWN_LINK.is_match(summary) + || Parser::new(summary) + .any(|event| matches!(event, Event::Start(Tag::Emphasis))) || summary.lines().any(|line| { let line = line.trim_start(); line.starts_with('#') @@ -1075,3 +1121,40 @@ fn has_enough_language_content(text: &str) -> bool { WORD.find_iter(text).count() >= 8 && text.trim().graphemes(true).count() >= 35 } + +#[cfg(test)] +mod tests { + use super::{has_summary_formatting, project_text_similarity}; + + #[test] + fn summary_detects_markdown_emphasis() { + for summary in ["*this*", "Adds *new features* to Minecraft", "*a*", "_this_"] { + assert!(has_summary_formatting(summary), "{summary:?}"); + } + } + + #[test] + fn summary_allows_literal_asterisks() { + for summary in ["A single * asterisk", "2 * 3 * 4", r"\*this\*", "An unmatched *asterisk"] { + assert!(!has_summary_formatting(summary), "{summary:?}"); + } + } + + #[test] + fn similarity_ignores_case_whitespace_and_unicode_composition() { + assert_eq!(project_text_similarity(" Café tools ", "CAFE\u{301}\nTOOLS"), 1.0); + } + + #[test] + fn similarity_distinguishes_the_eighty_percent_boundary() { + assert!(project_text_similarity("abcde", "abcdx") >= 0.8); + assert!(project_text_similarity("abcde", "abcxy") < 0.8); + } + + #[test] + fn empty_fields_do_not_match() { + assert_eq!(project_text_similarity(" ", "\n"), 0.0); + assert_eq!(project_text_similarity("", "some text"), 0.0); + assert_eq!(project_text_similarity("some text", ""), 0.0); + } +} diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 023aca2028..3e7373dc4c 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -658,6 +658,60 @@ async fn test_leaving_review_skips_validation() { .await; } +#[actix_rt::test] +async fn test_description_similarity_to_summary() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + let summary = "Explore new worlds with configurable tools and adventures."; + let response = api + .edit_project( + project_slug, + json!({ "status": "draft", "summary": summary }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + for (description, expected_match) in [ + (summary.to_string(), true), + (format!("**{}**\n\n```yaml\nsetting: true\n```", summary.to_uppercase()), true), + (format!("{summary} Players can discover custom structures, configure individual features, and follow detailed installation instructions for their preferred loader."), false), + (String::new(), false), + ] { + let response = api + .edit_project( + project_slug, + json!({ "description": description }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let request = test::TestRequest::get() + .uri(&format!("/v3/project/{project_slug}/validate")) + .append_pat(USER_USER_PAT) + .to_request(); + let response = api.call(request).await; + assert_status!(&response, StatusCode::OK); + let validation: serde_json::Value = test::read_body_json(response).await; + let matching_nag = validation["nags"] + .as_array() + .unwrap() + .iter() + .find(|nag| nag["kind"] == "project_description_matches_summary"); + assert_eq!(matching_nag.is_some(), expected_match, "{description}"); + if let Some(nag) = matching_nag { + assert_eq!(nag["severity"], "required"); + } + } + }, + ) + .await; +} + #[actix_rt::test] async fn test_plugin_and_datapack_validation_use_mod_tags() { with_test_environment( diff --git a/packages/api-client/src/modules/labrinth/types.ts b/packages/api-client/src/modules/labrinth/types.ts index ee90f01f86..e105386f05 100644 --- a/packages/api-client/src/modules/labrinth/types.ts +++ b/packages/api-client/src/modules/labrinth/types.ts @@ -1146,6 +1146,7 @@ export namespace Labrinth { | 'long-headers' | 'missing-alt-text' | 'project-description-banned-link' + | 'project-description-matches-summary' | 'project-description-non-english' | 'project-description-non-standard-text' | 'project-description-profanity' diff --git a/packages/moderation/src/data/nags/description.ts b/packages/moderation/src/data/nags/description.ts index abca386ac9..27c3b772a8 100644 --- a/packages/moderation/src/data/nags/description.ts +++ b/packages/moderation/src/data/nags/description.ts @@ -3,6 +3,14 @@ import { defineMessages } from '@modrinth/ui' import type { NagDefinitions } from './types.ts' const messages = defineMessages({ + reviewTitle: { + id: 'nags.project-description-content.title', + defaultMessage: 'Review the description', + }, + matchesSummary: { + id: 'project.text-validation.description-matches-summary', + defaultMessage: `Your project's description should provide unique information and not repeat the project's summary.`, + }, addTitle: { id: 'nags.add-description.title', defaultMessage: 'Add a description' }, add: { id: 'nags.add-description.description', @@ -77,6 +85,11 @@ const messages = defineMessages({ }) export const descriptionNags = { + 'project-description-matches-summary': { + title: messages.reviewTitle, + description: messages.matchesSummary, + destination: 'description', + }, 'add-description': { title: messages.addTitle, description: messages.add, From f1e7d68d207b6ac7147c991d9020649ebb1fc8cb Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 11:12:31 -0600 Subject: [PATCH 05/42] feat: hide publishing checklist for staff when theres no requred nags --- .../ui/moderation/ModerationProjectNags.vue | 11 +++++++++++ apps/frontend/src/composables/featureFlags.ts | 1 + 2 files changed, 12 insertions(+) diff --git a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue index 065a5eb2b0..26ff89cf7d 100644 --- a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue +++ b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue @@ -141,6 +141,7 @@ import { type MessageDescriptor, useVIntl, } from '@modrinth/ui' +import { isStaff } from '@modrinth/utils' import type { Component } from 'vue' import { computed, nextTick, onBeforeUnmount, onMounted, ref, watch } from 'vue' @@ -222,6 +223,7 @@ const messages = defineMessages({ const { formatMessage } = useVIntl() const { addNotification } = injectNotificationManager() +const flags = useFeatureFlags() const props = withDefaults(defineProps(), { versions: () => [], @@ -419,6 +421,15 @@ function isNagComplete(nag: Nag): boolean { const visibleNags = computed(() => { const finalNags = applicableNags.value.filter((nag) => !isNagComplete(nag)) + if ( + isProcessing.value && + isStaff(props.currentMember?.user) && + !flags.value.alwaysShowPublishingChecklistForStaff && + !finalNags.some((nag) => nag.status === 'required') + ) { + return [] + } + if (props.project.status === 'draft') { finalNags.push({ id: 'submit-for-review', diff --git a/apps/frontend/src/composables/featureFlags.ts b/apps/frontend/src/composables/featureFlags.ts index 1165ef04cc..8cd1a685da 100644 --- a/apps/frontend/src/composables/featureFlags.ts +++ b/apps/frontend/src/composables/featureFlags.ts @@ -53,6 +53,7 @@ export const DEFAULT_FEATURE_FLAGS = validateValues({ alwaysIgnoreErrorBanner: false, showViewProdRouteBanner: false, showModeratorProjectMemberUi: false, + alwaysShowPublishingChecklistForStaff: false, archonApiStaging: false, showHostingAccessInstanceAuditLog: false, versionDevInfoCollapsed: true, From c549709d8dadc6819cf4418be248c389ae0ab429 Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 11:23:33 -0600 Subject: [PATCH 06/42] feat: only show disclosures suggestion if no disclosures added --- .../src/validate/project/disclosures.rs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/apps/labrinth/src/validate/project/disclosures.rs b/apps/labrinth/src/validate/project/disclosures.rs index 6749399cbd..67ff610671 100644 --- a/apps/labrinth/src/validate/project/disclosures.rs +++ b/apps/labrinth/src/validate/project/disclosures.rs @@ -7,15 +7,18 @@ pub(super) fn validate( project: &crate::models::projects::Project, disclosures: Option<&[ProjectDisclosure]>, ) -> Vec { - let (project_type, _) = - LegacyProject::get_project_type(&project.project_types); - let mut nags = vec![ - ProjectNag::new( - ProjectNagKind::CheckDisclosures, - ProjectNagSeverity::Suggestion, - ) - .with_details(serde_json::json!({ "project_type": project_type })), - ]; + let mut nags = Vec::new(); + if disclosures.is_some_and(|disclosures| disclosures.is_empty()) { + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + nags.push( + ProjectNag::new( + ProjectNagKind::CheckDisclosures, + ProjectNagSeverity::Suggestion, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ); + } if disclosures.is_some_and(|disclosures| { disclosures.iter().any(disclosure_has_paired_html) From be4d76592817f38bc94e849a14fe65bdfc7ac516 Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 12:29:57 -0600 Subject: [PATCH 07/42] format --- .../src/validate/project/disclosures.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/labrinth/src/validate/project/disclosures.rs b/apps/labrinth/src/validate/project/disclosures.rs index 67ff610671..fc58e6073d 100644 --- a/apps/labrinth/src/validate/project/disclosures.rs +++ b/apps/labrinth/src/validate/project/disclosures.rs @@ -7,18 +7,18 @@ pub(super) fn validate( project: &crate::models::projects::Project, disclosures: Option<&[ProjectDisclosure]>, ) -> Vec { - let mut nags = Vec::new(); - if disclosures.is_some_and(|disclosures| disclosures.is_empty()) { - let (project_type, _) = - LegacyProject::get_project_type(&project.project_types); - nags.push( - ProjectNag::new( - ProjectNagKind::CheckDisclosures, - ProjectNagSeverity::Suggestion, - ) - .with_details(serde_json::json!({ "project_type": project_type })), - ); - } + let mut nags = Vec::new(); + if disclosures.is_some_and(|disclosures| disclosures.is_empty()) { + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + nags.push( + ProjectNag::new( + ProjectNagKind::CheckDisclosures, + ProjectNagSeverity::Suggestion, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ); + } if disclosures.is_some_and(|disclosures| { disclosures.iter().any(disclosure_has_paired_html) From 12a2858102b6652eba4f29dbeb3ac60101b7ab10 Mon Sep 17 00:00:00 2001 From: tdgao Date: Wed, 9 Sep 2026 12:47:23 -0600 Subject: [PATCH 08/42] feat: switch language detection to use lingua, and tune for less false positives --- Cargo.lock | 837 +++++++++++++++++- Cargo.toml | 2 +- apps/labrinth/Cargo.toml | 2 +- .../src/routes/v3/projects/validate.rs | 28 +- .../labrinth/src/validate/project/language.rs | 487 ++++++++++ apps/labrinth/src/validate/project/mod.rs | 1 + apps/labrinth/src/validate/project/summary.rs | 4 +- apps/labrinth/src/validate/project/text.rs | 224 +++-- 8 files changed, 1437 insertions(+), 148 deletions(-) create mode 100644 apps/labrinth/src/validate/project/language.rs diff --git a/Cargo.lock b/Cargo.lock index 4b06422b59..28231e36b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2529,6 +2529,15 @@ dependencies = [ "libc", ] +[[package]] +name = "counter" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e8e052be91f1c8aae2c1d81307d9f6e67f5f37001e3ddee419e971e73f03bc" +dependencies = [ + "num-traits", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -3754,6 +3763,9 @@ name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +dependencies = [ + "getrandom 0.2.16", +] [[package]] name = "fax" @@ -3963,6 +3975,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" + [[package]] name = "funty" version = "2.0.0" @@ -5296,6 +5314,25 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35a84fd5aa25fae5c0f4a33d9cac2ca017fc622cbd089be2229993514990f870" +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indenter" version = "0.3.4" @@ -5746,6 +5783,7 @@ dependencies = [ "json-patch", "labrinth", "lettre", + "lingua", "linkify", "modrinth-content-management", "modrinth-util", @@ -5801,7 +5839,6 @@ dependencies = [ "webauthn-rs", "webauthn-rs-proto", "webp", - "whatlang", "woothee", "xredis", "yaserde", @@ -5999,6 +6036,778 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "lingua" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40d9129bb9fe42c95d1bd420d6891607eaff17df16ee15674aed2d05b0ec8f4" +dependencies = [ + "counter", + "dashmap", + "fastrand 2.3.0", + "fst", + "include_dir", + "itertools 0.14.0", + "lingua-afrikaans-language-model", + "lingua-albanian-language-model", + "lingua-arabic-language-model", + "lingua-armenian-language-model", + "lingua-azerbaijani-language-model", + "lingua-basque-language-model", + "lingua-belarusian-language-model", + "lingua-bengali-language-model", + "lingua-bokmal-language-model", + "lingua-bosnian-language-model", + "lingua-bulgarian-language-model", + "lingua-catalan-language-model", + "lingua-chinese-language-model", + "lingua-croatian-language-model", + "lingua-czech-language-model", + "lingua-danish-language-model", + "lingua-dutch-language-model", + "lingua-english-language-model", + "lingua-esperanto-language-model", + "lingua-estonian-language-model", + "lingua-finnish-language-model", + "lingua-french-language-model", + "lingua-ganda-language-model", + "lingua-georgian-language-model", + "lingua-german-language-model", + "lingua-greek-language-model", + "lingua-gujarati-language-model", + "lingua-hebrew-language-model", + "lingua-hindi-language-model", + "lingua-hungarian-language-model", + "lingua-icelandic-language-model", + "lingua-indonesian-language-model", + "lingua-irish-language-model", + "lingua-italian-language-model", + "lingua-japanese-language-model", + "lingua-kazakh-language-model", + "lingua-korean-language-model", + "lingua-latin-language-model", + "lingua-latvian-language-model", + "lingua-lithuanian-language-model", + "lingua-macedonian-language-model", + "lingua-malay-language-model", + "lingua-maori-language-model", + "lingua-marathi-language-model", + "lingua-mongolian-language-model", + "lingua-nynorsk-language-model", + "lingua-persian-language-model", + "lingua-polish-language-model", + "lingua-portuguese-language-model", + "lingua-punjabi-language-model", + "lingua-romanian-language-model", + "lingua-russian-language-model", + "lingua-serbian-language-model", + "lingua-shona-language-model", + "lingua-slovak-language-model", + "lingua-slovene-language-model", + "lingua-somali-language-model", + "lingua-sotho-language-model", + "lingua-spanish-language-model", + "lingua-swahili-language-model", + "lingua-swedish-language-model", + "lingua-tagalog-language-model", + "lingua-tamil-language-model", + "lingua-telugu-language-model", + "lingua-thai-language-model", + "lingua-tsonga-language-model", + "lingua-tswana-language-model", + "lingua-turkish-language-model", + "lingua-ukrainian-language-model", + "lingua-urdu-language-model", + "lingua-vietnamese-language-model", + "lingua-welsh-language-model", + "lingua-xhosa-language-model", + "lingua-yoruba-language-model", + "lingua-zulu-language-model", + "maplit", + "rayon", + "regex", + "serde", + "serde-wasm-bindgen", + "strum", + "strum_macros", + "wasm-bindgen", +] + +[[package]] +name = "lingua-afrikaans-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2810a47263f58358fa22acc425b330a334382f33fcee8f205be672a7ec6d8d4a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-albanian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81be672350a5a37c9aae935dd8520295289cc02d098929484b963b60fb42e174" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-arabic-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcc254ff44209c2a50dea58a644f9c257a0aa5cd7e1739fd9c373fe4c7456cf" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-armenian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf4e5825cb3a2b11d8bd7baa6788167b65ffbf8ce21dc9b9dcb5e6adee1d5f0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-azerbaijani-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdcc775ff75bcc6de0ea5330f65e62644622e29c980a72b437780ba57bba838" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-basque-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cbea869a4556800f2798c61559fd2e87fd65d7698ce27ad6fba5b1d9ec1fbe9" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-belarusian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2887044d4c11388e15692bf953f70468282600c426e1cd6a208a93aeca3aedfd" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bengali-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f94131563e0e0c571cebf03f867d0e1b7dd15278f7cab3e4ba8b0f8026072cc" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bokmal-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b443b1ec0498bcf43594f9bf65ae8f5660b5bf21eb769950a245ced8e376ac23" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bosnian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04f540ed0441bcdbb0b36c43f306bc2b0892921ceb95e865fe0b8b0c39bb82c4" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bulgarian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4679441ff2d94b21a3d76293ffd5310a012ea62247b7b2066939e2728cbfc0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-catalan-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7366ac180de87eb19f32063b8eed1db270ab333f7a450d96250e53b14b0a750f" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-chinese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca7fa9f7671d684c82c168725f380fc873f14d6f4e8c82f0da681bcc0048d1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-croatian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86b1346b98397e7cf87e8f3fe544fce6514918e6bde253a014eea7ca03c31d8" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-czech-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348f06e4f90e1c2bc597ea3caf52abb9d3dd09ba227fa7cf4f2dc723a9810e98" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-danish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "363953413132601a06fe0cae1fffb5d519d0cfb378049cc78b3893edd03a194f" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-dutch-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0da523f4726c05ed557b5a309315c1c161a5ec9ca6036c6c1c799ce492b698" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-english-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97102de08b134a49f1cce05a1b6f5bf08ef21fe858074ae2b794e7892c43dd4b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-esperanto-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d2a7ae84bfa3ed2a05815ca4a81d34cee7da1a77e830ab933c4debc21d6532" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-estonian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7095f107a6e89147a9066db6a3f9e1b0a4706323d59a4bb58fa8218d74c1fde8" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-finnish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5b3c44089812704148c89dbe77fb34122c2f6e8182bbf0f72ea69c92e948e9e" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-french-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45508227e42c9cc5eb202e17c4c40e38ea7b9be9421faeb3ab0fb7ac37d9c681" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-ganda-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d4b9fba87a72ae0f929fbe0bdc78f71be78d14475219718e70fd686b281082" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-georgian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f3568595e73295b905ec54cc2dbda915efaee5f7abf91d572e9d44a843e81d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-german-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f584da803e8c135ea22dd3ed321a6b3e2ff3465559606be0a597924ecd465cb2" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-greek-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5da9a688ceb41963f3555526a2e4431e0e5208a3320e56b72b1de998ba6b1c7" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-gujarati-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e495d0421f5e4d6b1d71ebf3a98b445c5ada6b03f033b6b62db2382e4d075663" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hebrew-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc49d9a174de884bba595c8b575ee37fdb1c1a728eeff608d43e1a7378048545" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hindi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca88af8dede1a30fcd79318b9c22f3662785fd87ed5b39881d19770651bf720" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hungarian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d7072b2cdf43438214e82b53c51ad85a16d657e5c07a5a77bef78a4cd39423c" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-icelandic-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "341fa17414af93a6ccf33c6efda9e65d1476a18785b075a16d53f66c9e19379d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-indonesian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe1f2e05145c3fb4172c01c4563f729938c85f77742948444523dfcda2c3e80d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-irish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5099708a7fc7081e4171dbc0905691e5e02639d8899558a105720f28522261db" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-italian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a87385ee42f6a0306df81066d4f77c225a337137fc15978a4b319c6bcf4bfe" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-japanese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0938f75de3ae5dcdc925d823ed409854ca14f6a653782b9a1ad5d899462fbe" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-kazakh-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af90d71a099b11f3b0677402a9894e95b0acb9459343735dfaec57af6be11ee6" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-korean-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa87f6c43ff894fc75159c021480d2fdf96882bf5bd235f8916ceb6b7caae561" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-latin-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757c1330e893bc532559397d976656a7bb0a42f391f7c946727ffb43a3761e46" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-latvian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1b2f30766e9e1ce10d960ce3eb2da6514fd0ee18c8d617a566b03126140e656" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-lithuanian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4a2a1994e48841c1d2d63099fbd68476615bf0d833fa14a3654f7043e38fc7" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-macedonian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "779ea3952ecade3aff6a34eea6f432a0d10652fafd9723a1b2258551b08f83b4" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-malay-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10936b2ac22076f9dbb968c71aacbf16c71ed36a92da10f077850d7ec5ded341" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-maori-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5bbc1e34392b0081209eec7d6b201a062612717c0670dad451066d9026261da" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-marathi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb68fa15d9a57a2ca2a71f8f59f455267bd8a8793216449697c2dd04898915a0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-mongolian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a72e6a2a3c9ecb22dbd8870bae40c167f767defd69f05c58e5467d233b6360d5" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-nynorsk-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5e7e54f8e04a5286aab3a8e22f395b42e9341cab1fdafdd04660a17e81fe8d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-persian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf084e05d33ed66d64461e6397b571fdf74fe93c13c4ac4c84a633e33d2d07b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-polish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72eb03e7940b3178c152138a16976e374ed0a3ddc80dc6f0b56984ab1189cb67" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-portuguese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "432eda7984456055033ffa168037be9afe0f5c9ecd891bf5f27435247d496b47" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-punjabi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "620ec0c678bfca1b99d5379f8024cfed24149556a0d55957799d870f3c71df26" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-romanian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1051aa082754acfc52110456e29fbbf6d2c648da9a4753a861c34f19325cfab8" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-russian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc0850578299531b50192be2c1de1af651f1289784253645e737fe456a6a138" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-serbian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df46949714d864f81138b3590878c89f06cf5d0af0e758a1e1217dd13983a896" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-shona-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ca30d90634fc427e79dddfa555524f5983a630ce809aeb02b708f434937965" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-slovak-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "529a596f3d90b2051b3ead90e864f6ad9e14b43716c3f252268e9e6e43ab28de" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-slovene-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db52978574611533d873b727281d1c1d0ab2b031331654c1729723b682bc76a1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-somali-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be79a11e6d682c7f1ecb342fe41169a59d6adb5eac3b4de77973fb7bbf936e39" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-sotho-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c476a7019592446989258b7f204cc7b5429b42a4d7eb686b80461a5f790e08bc" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-spanish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56395a8d96c892130a9efb433c5f042977fb75ae5fb6e0058c8814a947459137" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-swahili-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "218763c238936496b53d88535e1a4db1eda30c278af682f02d6b836edb08162a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-swedish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c965c5b11d6e1e619a98e393d16f73a0754cb05b5b067b537fd25e9b74bec0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tagalog-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7e48a2d97dc20ebe3618c794d45c363299e4930cc2ff2fd21b174130d75e8d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tamil-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212e6ed4c50fc9b06278f5b401ba17eeece43ccc84515e95ece76efd16e55d9b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-telugu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1a68d768b356c89f0a68280ab9a4adec487ba3b0ba1d2dda244a86ed712797" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-thai-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2582eeabf02e39680856c3f88d3b93fcdb351009407d38a4735326c8b0a33bce" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tsonga-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e34d4bff2ed22b8b666b67a9a019c94fc01097764d7d7bc0a4f943abf88efd" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tswana-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa7f52c30b53ed63ac9bf9ddefaf746c983e668f917f4133d4cda39a009b7e1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-turkish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2ab6b46c596ebd7f58e0db7a3d732c42a2fa48b77094a0737e920a5c47dca0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-ukrainian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86175a68d53b3b1e3aaf2e2fa61d31e8f03df7e0fcb51d7e1dd62fce86cb393" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-urdu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d308fdae7c8caa05ffe455e8a2e568e5f018cb2742f588a810ecf306b890622" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-vietnamese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ef9a92dce65c6e9fea7f2343e51de8b5b9cd764fa8ae3592c4ba3edd992d4a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-welsh-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1057abcd15da2c81baafe25cefa148b5b56d382d645af749084987e1a6d576e2" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-xhosa-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d165e4dfe98f8ed1a9f768e5fefe13d472ad1c5600e7ceab6a1a4a942be4c2ee" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-yoruba-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6da42216997d892ce8822eae591084555469e1d32e44fe6e181996fb200492d3" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-zulu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e429650453aa15779df3560ce9150c1cdfa48bbb8975e95a925cf868b2a3dc0a" +dependencies = [ + "include_dir", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -6121,6 +6930,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "mappings" version = "0.7.1" @@ -9800,6 +10615,17 @@ dependencies = [ "typeid", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_bytes" version = "0.11.19" @@ -13283,15 +14109,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "whatlang" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e8f38b596e2a359b755342473520a99421e43658548c79489ee221b728c107" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "which" version = "7.0.3" diff --git a/Cargo.toml b/Cargo.toml index 79bbd5591f..aa2fc058bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ lettre = { version = "0.11.19", default-features = false, features = [ "tokio1", "tokio1-rustls", ] } +lingua = "=1.8.0" linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ "checked-decode", @@ -248,7 +249,6 @@ webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" webp = { version = "0.3.1", default-features = false } webview2-com = "0.38.0" # Should be updated in lockstep with wry -whatlang = "0.18.0" whoami = "1.6.1" windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index f2325ec3ba..9f8599feb5 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -72,6 +72,7 @@ image = { workspace = true, features = [ itertools = { workspace = true } json-patch = { workspace = true } lettre = { workspace = true } +lingua = { workspace = true } linkify = { workspace = true } modrinth-content-management = { workspace = true } modrinth-util = { workspace = true, features = ["decimal", "sentry", "utoipa"] } @@ -138,7 +139,6 @@ webauthn-rs = { workspace = true, features = [ ] } webauthn-rs-proto = { workspace = true } webp = { workspace = true } -whatlang = { workspace = true } woothee = { workspace = true } xredis = { workspace = true } yaserde = { workspace = true, features = ["derive"] } diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs index 877242e7b2..8fe0e2d745 100644 --- a/apps/labrinth/src/routes/v3/projects/validate.rs +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -67,12 +67,17 @@ pub(crate) async fn ensure_project_is_valid_for_review( .collect::>(); let project = Project::from(reloaded_project.clone()); - if has_required_nags_with_context( - &project, - &versions, - &available_categories, - &disclosures, - ) { + let has_required_nags = web::block(move || { + has_required_nags_with_context( + &project, + &versions, + &available_categories, + &disclosures, + ) + }) + .await + .wrap_internal_err("validating project for review")?; + if has_required_nags { return Err(ApiError::Request(eyre!( "project must have no required validation nags before or while under review" ))); @@ -157,12 +162,15 @@ pub async fn validate( .collect::>(); let project = Project::from(project); - Ok(web::Json(ProjectValidationResponse { - nags: validate_project( + let nags = web::block(move || { + validate_project( &project, &versions, &available_categories, &disclosures, - ), - })) + ) + }) + .await + .wrap_internal_err("validating project")?; + Ok(web::Json(ProjectValidationResponse { nags })) } diff --git a/apps/labrinth/src/validate/project/language.rs b/apps/labrinth/src/validate/project/language.rs new file mode 100644 index 0000000000..bb1ccc8362 --- /dev/null +++ b/apps/labrinth/src/validate/project/language.rs @@ -0,0 +1,487 @@ +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, LazyLock, Mutex}; + +use lingua::{Language, LanguageDetector, LanguageDetectorBuilder}; +use regex::Regex; +use unicode_normalization::UnicodeNormalization; +use unicode_segmentation::UnicodeSegmentation; + +const MIN_ENGLISH_TO_BEST_RATIO: f64 = 0.5; +const MIN_DESCRIPTION_CONFIDENCE: f64 = 0.35; +const MIN_DESCRIPTION_MARGIN: f64 = 0.15; +const MIN_NON_ENGLISH_CONFIDENCE: f64 = 0.8; +const MIN_NON_LATIN_CONFIDENCE: f64 = 0.5; +const MIN_DESCRIPTION_ENGLISH_PROPORTION: f64 = 0.2; +const MIN_PASSAGE_WORDS: usize = 4; +const MIN_PASSAGE_CHARS: usize = 25; + +static DETECTOR: LazyLock = + LazyLock::new(|| DetectorState::new()); + +struct BoundedCache { + entries: HashMap, (T, usize)>, + order: VecDeque>, + bytes: usize, +} + +impl BoundedCache { + fn new() -> Self { + Self { + entries: HashMap::new(), + order: VecDeque::new(), + bytes: 0, + } + } + + fn get(&self, text: &str) -> Option { + self.entries.get(text).map(|(value, _)| value.clone()) + } + + fn insert(&mut self, text: &str, value: T, value_bytes: usize) { + const MAX_BYTES: usize = 2 * 1024 * 1024; + const MAX_ENTRIES: usize = 2048; + let bytes = text.len() + value_bytes; + if bytes > MAX_BYTES || self.entries.contains_key(text) { + return; + } + while self.bytes + bytes > MAX_BYTES + || self.entries.len() >= MAX_ENTRIES + { + let Some(key) = self.order.pop_front() else { + break; + }; + if let Some((_, bytes)) = self.entries.remove(&key) { + self.bytes -= bytes; + } + } + let key: Arc = text.into(); + self.order.push_back(key.clone()); + self.entries.insert(key, (value, bytes)); + self.bytes += bytes; + } +} + +struct DetectorState { + detector: LanguageDetector, + scores: Mutex>, + spans: Mutex>>, +} + +impl DetectorState { + fn new() -> Self { + Self { + detector: LanguageDetectorBuilder::from_all_languages().build(), + scores: Mutex::new(BoundedCache::new()), + spans: Mutex::new(BoundedCache::new()), + } + } +} + +static WORD: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); + +static NON_LATIN_LETTER: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); +const MIN_DESCRIPTION_WORDS: usize = 8; +const MIN_DESCRIPTION_CHARS: usize = 35; +const MIN_ENGLISH_GRAMMATICAL_WORDS: usize = 2; + +struct Passage { + text: String, + language: Option, + confidence: f64, + runner_up_confidence: f64, + eligible: bool, + qualifies_as_english: bool, + confidently_non_english: bool, + alphabetic_words: usize, + english_grammatical_words: usize, +} + +#[derive(Clone, Copy, Default)] +struct Detection { + runner_up_confidence: f64, + language: Option, + confidence: f64, + english_confidence: f64, +} + +impl Detection { + fn is_english(&self) -> bool { + self.english_to_best_ratio() >= MIN_ENGLISH_TO_BEST_RATIO + } + + fn is_confidently_non_english(&self, text: &str) -> bool { + let has_non_latin_evidence = + NON_LATIN_LETTER.find_iter(text).count() >= 5; + let minimum_confidence = if has_non_latin_evidence { + MIN_NON_LATIN_CONFIDENCE + } else { + MIN_NON_ENGLISH_CONFIDENCE + }; + self.confidence >= minimum_confidence + && !self.is_english() + && (has_non_latin_evidence + || (text.graphemes(true).count() >= MIN_PASSAGE_CHARS + && alphabetic_word_count(text) >= MIN_PASSAGE_WORDS)) + } + + fn english_to_best_ratio(&self) -> f64 { + if self.confidence > 0.0 { + self.english_confidence / self.confidence + } else { + 0.0 + } + } +} + +fn detect(text: &str, detector: &DetectorState) -> Detection { + if let Some(value) = detector + .scores + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(text) + { + return value; + } + let scores = detector.detector.compute_language_confidence_values(text); + let english_confidence = scores + .iter() + .find(|(language, _)| *language == Language::English) + .map_or(0.0, |(_, confidence)| *confidence); + let best = scores.first().filter(|(_, confidence)| *confidence > 0.0); + let detection = Detection { + runner_up_confidence: scores.get(1).map_or(0.0, |(_, score)| *score), + language: best.map(|(language, _)| *language), + confidence: best.map_or(0.0, |(_, confidence)| *confidence), + english_confidence, + }; + detector + .scores + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(text, detection, std::mem::size_of::()); + detection +} + +/// Use Lingua's inferred boundaries, then rescore each span independently so +/// English can qualify even when another language has the highest score. +fn mixed_language_passages( + text: &str, + detector: &DetectorState, +) -> Vec { + let cached = detector + .spans + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(text); + let spans = cached.unwrap_or_else(|| { + let spans: Arc<[(usize, usize)]> = detector + .detector + .detect_multiple_languages_of(text) + .into_iter() + .map(|span| (span.start_index(), span.end_index())) + .collect(); + detector + .spans + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(text, spans.clone(), std::mem::size_of_val(spans.as_ref())); + spans + }); + spans + .iter() + .map(|(start, end)| text[*start..*end].trim().to_owned()) + .collect() +} + +fn has_enough_description_content(text: &str) -> bool { + alphabetic_word_count(text) >= MIN_DESCRIPTION_WORDS + && text.trim().graphemes(true).count() >= MIN_DESCRIPTION_CHARS +} + +/// Keep script runs disjoint so surrounding foreign text cannot qualify a short English fragment. +fn script_passages(text: &str) -> Vec<&str> { + let mut passages = Vec::new(); + let mut start = 0; + let mut previous = None; + for (index, character) in text.char_indices() { + if !character.is_alphabetic() { + continue; + } + let mut buffer = [0; 4]; + let non_latin = + NON_LATIN_LETTER.is_match(character.encode_utf8(&mut buffer)); + if previous.is_some_and(|previous| previous != non_latin) { + passages.push(text[start..index].trim()); + start = index; + } + previous = Some(non_latin); + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages +} + +fn description_passage(text: &str) -> Passage { + let mut passage = classify_passage( + text.to_owned(), + has_enough_description_content(text), + 1.0, + &DETECTOR, + has_enough_description_content(text) + || NON_LATIN_LETTER.find_iter(text).count() >= 5, + ); + let strong = passage.confidence >= MIN_DESCRIPTION_CONFIDENCE + && passage.confidence - passage.runner_up_confidence + >= MIN_DESCRIPTION_MARGIN; + let foreign_eligible = + passage.eligible || NON_LATIN_LETTER.find_iter(text).count() >= 5; + passage.qualifies_as_english = passage.eligible + && strong + && (passage.english_grammatical_words >= MIN_ENGLISH_GRAMMATICAL_WORDS + || has_lowercase_prose(text)) + && passage.language == Some(Language::English); + passage.confidently_non_english = foreign_eligible + && strong + && passage + .language + .is_some_and(|language| language != Language::English); + passage.eligible = foreign_eligible; + passage +} + +/// Capitalized names and acronyms alone do not establish English prose. +fn has_lowercase_prose(text: &str) -> bool { + WORD.find_iter(text) + .filter(|word| { + let word = word.as_str(); + word.chars().all(char::is_lowercase) && word.chars().count() > 1 + }) + .take(2) + .count() + >= 2 +} + +fn summary_translation_passage(text: &str) -> Passage { + let mut passage = description_passage(text); + if passage.qualifies_as_english + && passage.english_grammatical_words < MIN_ENGLISH_GRAMMATICAL_WORDS + { + passage.qualifies_as_english = false; + } + passage +} + +/// Bound detector input without overlapping or splitting words. +fn bounded_passages(text: &str) -> Vec<&str> { + let mut passages = Vec::new(); + let mut start = 0; + for (index, character) in text.char_indices() { + if index - start >= 4000 && character.is_whitespace() { + passages.push(text[start..index].trim()); + start = index + character.len_utf8(); + } + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages +} + +/// Only replace a coherent block when Lingua finds independently supported English and foreign portions. +fn description_passages(text: &str) -> Vec { + bounded_passages(text) + .into_iter() + .flat_map(script_passages) + .flat_map(|text| { + let whole = description_passage(text); + if whole.alphabetic_words < MIN_DESCRIPTION_WORDS * 2 { + return vec![whole]; + } + let spans = mixed_language_passages(text, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + let candidates = spans + .iter() + .map(|span| description_passage(span)) + .collect::>(); + if candidates.iter().any(|span| span.qualifies_as_english) + && candidates.iter().any(|span| span.confidently_non_english) + { + candidates + } else { + vec![whole] + } + }) + .collect() +} + +fn alphabetic_word_count(text: &str) -> usize { + WORD.find_iter(text) + .filter(|word| word.as_str().chars().any(char::is_alphabetic)) + .count() +} + +/// Require grammatical context before treating technical names as an English translation. +fn english_grammatical_word_count(text: &str) -> usize { + WORD.find_iter(text) + .filter(|word| { + matches!( + word.as_str().to_lowercase().as_str(), + "a" | "an" + | "the" | "this" + | "that" | "these" + | "those" | "it" + | "its" | "you" | "your" + | "we" | "our" | "they" + | "their" | "to" + | "for" | "with" + | "from" | "in" | "on" + | "of" | "and" | "or" + | "by" | "into" | "is" + | "are" | "be" | "can" + | "will" | "has" + | "have" | "does" + | "not" + ) + }) + .count() +} + +fn is_eligible_passage(text: &str) -> bool { + alphabetic_word_count(text) >= MIN_PASSAGE_WORDS + && text.graphemes(true).count() >= MIN_PASSAGE_CHARS +} + +fn classify_passage( + text: String, + eligible: bool, + minimum_ratio: f64, + detector: &DetectorState, + should_detect: bool, +) -> Passage { + let detection = if should_detect { + detect(&text, detector) + } else { + Detection::default() + }; + Passage { + runner_up_confidence: detection.runner_up_confidence, + qualifies_as_english: eligible + && detection.english_to_best_ratio() >= minimum_ratio, + confidently_non_english: eligible + && detection.is_confidently_non_english(&text), + alphabetic_words: alphabetic_word_count(&text), + english_grammatical_words: english_grammatical_word_count(&text), + text, + language: detection.language, + confidence: detection.confidence, + eligible, + } +} + +/// A foreign summary needs a contiguous English passage, not scattered product names. +fn summary_rescue_passages(text: &str) -> Vec { + script_passages(text) + .into_iter() + .filter(|text| !NON_LATIN_LETTER.is_match(text)) + .flat_map(|text| { + let normalized = + text.split_whitespace().collect::>().join(" "); + let whole = summary_translation_passage(&normalized); + if whole.qualifies_as_english + || !whole.eligible + || whole.english_grammatical_words + < MIN_ENGLISH_GRAMMATICAL_WORDS + { + return vec![whole]; + } + let spans = mixed_language_passages(&normalized, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + spans + .into_iter() + .map(|span| summary_translation_passage(&span)) + .collect() + }) + .collect() +} + +pub(super) fn is_likely_english_summary(text: &str) -> bool { + let normalized: String = text.nfkc().collect(); + let mut passages = vec![classify_passage( + normalized.clone(), + true, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + true, + )]; + let requires_strong_rescue = passages[0].confidently_non_english; + if requires_strong_rescue { + passages.extend(summary_rescue_passages(&normalized)); + } else if NON_LATIN_LETTER.is_match(&normalized) { + let latin = NON_LATIN_LETTER.replace_all(&normalized, " "); + passages[0].qualifies_as_english = false; + passages.push(classify_passage( + latin.to_string(), + is_eligible_passage(&latin), + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + is_eligible_passage(&latin), + )); + } else if !passages[0].qualifies_as_english { + passages.extend( + mixed_language_passages(&normalized, &DETECTOR) + .into_iter() + .map(|text| { + let eligible = is_eligible_passage(&text); + classify_passage( + text, + eligible, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + eligible, + ) + }), + ); + } + passages.iter().any(|passage| passage.qualifies_as_english) + || !passages + .iter() + .any(|passage| passage.confidently_non_english) +} + +pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { + let passages = blocks + .iter() + .flat_map(|block| { + let normalized: String = block.nfkc().collect(); + description_passages(&normalized) + }) + .collect::>(); + let english_utf16_length: usize = passages + .iter() + .filter(|passage| passage.qualifies_as_english) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let total_utf16_length: usize = passages + .iter() + .filter(|passage| passage.eligible) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let has_foreign_evidence = passages + .iter() + .any(|passage| passage.confidently_non_english); + let english_proportion = if total_utf16_length == 0 { + 0.0 + } else { + english_utf16_length as f64 / total_utf16_length as f64 + }; + + !has_foreign_evidence + || english_proportion >= MIN_DESCRIPTION_ENGLISH_PROPORTION +} diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index 28ea0df273..a9f94f6242 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -9,6 +9,7 @@ mod description; mod disclosures; mod gallery; mod icon; +mod language; mod license; mod links; mod moderation; diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 43ac090a06..23f14ea2ec 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -132,7 +132,7 @@ fn requires_language_nag( contains_link: bool, has_spam: bool, ) -> bool { - is_non_english_text(project, normalized_summary) - && !contains_link + !contains_link && !has_spam + && is_non_english_text(project, normalized_summary) } diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index a4efd930ec..0dda85d15d 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::sync::LazyLock; use linkify::{LinkFinder, LinkKind}; @@ -7,15 +7,16 @@ use regex::Regex; use unicode_normalization::UnicodeNormalization; use unicode_segmentation::UnicodeSegmentation; use url::Url; -use whatlang::{Detector, Lang}; use crate::models::exp::minecraft::Language; use crate::models::projects::Project; +pub(super) use super::language::{ + has_sufficient_english_blocks, is_likely_english_summary, +}; + static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); -static NON_LATIN_LETTER: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); static SPAM_TOKEN: LazyLock = LazyLock::new(|| { Regex::new(r#"[\p{L}\p{M}\p{N}]+(?:['_\u{2019}.:+/-][\p{L}\p{M}\p{N}]+)*"#) .unwrap() @@ -25,7 +26,6 @@ static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); finder }); -static LANGUAGE_DETECTOR: LazyLock = LazyLock::new(Detector::new); static MARKDOWN_LINK: LazyLock = LazyLock::new(|| Regex::new(r"!?\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_TAG: LazyLock = @@ -39,8 +39,6 @@ static HTML_CLOSE_TAG: LazyLock = LazyLock::new(|| Regex::new(r"(?is)").unwrap()); static CODE_BLOCK: LazyLock = LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); -static DESCRIPTION_BLOCK_BREAK: LazyLock = - LazyLock::new(|| Regex::new(r"\n\s*\n+").unwrap()); static INLINE_CODE: LazyLock = LazyLock::new(|| Regex::new(r"`([^`]*)`").unwrap()); static MARKDOWN_IMAGE: LazyLock = @@ -919,7 +917,7 @@ pub(super) fn has_paired_html_formatting(text: &str) -> bool { }) } -pub(super) fn extract_description_text(markdown: &str) -> String { +fn strip_description_markup(markdown: &str) -> String { let without_code = INLINE_CODE.replace_all(markdown, " "); let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); @@ -934,8 +932,11 @@ pub(super) fn extract_description_text(markdown: &str) -> String { .map_or_else(|| " ".to_owned(), |alt| alt.as_str().to_owned()) }, ); - let without_html = HTML_TAG.replace_all(&with_html_image_alt, " "); - without_html + HTML_TAG.replace_all(&with_html_image_alt, " ").into_owned() +} + +pub(super) fn extract_description_text(markdown: &str) -> String { + strip_description_markup(markdown) .lines() .map(|line| line.trim_start_matches(['>', '#'])) .collect::>() @@ -946,12 +947,96 @@ pub(super) fn extract_description_text(markdown: &str) -> String { .join(" ") } +static LANGUAGE_CONFIG_ENTRY: LazyLock = LazyLock::new(|| { + Regex::new(r"^([A-Za-z_][A-Za-z0-9_.-]*):(?:\s+(.*))?$").unwrap() +}); +static LANGUAGE_IDENTIFIER: LazyLock = LazyLock::new(|| { + Regex::new(r"\b[A-Za-z][A-Za-z0-9_-]*(?:\.[A-Za-z][A-Za-z0-9_-]*)+\b") + .unwrap() +}); +static LANGUAGE_COMMAND: LazyLock = LazyLock::new(|| { + Regex::new(r"(^|[\s|,(])/[A-Za-z][A-Za-z0-9_:./-]*").unwrap() +}); +static LANGUAGE_ARGUMENT: LazyLock = + LazyLock::new(|| Regex::new(r"\[[A-Za-z][A-Za-z0-9_ /|.-]*\]").unwrap()); + +/// Remove machine syntax before punctuation normalization turns identifiers into words. +fn description_language_input(markdown: &str) -> String { + let readable = strip_description_markup(markdown); + let readable = text_without_explicit_links(&readable); + let mut in_yaml = false; + readable + .lines() + .map(|line| { + let trimmed = line.trim(); + if matches!(trimmed, "yaml" | "yml") { + in_yaml = true; + return String::new(); + } + if let Some(entry) = LANGUAGE_CONFIG_ENTRY.captures(trimmed) { + let value = entry.get(2).map_or("", |value| value.as_str()); + let machine_value = + matches!(value, "true" | "false" | "null" | "~") + || value.parse::().is_ok() + || value.starts_with(['"', '\'']); + if in_yaml || entry[1].contains('_') || machine_value { + return String::new(); + } + } else if !trimmed.is_empty() { + in_yaml = false; + } + let has_command = LANGUAGE_COMMAND.is_match(line); + let without_commands = LANGUAGE_COMMAND.replace_all(line, "$1"); + let without_identifiers = + LANGUAGE_IDENTIFIER.replace_all(&without_commands, " "); + if has_command { + LANGUAGE_ARGUMENT + .replace_all(&without_identifiers, " ") + .into_owned() + } else { + without_identifiers.into_owned() + } + }) + .collect::>() + .join("\n") +} + pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - DESCRIPTION_BLOCK_BREAK - .split(markdown) - .map(extract_description_text) - .filter(|block| !block.is_empty()) - .collect() + let readable = description_language_input(markdown); + let mut blocks = Vec::new(); + let mut paragraph = Vec::new(); + let mut heading_only = false; + let mut blank = false; + for line in readable.lines() { + let line = line.trim(); + if line.is_empty() { + blank = true; + continue; + } + let heading = line.starts_with('#'); + let metadata_list = line + .split_once(':') + .is_some_and(|(_, values)| values.matches(',').count() >= 3); + if (heading + || metadata_list + || (blank + && !heading_only + && (WORD.find_iter(¶graph.join(" ")).count() >= 8 + || WORD.find_iter(line).count() >= 8))) + && !paragraph.is_empty() + { + blocks.push(extract_description_text(¶graph.join("\n"))); + paragraph.clear(); + } + paragraph.push(line); + heading_only = heading; + blank = false; + } + if !paragraph.is_empty() { + blocks.push(extract_description_text(¶graph.join("\n"))); + } + blocks.retain(|block| !block.is_empty()); + blocks } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { @@ -1013,115 +1098,6 @@ pub(super) fn project_requires_english(project: &Project) -> bool { || is_english_server } -pub(super) fn is_likely_english_summary(text: &str) -> bool { - let detection_text = normalize_language_text(text); - if has_dominant_non_latin_script(&detection_text) { - return false; - } - - if !has_enough_language_content(&detection_text) { - return true; - } - - LANGUAGE_DETECTOR - .detect(&detection_text) - .is_none_or(|info| info.lang() == Lang::Eng || !info.is_reliable()) -} - -pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { - let mut english_chunks = 0; - let mut non_english_chunks = 0; - - for block in blocks { - let detection_text = normalize_language_text(block); - if has_dominant_non_latin_script(&detection_text) { - non_english_chunks += 1; - - let latin_text = NON_LATIN_LETTER.replace_all(&detection_text, " "); - if has_enough_language_content(&latin_text) - && LANGUAGE_DETECTOR - .detect(&latin_text) - .is_some_and(|info| info.lang() == Lang::Eng) - { - english_chunks += 1; - } - continue; - } - - for chunk in language_chunks(&detection_text) { - let Some(info) = LANGUAGE_DETECTOR.detect(&chunk) else { - continue; - }; - - if info.lang() == Lang::Eng { - english_chunks += 1; - } else if info.is_reliable() { - non_english_chunks += 1; - } - } - } - - let classified_chunks = english_chunks + non_english_chunks; - classified_chunks == 0 || english_chunks * 10 >= classified_chunks * 3 -} - -fn normalize_language_text(text: &str) -> String { - text.nfkc().collect() -} - -fn has_dominant_non_latin_script(text: &str) -> bool { - const MIN_NON_LATIN_LETTERS: usize = 5; - - let non_latin_letters = NON_LATIN_LETTER.find_iter(text).count(); - let alphabetic_letters = text - .chars() - .filter(|character| character.is_alphabetic()) - .count(); - non_latin_letters >= MIN_NON_LATIN_LETTERS - && non_latin_letters * 2 >= alphabetic_letters -} - -fn language_chunks(block: &str) -> Vec { - const CHUNK_WORDS: usize = 24; - const CHUNK_STRIDE_WORDS: usize = 12; - - let words = WORD - .find_iter(block) - .map(|word| word.as_str()) - .collect::>(); - if words.len() < 8 { - return Vec::new(); - } - if words.len() <= CHUNK_WORDS { - let chunk = words.join(" "); - return has_enough_language_content(&chunk) - .then_some(chunk) - .into_iter() - .collect(); - } - - let mut starts = BTreeSet::new(); - let mut start = 0; - while start + 8 <= words.len() { - starts.insert(start); - start += CHUNK_STRIDE_WORDS; - } - starts.insert(words.len() - CHUNK_WORDS); - - starts - .into_iter() - .map(|start| { - words[start..(start + CHUNK_WORDS).min(words.len())].join(" ") - }) - .filter(|chunk| has_enough_language_content(chunk)) - .collect() -} - -fn has_enough_language_content(text: &str) -> bool { - WORD.find_iter(text).count() >= 8 - && text.trim().graphemes(true).count() >= 35 -} - #[cfg(test)] mod tests { use super::{has_summary_formatting, project_text_similarity}; From d0c84ccd687c93ffdbe38dec720b7679a96441e4 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 10:51:47 -0600 Subject: [PATCH 09/42] refactor: clean up description preprocessing --- apps/labrinth/src/validate/project/text.rs | 302 +++++++++++---------- 1 file changed, 153 insertions(+), 149 deletions(-) diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 0dda85d15d..479f0dff9f 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -12,7 +12,7 @@ use crate::models::exp::minecraft::Language; use crate::models::projects::Project; pub(super) use super::language::{ - has_sufficient_english_blocks, is_likely_english_summary, + has_sufficient_english_blocks, is_likely_english_summary, }; static WORD: LazyLock = @@ -129,46 +129,46 @@ pub(super) fn normalize_project_field_text(text: &str) -> String { } pub(super) fn project_text_similarity(left: &str, right: &str) -> f64 { - let left = normalized_for_similarity(left); - let right = normalized_for_similarity(right); - let longest_length = left.len().max(right.len()); - if longest_length == 0 { - return 0.0; - } + let left = normalized_for_similarity(left); + let right = normalized_for_similarity(right); + let longest_length = left.len().max(right.len()); + if longest_length == 0 { + return 0.0; + } - 1.0 - levenshtein_distance(&left, &right) as f64 / longest_length as f64 + 1.0 - levenshtein_distance(&left, &right) as f64 / longest_length as f64 } fn normalized_for_similarity(text: &str) -> Vec { - normalize_project_field_text(text) - .to_lowercase() - .chars() - .filter(|character| !character.is_whitespace()) - .collect() + normalize_project_field_text(text) + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() } fn levenshtein_distance(left: &[char], right: &[char]) -> usize { - if left.len() > right.len() { - return levenshtein_distance(right, left); - } + if left.len() > right.len() { + return levenshtein_distance(right, left); + } - let mut previous_row = (0..=left.len()).collect::>(); - for (right_index, right_character) in right.iter().enumerate() { - let mut current_row = Vec::with_capacity(left.len() + 1); - current_row.push(right_index + 1); - for (left_index, left_character) in left.iter().enumerate() { - current_row.push( - (current_row[left_index] + 1) - .min(previous_row[left_index + 1] + 1) - .min( - previous_row[left_index] - + usize::from(left_character != right_character), - ), - ); - } - previous_row = current_row; - } - previous_row[left.len()] + let mut previous_row = (0..=left.len()).collect::>(); + for (right_index, right_character) in right.iter().enumerate() { + let mut current_row = Vec::with_capacity(left.len() + 1); + current_row.push(right_index + 1); + for (left_index, left_character) in left.iter().enumerate() { + current_row.push( + (current_row[left_index] + 1) + .min(previous_row[left_index + 1] + 1) + .min( + previous_row[left_index] + + usize::from(left_character != right_character), + ), + ); + } + previous_row = current_row; + } + previous_row[left.len()] } pub(super) fn js_string_length(text: &str) -> usize { @@ -880,8 +880,8 @@ pub(super) fn find_link_or_ip(text: &str) -> Option { pub(super) fn has_summary_formatting(summary: &str) -> bool { has_paired_html_formatting(summary) || MARKDOWN_LINK.is_match(summary) - || Parser::new(summary) - .any(|event| matches!(event, Event::Start(Tag::Emphasis))) + || Parser::new(summary) + .any(|event| matches!(event, Event::Start(Tag::Emphasis))) || summary.lines().any(|line| { let line = line.trim_start(); line.starts_with('#') @@ -947,96 +947,87 @@ pub(super) fn extract_description_text(markdown: &str) -> String { .join(" ") } -static LANGUAGE_CONFIG_ENTRY: LazyLock = LazyLock::new(|| { - Regex::new(r"^([A-Za-z_][A-Za-z0-9_.-]*):(?:\s+(.*))?$").unwrap() -}); -static LANGUAGE_IDENTIFIER: LazyLock = LazyLock::new(|| { - Regex::new(r"\b[A-Za-z][A-Za-z0-9_-]*(?:\.[A-Za-z][A-Za-z0-9_-]*)+\b") - .unwrap() -}); -static LANGUAGE_COMMAND: LazyLock = LazyLock::new(|| { - Regex::new(r"(^|[\s|,(])/[A-Za-z][A-Za-z0-9_:./-]*").unwrap() -}); -static LANGUAGE_ARGUMENT: LazyLock = - LazyLock::new(|| Regex::new(r"\[[A-Za-z][A-Za-z0-9_ /|.-]*\]").unwrap()); +fn strip_dotted_identifiers(text: &str) -> String { + static IDENTIFIER: LazyLock = LazyLock::new(|| { + // matches dotted identifiers such as `com.example.mod` or `config.enabled` + Regex::new(r"\b[A-Za-z][A-Za-z0-9_-]*(?:\.[A-Za-z][A-Za-z0-9_-]*)+\b") + .unwrap() + }); + + IDENTIFIER.replace_all(text, " ").into_owned() +} + +fn strip_command_arguments(text: &str) -> String { + static ARGUMENT: LazyLock = LazyLock::new(|| { + // matches bracketed argument placeholders such as `[player]` or `[on|off]` + Regex::new(r"\[[A-Za-z][A-Za-z0-9_ /|.-]*\]").unwrap() + }); + + ARGUMENT.replace_all(text, " ").into_owned() +} + +fn strip_technical_syntax(line: &str) -> String { + static COMMAND: LazyLock = LazyLock::new(|| { + // matches slash commands such as `/help` or `/minecraft:give`. + Regex::new(r"(^|[\s|,(])/[A-Za-z][A-Za-z0-9_:./-]*").unwrap() + }); + + let without_commands = COMMAND.replace_all(line, "$1"); + let without_identifiers = strip_dotted_identifiers(&without_commands); + + if COMMAND.is_match(line) { + strip_command_arguments(&without_identifiers) + } else { + without_identifiers + } +} -/// Remove machine syntax before punctuation normalization turns identifiers into words. fn description_language_input(markdown: &str) -> String { - let readable = strip_description_markup(markdown); - let readable = text_without_explicit_links(&readable); - let mut in_yaml = false; - readable - .lines() - .map(|line| { - let trimmed = line.trim(); - if matches!(trimmed, "yaml" | "yml") { - in_yaml = true; - return String::new(); - } - if let Some(entry) = LANGUAGE_CONFIG_ENTRY.captures(trimmed) { - let value = entry.get(2).map_or("", |value| value.as_str()); - let machine_value = - matches!(value, "true" | "false" | "null" | "~") - || value.parse::().is_ok() - || value.starts_with(['"', '\'']); - if in_yaml || entry[1].contains('_') || machine_value { - return String::new(); - } - } else if !trimmed.is_empty() { - in_yaml = false; - } - let has_command = LANGUAGE_COMMAND.is_match(line); - let without_commands = LANGUAGE_COMMAND.replace_all(line, "$1"); - let without_identifiers = - LANGUAGE_IDENTIFIER.replace_all(&without_commands, " "); - if has_command { - LANGUAGE_ARGUMENT - .replace_all(&without_identifiers, " ") - .into_owned() - } else { - without_identifiers.into_owned() - } - }) - .collect::>() - .join("\n") + let readable = strip_description_markup(markdown); + let readable = text_without_explicit_links(&readable); + readable + .lines() + .map(strip_technical_syntax) + .collect::>() + .join("\n") } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - let readable = description_language_input(markdown); - let mut blocks = Vec::new(); - let mut paragraph = Vec::new(); - let mut heading_only = false; - let mut blank = false; - for line in readable.lines() { - let line = line.trim(); - if line.is_empty() { - blank = true; - continue; - } - let heading = line.starts_with('#'); - let metadata_list = line - .split_once(':') - .is_some_and(|(_, values)| values.matches(',').count() >= 3); - if (heading - || metadata_list - || (blank - && !heading_only - && (WORD.find_iter(¶graph.join(" ")).count() >= 8 - || WORD.find_iter(line).count() >= 8))) - && !paragraph.is_empty() - { - blocks.push(extract_description_text(¶graph.join("\n"))); - paragraph.clear(); - } - paragraph.push(line); - heading_only = heading; - blank = false; - } - if !paragraph.is_empty() { - blocks.push(extract_description_text(¶graph.join("\n"))); - } - blocks.retain(|block| !block.is_empty()); - blocks + let readable = description_language_input(markdown); + let mut blocks = Vec::new(); + let mut paragraph = Vec::new(); + let mut heading_only = false; + let mut blank = false; + for line in readable.lines() { + let line = line.trim(); + if line.is_empty() { + blank = true; + continue; + } + let heading = line.starts_with('#'); + let metadata_list = line + .split_once(':') + .is_some_and(|(_, values)| values.matches(',').count() >= 3); + if (heading + || metadata_list + || (blank + && !heading_only + && (WORD.find_iter(¶graph.join(" ")).count() >= 8 + || WORD.find_iter(line).count() >= 8))) + && !paragraph.is_empty() + { + blocks.push(extract_description_text(¶graph.join("\n"))); + paragraph.clear(); + } + paragraph.push(line); + heading_only = heading; + blank = false; + } + if !paragraph.is_empty() { + blocks.push(extract_description_text(¶graph.join("\n"))); + } + blocks.retain(|block| !block.is_empty()); + blocks } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { @@ -1100,37 +1091,50 @@ pub(super) fn project_requires_english(project: &Project) -> bool { #[cfg(test)] mod tests { - use super::{has_summary_formatting, project_text_similarity}; - - #[test] - fn summary_detects_markdown_emphasis() { - for summary in ["*this*", "Adds *new features* to Minecraft", "*a*", "_this_"] { - assert!(has_summary_formatting(summary), "{summary:?}"); - } - } + use super::{has_summary_formatting, project_text_similarity}; + + #[test] + fn summary_detects_markdown_emphasis() { + for summary in [ + "*this*", + "Adds *new features* to Minecraft", + "*a*", + "_this_", + ] { + assert!(has_summary_formatting(summary), "{summary:?}"); + } + } - #[test] - fn summary_allows_literal_asterisks() { - for summary in ["A single * asterisk", "2 * 3 * 4", r"\*this\*", "An unmatched *asterisk"] { - assert!(!has_summary_formatting(summary), "{summary:?}"); - } - } + #[test] + fn summary_allows_literal_asterisks() { + for summary in [ + "A single * asterisk", + "2 * 3 * 4", + r"\*this\*", + "An unmatched *asterisk", + ] { + assert!(!has_summary_formatting(summary), "{summary:?}"); + } + } - #[test] - fn similarity_ignores_case_whitespace_and_unicode_composition() { - assert_eq!(project_text_similarity(" Café tools ", "CAFE\u{301}\nTOOLS"), 1.0); - } + #[test] + fn similarity_ignores_case_whitespace_and_unicode_composition() { + assert_eq!( + project_text_similarity(" Café tools ", "CAFE\u{301}\nTOOLS"), + 1.0 + ); + } - #[test] - fn similarity_distinguishes_the_eighty_percent_boundary() { - assert!(project_text_similarity("abcde", "abcdx") >= 0.8); - assert!(project_text_similarity("abcde", "abcxy") < 0.8); - } + #[test] + fn similarity_distinguishes_the_eighty_percent_boundary() { + assert!(project_text_similarity("abcde", "abcdx") >= 0.8); + assert!(project_text_similarity("abcde", "abcxy") < 0.8); + } - #[test] - fn empty_fields_do_not_match() { - assert_eq!(project_text_similarity(" ", "\n"), 0.0); - assert_eq!(project_text_similarity("", "some text"), 0.0); - assert_eq!(project_text_similarity("some text", ""), 0.0); - } + #[test] + fn empty_fields_do_not_match() { + assert_eq!(project_text_similarity(" ", "\n"), 0.0); + assert_eq!(project_text_similarity("", "some text"), 0.0); + assert_eq!(project_text_similarity("some text", ""), 0.0); + } } From a40401748eea59ac1f6be7ab1601abf63f59b972 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 11:36:56 -0600 Subject: [PATCH 10/42] remove: unnecessary caching --- .../labrinth/src/validate/project/language.rs | 118 ++---------------- 1 file changed, 13 insertions(+), 105 deletions(-) diff --git a/apps/labrinth/src/validate/project/language.rs b/apps/labrinth/src/validate/project/language.rs index bb1ccc8362..6b285941db 100644 --- a/apps/labrinth/src/validate/project/language.rs +++ b/apps/labrinth/src/validate/project/language.rs @@ -1,5 +1,4 @@ -use std::collections::{HashMap, VecDeque}; -use std::sync::{Arc, LazyLock, Mutex}; +use std::sync::LazyLock; use lingua::{Language, LanguageDetector, LanguageDetectorBuilder}; use regex::Regex; @@ -15,67 +14,8 @@ const MIN_DESCRIPTION_ENGLISH_PROPORTION: f64 = 0.2; const MIN_PASSAGE_WORDS: usize = 4; const MIN_PASSAGE_CHARS: usize = 25; -static DETECTOR: LazyLock = - LazyLock::new(|| DetectorState::new()); - -struct BoundedCache { - entries: HashMap, (T, usize)>, - order: VecDeque>, - bytes: usize, -} - -impl BoundedCache { - fn new() -> Self { - Self { - entries: HashMap::new(), - order: VecDeque::new(), - bytes: 0, - } - } - - fn get(&self, text: &str) -> Option { - self.entries.get(text).map(|(value, _)| value.clone()) - } - - fn insert(&mut self, text: &str, value: T, value_bytes: usize) { - const MAX_BYTES: usize = 2 * 1024 * 1024; - const MAX_ENTRIES: usize = 2048; - let bytes = text.len() + value_bytes; - if bytes > MAX_BYTES || self.entries.contains_key(text) { - return; - } - while self.bytes + bytes > MAX_BYTES - || self.entries.len() >= MAX_ENTRIES - { - let Some(key) = self.order.pop_front() else { - break; - }; - if let Some((_, bytes)) = self.entries.remove(&key) { - self.bytes -= bytes; - } - } - let key: Arc = text.into(); - self.order.push_back(key.clone()); - self.entries.insert(key, (value, bytes)); - self.bytes += bytes; - } -} - -struct DetectorState { - detector: LanguageDetector, - scores: Mutex>, - spans: Mutex>>, -} - -impl DetectorState { - fn new() -> Self { - Self { - detector: LanguageDetectorBuilder::from_all_languages().build(), - scores: Mutex::new(BoundedCache::new()), - spans: Mutex::new(BoundedCache::new()), - } - } -} +static DETECTOR: LazyLock = + LazyLock::new(|| LanguageDetectorBuilder::from_all_languages().build()); static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); @@ -135,63 +75,31 @@ impl Detection { } } -fn detect(text: &str, detector: &DetectorState) -> Detection { - if let Some(value) = detector - .scores - .lock() - .unwrap_or_else(|error| error.into_inner()) - .get(text) - { - return value; - } - let scores = detector.detector.compute_language_confidence_values(text); +fn detect(text: &str, detector: &LanguageDetector) -> Detection { + let scores = detector.compute_language_confidence_values(text); let english_confidence = scores .iter() .find(|(language, _)| *language == Language::English) .map_or(0.0, |(_, confidence)| *confidence); let best = scores.first().filter(|(_, confidence)| *confidence > 0.0); - let detection = Detection { + Detection { runner_up_confidence: scores.get(1).map_or(0.0, |(_, score)| *score), language: best.map(|(language, _)| *language), confidence: best.map_or(0.0, |(_, confidence)| *confidence), english_confidence, - }; - detector - .scores - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(text, detection, std::mem::size_of::()); - detection + } } /// Use Lingua's inferred boundaries, then rescore each span independently so /// English can qualify even when another language has the highest score. fn mixed_language_passages( text: &str, - detector: &DetectorState, + detector: &LanguageDetector, ) -> Vec { - let cached = detector - .spans - .lock() - .unwrap_or_else(|error| error.into_inner()) - .get(text); - let spans = cached.unwrap_or_else(|| { - let spans: Arc<[(usize, usize)]> = detector - .detector - .detect_multiple_languages_of(text) - .into_iter() - .map(|span| (span.start_index(), span.end_index())) - .collect(); - detector - .spans - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(text, spans.clone(), std::mem::size_of_val(spans.as_ref())); - spans - }); - spans - .iter() - .map(|(start, end)| text[*start..*end].trim().to_owned()) + detector + .detect_multiple_languages_of(text) + .into_iter() + .map(|span| text[span.start_index()..span.end_index()].trim().to_owned()) .collect() } @@ -360,7 +268,7 @@ fn classify_passage( text: String, eligible: bool, minimum_ratio: f64, - detector: &DetectorState, + detector: &LanguageDetector, should_detect: bool, ) -> Passage { let detection = if should_detect { From 1b532fda7aa01d7b4fd8231f9d95b27aa8b739f5 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 12:22:45 -0600 Subject: [PATCH 11/42] feat: tune description block extraction --- .../src/validate/project/description.rs | 27 +- .../labrinth/src/validate/project/language.rs | 641 ++++++++++-------- apps/labrinth/src/validate/project/summary.rs | 15 +- apps/labrinth/src/validate/project/text.rs | 109 +-- 4 files changed, 433 insertions(+), 359 deletions(-) diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index 269aea48d2..6d76de6062 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -7,14 +7,16 @@ use super::text::{ ProfanityKind, contains_description_spam, extract_description_blocks, extract_description_text, find_banned_description_link, has_image_without_alt_text, has_sufficient_english_blocks, - js_string_length, non_standard_text_ratio, normalize_project_field_text, - profanity_matches, project_requires_english, project_text_similarity, + is_confidently_non_english_short_text, js_string_length, + non_standard_text_ratio, normalize_project_field_text, profanity_matches, + project_requires_english, project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; const MIN_DESCRIPTION_CHARS: usize = 125; +const MIN_DESCRIPTION_LANGUAGE_CHARS: usize = 35; const MAX_DESCRIPTION_SUMMARY_SIMILARITY: f64 = 0.8; const MAX_PROFANITY_COUNT: usize = 2; const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; @@ -92,14 +94,14 @@ pub(super) fn validate(project: &Project) -> Vec { ); } } - if project_text_similarity(&normalized_text, &project.summary) - >= MAX_DESCRIPTION_SUMMARY_SIMILARITY - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionMatchesSummary, - ProjectNagSeverity::Required, - )); - } + if project_text_similarity(&normalized_text, &project.summary) + >= MAX_DESCRIPTION_SUMMARY_SIMILARITY + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionMatchesSummary, + ProjectNagSeverity::Required, + )); + } if has_spam { nags.push(ProjectNag::new( ProjectNagKind::ProjectDescriptionSpam, @@ -160,8 +162,11 @@ fn is_non_english_text( text: &str, blocks: &[String], ) -> bool { + let length = js_string_length(text); project_requires_english(project) - && js_string_length(text) >= MIN_DESCRIPTION_CHARS + && length >= MIN_DESCRIPTION_LANGUAGE_CHARS + && (length >= MIN_DESCRIPTION_CHARS + || is_confidently_non_english_short_text(&blocks.join(" "))) && !has_sufficient_english_blocks(blocks) } diff --git a/apps/labrinth/src/validate/project/language.rs b/apps/labrinth/src/validate/project/language.rs index 6b285941db..3d6daef67b 100644 --- a/apps/labrinth/src/validate/project/language.rs +++ b/apps/labrinth/src/validate/project/language.rs @@ -9,387 +9,428 @@ const MIN_ENGLISH_TO_BEST_RATIO: f64 = 0.5; const MIN_DESCRIPTION_CONFIDENCE: f64 = 0.35; const MIN_DESCRIPTION_MARGIN: f64 = 0.15; const MIN_NON_ENGLISH_CONFIDENCE: f64 = 0.8; +const MIN_DOMINANT_NON_ENGLISH_CONFIDENCE: f64 = 0.6; +const MIN_DOMINANT_NON_ENGLISH_MARGIN: f64 = 0.6; +const MAX_DOMINANT_ENGLISH_CONFIDENCE: f64 = 0.05; const MIN_NON_LATIN_CONFIDENCE: f64 = 0.5; +const MIN_SHORT_TEXT_CONFIDENCE: f64 = 0.95; +const MIN_SHORT_TEXT_MARGIN: f64 = 0.5; const MIN_DESCRIPTION_ENGLISH_PROPORTION: f64 = 0.2; const MIN_PASSAGE_WORDS: usize = 4; const MIN_PASSAGE_CHARS: usize = 25; static DETECTOR: LazyLock = - LazyLock::new(|| LanguageDetectorBuilder::from_all_languages().build()); + LazyLock::new(|| LanguageDetectorBuilder::from_all_languages().build()); static WORD: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); static NON_LATIN_LETTER: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); + LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); const MIN_DESCRIPTION_WORDS: usize = 8; const MIN_DESCRIPTION_CHARS: usize = 35; const MIN_ENGLISH_GRAMMATICAL_WORDS: usize = 2; struct Passage { - text: String, - language: Option, - confidence: f64, - runner_up_confidence: f64, - eligible: bool, - qualifies_as_english: bool, - confidently_non_english: bool, - alphabetic_words: usize, - english_grammatical_words: usize, + text: String, + language: Option, + confidence: f64, + runner_up_confidence: f64, + eligible: bool, + qualifies_as_english: bool, + confidently_non_english: bool, + alphabetic_words: usize, + english_grammatical_words: usize, } #[derive(Clone, Copy, Default)] struct Detection { - runner_up_confidence: f64, - language: Option, - confidence: f64, - english_confidence: f64, + runner_up_confidence: f64, + language: Option, + confidence: f64, + english_confidence: f64, } impl Detection { - fn is_english(&self) -> bool { - self.english_to_best_ratio() >= MIN_ENGLISH_TO_BEST_RATIO - } + fn is_english(&self) -> bool { + self.english_to_best_ratio() >= MIN_ENGLISH_TO_BEST_RATIO + } - fn is_confidently_non_english(&self, text: &str) -> bool { - let has_non_latin_evidence = - NON_LATIN_LETTER.find_iter(text).count() >= 5; - let minimum_confidence = if has_non_latin_evidence { - MIN_NON_LATIN_CONFIDENCE - } else { - MIN_NON_ENGLISH_CONFIDENCE - }; - self.confidence >= minimum_confidence - && !self.is_english() - && (has_non_latin_evidence - || (text.graphemes(true).count() >= MIN_PASSAGE_CHARS - && alphabetic_word_count(text) >= MIN_PASSAGE_WORDS)) - } + fn is_confidently_non_english(&self, text: &str) -> bool { + let has_non_latin_evidence = + NON_LATIN_LETTER.find_iter(text).count() >= 5; + let minimum_confidence = if has_non_latin_evidence { + MIN_NON_LATIN_CONFIDENCE + } else { + MIN_NON_ENGLISH_CONFIDENCE + }; + let clearly_dominant = self.confidence + >= MIN_DOMINANT_NON_ENGLISH_CONFIDENCE + && self.confidence - self.runner_up_confidence + >= MIN_DOMINANT_NON_ENGLISH_MARGIN + && self.english_confidence < MAX_DOMINANT_ENGLISH_CONFIDENCE; + (self.confidence >= minimum_confidence || clearly_dominant) + && !self.is_english() + && (has_non_latin_evidence + || (text.graphemes(true).count() >= MIN_PASSAGE_CHARS + && alphabetic_word_count(text) >= MIN_PASSAGE_WORDS)) + } - fn english_to_best_ratio(&self) -> f64 { - if self.confidence > 0.0 { - self.english_confidence / self.confidence - } else { - 0.0 - } - } + fn english_to_best_ratio(&self) -> f64 { + if self.confidence > 0.0 { + self.english_confidence / self.confidence + } else { + 0.0 + } + } } fn detect(text: &str, detector: &LanguageDetector) -> Detection { - let scores = detector.compute_language_confidence_values(text); - let english_confidence = scores - .iter() - .find(|(language, _)| *language == Language::English) - .map_or(0.0, |(_, confidence)| *confidence); - let best = scores.first().filter(|(_, confidence)| *confidence > 0.0); - Detection { - runner_up_confidence: scores.get(1).map_or(0.0, |(_, score)| *score), - language: best.map(|(language, _)| *language), - confidence: best.map_or(0.0, |(_, confidence)| *confidence), - english_confidence, - } + let scores = detector.compute_language_confidence_values(text); + let english_confidence = scores + .iter() + .find(|(language, _)| *language == Language::English) + .map_or(0.0, |(_, confidence)| *confidence); + let best = scores.first().filter(|(_, confidence)| *confidence > 0.0); + Detection { + runner_up_confidence: scores.get(1).map_or(0.0, |(_, score)| *score), + language: best.map(|(language, _)| *language), + confidence: best.map_or(0.0, |(_, confidence)| *confidence), + english_confidence, + } +} + +/// Short fields need stronger evidence before bypassing the usual length gate. +pub(super) fn is_confidently_non_english_short_text(text: &str) -> bool { + let normalized: String = text.nfkc().collect(); + let detection = detect(&normalized, &DETECTOR); + detection.confidence >= MIN_SHORT_TEXT_CONFIDENCE + && detection.confidence - detection.runner_up_confidence + >= MIN_SHORT_TEXT_MARGIN + && detection.is_confidently_non_english(&normalized) } /// Use Lingua's inferred boundaries, then rescore each span independently so /// English can qualify even when another language has the highest score. fn mixed_language_passages( - text: &str, - detector: &LanguageDetector, + text: &str, + detector: &LanguageDetector, ) -> Vec { - detector - .detect_multiple_languages_of(text) - .into_iter() - .map(|span| text[span.start_index()..span.end_index()].trim().to_owned()) - .collect() + detector + .detect_multiple_languages_of(text) + .into_iter() + .map(|span| { + text[span.start_index()..span.end_index()].trim().to_owned() + }) + .collect() } fn has_enough_description_content(text: &str) -> bool { - alphabetic_word_count(text) >= MIN_DESCRIPTION_WORDS - && text.trim().graphemes(true).count() >= MIN_DESCRIPTION_CHARS + alphabetic_word_count(text) >= MIN_DESCRIPTION_WORDS + && text.trim().graphemes(true).count() >= MIN_DESCRIPTION_CHARS } /// Keep script runs disjoint so surrounding foreign text cannot qualify a short English fragment. fn script_passages(text: &str) -> Vec<&str> { - let mut passages = Vec::new(); - let mut start = 0; - let mut previous = None; - for (index, character) in text.char_indices() { - if !character.is_alphabetic() { - continue; - } - let mut buffer = [0; 4]; - let non_latin = - NON_LATIN_LETTER.is_match(character.encode_utf8(&mut buffer)); - if previous.is_some_and(|previous| previous != non_latin) { - passages.push(text[start..index].trim()); - start = index; - } - previous = Some(non_latin); - } - if !text[start..].trim().is_empty() { - passages.push(text[start..].trim()); - } - passages + let mut passages = Vec::new(); + let mut start = 0; + let mut previous = None; + for (index, character) in text.char_indices() { + if !character.is_alphabetic() { + continue; + } + let mut buffer = [0; 4]; + let non_latin = + NON_LATIN_LETTER.is_match(character.encode_utf8(&mut buffer)); + if previous.is_some_and(|previous| previous != non_latin) { + passages.push(text[start..index].trim()); + start = index; + } + previous = Some(non_latin); + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages } fn description_passage(text: &str) -> Passage { - let mut passage = classify_passage( - text.to_owned(), - has_enough_description_content(text), - 1.0, - &DETECTOR, - has_enough_description_content(text) - || NON_LATIN_LETTER.find_iter(text).count() >= 5, - ); - let strong = passage.confidence >= MIN_DESCRIPTION_CONFIDENCE - && passage.confidence - passage.runner_up_confidence - >= MIN_DESCRIPTION_MARGIN; - let foreign_eligible = - passage.eligible || NON_LATIN_LETTER.find_iter(text).count() >= 5; - passage.qualifies_as_english = passage.eligible - && strong - && (passage.english_grammatical_words >= MIN_ENGLISH_GRAMMATICAL_WORDS - || has_lowercase_prose(text)) - && passage.language == Some(Language::English); - passage.confidently_non_english = foreign_eligible - && strong - && passage - .language - .is_some_and(|language| language != Language::English); - passage.eligible = foreign_eligible; - passage + let mut passage = classify_passage( + text.to_owned(), + has_enough_description_content(text), + 1.0, + &DETECTOR, + has_enough_description_content(text) + || NON_LATIN_LETTER.find_iter(text).count() >= 5, + ); + let strong = passage.confidence >= MIN_DESCRIPTION_CONFIDENCE + && passage.confidence - passage.runner_up_confidence + >= MIN_DESCRIPTION_MARGIN; + let foreign_eligible = + passage.eligible || NON_LATIN_LETTER.find_iter(text).count() >= 5; + passage.qualifies_as_english = passage.eligible + && strong + && (passage.english_grammatical_words >= MIN_ENGLISH_GRAMMATICAL_WORDS + || has_lowercase_prose(text)) + && passage.language == Some(Language::English); + passage.confidently_non_english = foreign_eligible + && strong + && passage + .language + .is_some_and(|language| language != Language::English); + passage.eligible = foreign_eligible; + passage } /// Capitalized names and acronyms alone do not establish English prose. fn has_lowercase_prose(text: &str) -> bool { - WORD.find_iter(text) - .filter(|word| { - let word = word.as_str(); - word.chars().all(char::is_lowercase) && word.chars().count() > 1 - }) - .take(2) - .count() - >= 2 + WORD.find_iter(text) + .filter(|word| { + let word = word.as_str(); + word.chars().all(char::is_lowercase) && word.chars().count() > 1 + }) + .take(2) + .count() + >= 2 } fn summary_translation_passage(text: &str) -> Passage { - let mut passage = description_passage(text); - if passage.qualifies_as_english - && passage.english_grammatical_words < MIN_ENGLISH_GRAMMATICAL_WORDS - { - passage.qualifies_as_english = false; - } - passage + let mut passage = description_passage(text); + if passage.qualifies_as_english + && passage.english_grammatical_words < MIN_ENGLISH_GRAMMATICAL_WORDS + { + passage.qualifies_as_english = false; + } + passage } /// Bound detector input without overlapping or splitting words. fn bounded_passages(text: &str) -> Vec<&str> { - let mut passages = Vec::new(); - let mut start = 0; - for (index, character) in text.char_indices() { - if index - start >= 4000 && character.is_whitespace() { - passages.push(text[start..index].trim()); - start = index + character.len_utf8(); - } - } - if !text[start..].trim().is_empty() { - passages.push(text[start..].trim()); - } - passages + let mut passages = Vec::new(); + let mut start = 0; + for (index, character) in text.char_indices() { + if index - start >= 4000 && character.is_whitespace() { + passages.push(text[start..index].trim()); + start = index + character.len_utf8(); + } + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages } /// Only replace a coherent block when Lingua finds independently supported English and foreign portions. fn description_passages(text: &str) -> Vec { - bounded_passages(text) - .into_iter() - .flat_map(script_passages) - .flat_map(|text| { - let whole = description_passage(text); - if whole.alphabetic_words < MIN_DESCRIPTION_WORDS * 2 { - return vec![whole]; - } - let spans = mixed_language_passages(text, &DETECTOR); - if spans.len() <= 1 { - return vec![whole]; - } - let candidates = spans - .iter() - .map(|span| description_passage(span)) - .collect::>(); - if candidates.iter().any(|span| span.qualifies_as_english) - && candidates.iter().any(|span| span.confidently_non_english) - { - candidates - } else { - vec![whole] - } - }) - .collect() + bounded_passages(text) + .into_iter() + .flat_map(script_passages) + .flat_map(|text| { + let whole = description_passage(text); + if whole.alphabetic_words < MIN_DESCRIPTION_WORDS * 2 { + return vec![whole]; + } + let spans = mixed_language_passages(text, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + let candidates = spans + .iter() + .map(|span| description_passage(span)) + .collect::>(); + if candidates.iter().any(|span| span.qualifies_as_english) + && candidates.iter().any(|span| span.confidently_non_english) + { + candidates + } else { + vec![whole] + } + }) + .collect() } fn alphabetic_word_count(text: &str) -> usize { - WORD.find_iter(text) - .filter(|word| word.as_str().chars().any(char::is_alphabetic)) - .count() + WORD.find_iter(text) + .filter(|word| word.as_str().chars().any(char::is_alphabetic)) + .count() } /// Require grammatical context before treating technical names as an English translation. fn english_grammatical_word_count(text: &str) -> usize { - WORD.find_iter(text) - .filter(|word| { - matches!( - word.as_str().to_lowercase().as_str(), - "a" | "an" - | "the" | "this" - | "that" | "these" - | "those" | "it" - | "its" | "you" | "your" - | "we" | "our" | "they" - | "their" | "to" - | "for" | "with" - | "from" | "in" | "on" - | "of" | "and" | "or" - | "by" | "into" | "is" - | "are" | "be" | "can" - | "will" | "has" - | "have" | "does" - | "not" - ) - }) - .count() + WORD.find_iter(text) + .filter(|word| { + matches!( + word.as_str().to_lowercase().as_str(), + "a" | "an" + | "the" + | "this" + | "that" + | "these" + | "those" + | "it" + | "its" + | "you" + | "your" + | "we" + | "our" + | "they" + | "their" + | "to" + | "for" + | "with" + | "from" + | "in" + | "on" + | "of" + | "and" + | "or" + | "by" + | "into" + | "is" + | "are" + | "be" + | "can" + | "will" + | "has" + | "have" + | "does" + | "not" + ) + }) + .count() } fn is_eligible_passage(text: &str) -> bool { - alphabetic_word_count(text) >= MIN_PASSAGE_WORDS - && text.graphemes(true).count() >= MIN_PASSAGE_CHARS + alphabetic_word_count(text) >= MIN_PASSAGE_WORDS + && text.graphemes(true).count() >= MIN_PASSAGE_CHARS } fn classify_passage( - text: String, - eligible: bool, - minimum_ratio: f64, - detector: &LanguageDetector, - should_detect: bool, + text: String, + eligible: bool, + minimum_ratio: f64, + detector: &LanguageDetector, + should_detect: bool, ) -> Passage { - let detection = if should_detect { - detect(&text, detector) - } else { - Detection::default() - }; - Passage { - runner_up_confidence: detection.runner_up_confidence, - qualifies_as_english: eligible - && detection.english_to_best_ratio() >= minimum_ratio, - confidently_non_english: eligible - && detection.is_confidently_non_english(&text), - alphabetic_words: alphabetic_word_count(&text), - english_grammatical_words: english_grammatical_word_count(&text), - text, - language: detection.language, - confidence: detection.confidence, - eligible, - } + let detection = if should_detect { + detect(&text, detector) + } else { + Detection::default() + }; + Passage { + runner_up_confidence: detection.runner_up_confidence, + qualifies_as_english: eligible + && detection.english_to_best_ratio() >= minimum_ratio, + confidently_non_english: eligible + && detection.is_confidently_non_english(&text), + alphabetic_words: alphabetic_word_count(&text), + english_grammatical_words: english_grammatical_word_count(&text), + text, + language: detection.language, + confidence: detection.confidence, + eligible, + } } /// A foreign summary needs a contiguous English passage, not scattered product names. fn summary_rescue_passages(text: &str) -> Vec { - script_passages(text) - .into_iter() - .filter(|text| !NON_LATIN_LETTER.is_match(text)) - .flat_map(|text| { - let normalized = - text.split_whitespace().collect::>().join(" "); - let whole = summary_translation_passage(&normalized); - if whole.qualifies_as_english - || !whole.eligible - || whole.english_grammatical_words - < MIN_ENGLISH_GRAMMATICAL_WORDS - { - return vec![whole]; - } - let spans = mixed_language_passages(&normalized, &DETECTOR); - if spans.len() <= 1 { - return vec![whole]; - } - spans - .into_iter() - .map(|span| summary_translation_passage(&span)) - .collect() - }) - .collect() + script_passages(text) + .into_iter() + .filter(|text| !NON_LATIN_LETTER.is_match(text)) + .flat_map(|text| { + let normalized = + text.split_whitespace().collect::>().join(" "); + let whole = summary_translation_passage(&normalized); + if whole.qualifies_as_english + || !whole.eligible + || whole.english_grammatical_words + < MIN_ENGLISH_GRAMMATICAL_WORDS + { + return vec![whole]; + } + let spans = mixed_language_passages(&normalized, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + spans + .into_iter() + .map(|span| summary_translation_passage(&span)) + .collect() + }) + .collect() } pub(super) fn is_likely_english_summary(text: &str) -> bool { - let normalized: String = text.nfkc().collect(); - let mut passages = vec![classify_passage( - normalized.clone(), - true, - MIN_ENGLISH_TO_BEST_RATIO, - &DETECTOR, - true, - )]; - let requires_strong_rescue = passages[0].confidently_non_english; - if requires_strong_rescue { - passages.extend(summary_rescue_passages(&normalized)); - } else if NON_LATIN_LETTER.is_match(&normalized) { - let latin = NON_LATIN_LETTER.replace_all(&normalized, " "); - passages[0].qualifies_as_english = false; - passages.push(classify_passage( - latin.to_string(), - is_eligible_passage(&latin), - MIN_ENGLISH_TO_BEST_RATIO, - &DETECTOR, - is_eligible_passage(&latin), - )); - } else if !passages[0].qualifies_as_english { - passages.extend( - mixed_language_passages(&normalized, &DETECTOR) - .into_iter() - .map(|text| { - let eligible = is_eligible_passage(&text); - classify_passage( - text, - eligible, - MIN_ENGLISH_TO_BEST_RATIO, - &DETECTOR, - eligible, - ) - }), - ); - } - passages.iter().any(|passage| passage.qualifies_as_english) - || !passages - .iter() - .any(|passage| passage.confidently_non_english) + let normalized: String = text.nfkc().collect(); + let mut passages = vec![classify_passage( + normalized.clone(), + true, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + true, + )]; + let requires_strong_rescue = passages[0].confidently_non_english; + if requires_strong_rescue { + passages.extend(summary_rescue_passages(&normalized)); + } else if NON_LATIN_LETTER.is_match(&normalized) { + let latin = NON_LATIN_LETTER.replace_all(&normalized, " "); + passages[0].qualifies_as_english = false; + passages.push(classify_passage( + latin.to_string(), + is_eligible_passage(&latin), + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + is_eligible_passage(&latin), + )); + } else if !passages[0].qualifies_as_english { + passages.extend( + mixed_language_passages(&normalized, &DETECTOR) + .into_iter() + .map(|text| { + let eligible = is_eligible_passage(&text); + classify_passage( + text, + eligible, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + eligible, + ) + }), + ); + } + passages.iter().any(|passage| passage.qualifies_as_english) + || !passages + .iter() + .any(|passage| passage.confidently_non_english) } pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { - let passages = blocks - .iter() - .flat_map(|block| { - let normalized: String = block.nfkc().collect(); - description_passages(&normalized) - }) - .collect::>(); - let english_utf16_length: usize = passages - .iter() - .filter(|passage| passage.qualifies_as_english) - .map(|passage| passage.text.encode_utf16().count()) - .sum(); - let total_utf16_length: usize = passages - .iter() - .filter(|passage| passage.eligible) - .map(|passage| passage.text.encode_utf16().count()) - .sum(); - let has_foreign_evidence = passages - .iter() - .any(|passage| passage.confidently_non_english); - let english_proportion = if total_utf16_length == 0 { - 0.0 - } else { - english_utf16_length as f64 / total_utf16_length as f64 - }; + let passages = blocks + .iter() + .flat_map(|block| { + let normalized: String = block.nfkc().collect(); + description_passages(&normalized) + }) + .collect::>(); + let english_utf16_length: usize = passages + .iter() + .filter(|passage| passage.qualifies_as_english) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let total_utf16_length: usize = passages + .iter() + .filter(|passage| passage.eligible) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let has_foreign_evidence = passages + .iter() + .any(|passage| passage.confidently_non_english); + let english_proportion = if total_utf16_length == 0 { + 0.0 + } else { + english_utf16_length as f64 / total_utf16_length as f64 + }; - !has_foreign_evidence - || english_proportion >= MIN_DESCRIPTION_ENGLISH_PROPORTION + !has_foreign_evidence + || english_proportion >= MIN_DESCRIPTION_ENGLISH_PROPORTION } diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 23f14ea2ec..89b006b869 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -1,8 +1,9 @@ use serde_json::json; use super::text::{ - ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, - has_summary_formatting, is_likely_english_summary, js_string_length, + ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, + has_summary_formatting, is_confidently_non_english_short_text, + is_likely_english_summary, js_string_length, normalize_project_field_text, profanity_matches, project_requires_english, project_text_similarity, }; @@ -11,6 +12,7 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; const MIN_SUMMARY_CHARS: usize = 25; +const MIN_SUMMARY_LANGUAGE_CHARS: usize = 10; const MAX_SUMMARY_NAME_SIMILARITY: f64 = 0.8; pub(super) fn validate(project: &Project) -> Vec { @@ -121,9 +123,12 @@ pub(super) fn is_non_english(project: &Project) -> bool { } fn is_non_english_text(project: &Project, normalized_summary: &str) -> bool { - project_requires_english(project) - && js_string_length(normalized_summary) >= MIN_SUMMARY_CHARS - && !is_likely_english_summary(normalized_summary) + let length = js_string_length(normalized_summary); + project_requires_english(project) + && length >= MIN_SUMMARY_LANGUAGE_CHARS + && (length >= MIN_SUMMARY_CHARS + || is_confidently_non_english_short_text(normalized_summary)) + && !is_likely_english_summary(normalized_summary) } fn requires_language_nag( diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 479f0dff9f..6f72b14f0a 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -12,7 +12,8 @@ use crate::models::exp::minecraft::Language; use crate::models::projects::Project; pub(super) use super::language::{ - has_sufficient_english_blocks, is_likely_english_summary, + has_sufficient_english_blocks, is_confidently_non_english_short_text, + is_likely_english_summary, }; static WORD: LazyLock = @@ -982,52 +983,74 @@ fn strip_technical_syntax(line: &str) -> String { } } +fn strip_loaders(text: &str) -> String { + static LOADER_WORD: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:fabric|forge|quilt|neoforge|liteloader)\b").unwrap() + }); + + LOADER_WORD.replace_all(text, " ").into_owned() +} + fn description_language_input(markdown: &str) -> String { - let readable = strip_description_markup(markdown); - let readable = text_without_explicit_links(&readable); - readable - .lines() - .map(strip_technical_syntax) - .collect::>() - .join("\n") + let readable = strip_description_markup(markdown); + let readable = text_without_explicit_links(&readable); + let readable = strip_loaders(&readable); + readable + .lines() + .map(strip_technical_syntax) + .collect::>() + .join("\n") } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - let readable = description_language_input(markdown); - let mut blocks = Vec::new(); - let mut paragraph = Vec::new(); - let mut heading_only = false; - let mut blank = false; - for line in readable.lines() { - let line = line.trim(); - if line.is_empty() { - blank = true; - continue; - } - let heading = line.starts_with('#'); - let metadata_list = line - .split_once(':') - .is_some_and(|(_, values)| values.matches(',').count() >= 3); - if (heading - || metadata_list - || (blank - && !heading_only - && (WORD.find_iter(¶graph.join(" ")).count() >= 8 - || WORD.find_iter(line).count() >= 8))) - && !paragraph.is_empty() - { - blocks.push(extract_description_text(¶graph.join("\n"))); - paragraph.clear(); - } - paragraph.push(line); - heading_only = heading; - blank = false; - } - if !paragraph.is_empty() { - blocks.push(extract_description_text(¶graph.join("\n"))); - } - blocks.retain(|block| !block.is_empty()); - blocks + const MIN_PARAGRAPH_WORDS: usize = 8; + + let readable = description_language_input(markdown); + let mut blocks = Vec::new(); + let mut paragraph = Vec::new(); + let mut previous_line_was_heading = false; + let mut follows_blank_line = false; + + for line in readable.lines() { + let line = line.trim(); + if line.is_empty() { + follows_blank_line = true; + continue; + } + + let is_heading = line.starts_with('#'); + let starts_new_paragraph = follows_blank_line + && !previous_line_was_heading + && (has_minimum_word_count(¶graph.join(" "), MIN_PARAGRAPH_WORDS) + || has_minimum_word_count(line, MIN_PARAGRAPH_WORDS)); + + if is_heading || starts_new_paragraph { + push_description_block(&mut blocks, &mut paragraph); + } + + paragraph.push(line); + previous_line_was_heading = is_heading; + follows_blank_line = false; + } + + push_description_block(&mut blocks, &mut paragraph); + blocks +} + +fn has_minimum_word_count(text: &str, minimum: usize) -> bool { + WORD.find_iter(text).take(minimum).count() == minimum +} + +fn push_description_block(blocks: &mut Vec, paragraph: &mut Vec<&str>) { + if paragraph.is_empty() { + return; + } + + let block = extract_description_text(¶graph.join("\n")); + if !block.is_empty() { + blocks.push(block); + } + paragraph.clear(); } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { From f7ab387c273cdc46238eb0fb436d9679c124a640 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 12:27:19 -0600 Subject: [PATCH 12/42] remove outdated test --- .../validate/project/description/markdown.rs | 78 ++++++++++--------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/apps/labrinth/src/validate/project/description/markdown.rs b/apps/labrinth/src/validate/project/description/markdown.rs index 5b6ccb21c2..dc6a9a3375 100644 --- a/apps/labrinth/src/validate/project/description/markdown.rs +++ b/apps/labrinth/src/validate/project/description/markdown.rs @@ -6,10 +6,10 @@ use regex::Regex; use unicode_segmentation::UnicodeSegmentation; static HTML_HEADER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?is)]*>(.*?)").unwrap() + Regex::new(r"(?is)]*>(.*?)").unwrap() }); static ADJACENT_HTML_HEADERS: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)\s*\s* = LazyLock::new(|| { Regex::new(r"(?is)\s*(?:]*>\s*)*$").unwrap() }); @@ -129,7 +129,7 @@ impl<'a> DescriptionMarkdown<'a> { let [previous, current] = headings else { return false; }; - previous.level == current.level + previous.level == current.level && self.markdown[previous.range.end..current.range.start] .trim() .is_empty() @@ -242,37 +242,43 @@ image: "![](/missing-alt.png)" ); } - #[test] - fn all_heading_levels_are_validated() { - for level in 1..=6 { - let prefix = "#".repeat(level); - let long_text = "heading ".repeat(12); - for (long_header, adjacent_headers, separated_headers) in [ - ( - format!("{prefix} {long_text}"), - format!("{prefix} First\n\n{prefix} Second"), - format!("{prefix} First\n\n```yaml\n# comment\n```\n\n{prefix} Second"), - ), - ( - format!("{long_text}"), - format!("First\n\nSecond"), - format!("First\n\n```yaml\n# comment\n```\n\nSecond"), - ), - ] { - let markdown = DescriptionMarkdown::parse(&long_header); - assert_eq!(markdown.long_header_count(), 1, "{long_header}"); - assert!(markdown.ends_with_header(), "{long_header}"); - assert!( - DescriptionMarkdown::parse(&adjacent_headers) - .has_adjacent_same_level_headers(), - "{adjacent_headers}" - ); - assert!( - !DescriptionMarkdown::parse(&separated_headers) - .has_adjacent_same_level_headers(), - "{separated_headers}" - ); - } - } - } + #[test] + fn all_heading_levels_are_validated() { + for level in 1..=6 { + let prefix = "#".repeat(level); + let long_text = "heading ".repeat(12); + for (long_header, adjacent_headers, separated_headers) in [ + ( + format!("{prefix} {long_text}"), + format!("{prefix} First\n\n{prefix} Second"), + format!( + "{prefix} First\n\n```yaml\n# comment\n```\n\n{prefix} Second" + ), + ), + ( + format!("{long_text}"), + format!( + "First\n\nSecond" + ), + format!( + "First\n\n```yaml\n# comment\n```\n\nSecond" + ), + ), + ] { + let markdown = DescriptionMarkdown::parse(&long_header); + assert_eq!(markdown.long_header_count(), 1, "{long_header}"); + assert!(markdown.ends_with_header(), "{long_header}"); + assert!( + DescriptionMarkdown::parse(&adjacent_headers) + .has_adjacent_same_level_headers(), + "{adjacent_headers}" + ); + assert!( + !DescriptionMarkdown::parse(&separated_headers) + .has_adjacent_same_level_headers(), + "{separated_headers}" + ); + } + } + } } From ca1eafe2870909a8f80c27527b87f57369d7c533 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 13:16:42 -0600 Subject: [PATCH 13/42] fmt --- apps/labrinth/src/validate/project/mod.rs | 2 +- apps/labrinth/src/validate/project/summary.rs | 21 ++- apps/labrinth/src/validate/project/text.rs | 129 +++++++++--------- apps/labrinth/tests/project.rs | 2 +- 4 files changed, 78 insertions(+), 76 deletions(-) diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index a9f94f6242..6151a6c8eb 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -69,7 +69,7 @@ pub enum ProjectNagKind { ProjectDescriptionProfanity, ProjectDescriptionNonStandardText, ProjectDescriptionNonEnglish, - ProjectDescriptionMatchesSummary, + ProjectDescriptionMatchesSummary, AddDescription, DescriptionTooShort, ProjectDescriptionSpam, diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 89b006b869..5037fbbcc6 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -1,11 +1,10 @@ use serde_json::json; use super::text::{ - ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, - has_summary_formatting, is_confidently_non_english_short_text, - is_likely_english_summary, js_string_length, - normalize_project_field_text, profanity_matches, project_requires_english, - project_text_similarity, + ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, + has_summary_formatting, is_confidently_non_english_short_text, + is_likely_english_summary, js_string_length, normalize_project_field_text, + profanity_matches, project_requires_english, project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; @@ -123,12 +122,12 @@ pub(super) fn is_non_english(project: &Project) -> bool { } fn is_non_english_text(project: &Project, normalized_summary: &str) -> bool { - let length = js_string_length(normalized_summary); - project_requires_english(project) - && length >= MIN_SUMMARY_LANGUAGE_CHARS - && (length >= MIN_SUMMARY_CHARS - || is_confidently_non_english_short_text(normalized_summary)) - && !is_likely_english_summary(normalized_summary) + let length = js_string_length(normalized_summary); + project_requires_english(project) + && length >= MIN_SUMMARY_LANGUAGE_CHARS + && (length >= MIN_SUMMARY_CHARS + || is_confidently_non_english_short_text(normalized_summary)) + && !is_likely_english_summary(normalized_summary) } fn requires_language_nag( diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 6f72b14f0a..710c5365d0 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -12,8 +12,8 @@ use crate::models::exp::minecraft::Language; use crate::models::projects::Project; pub(super) use super::language::{ - has_sufficient_english_blocks, is_confidently_non_english_short_text, - is_likely_english_summary, + has_sufficient_english_blocks, is_confidently_non_english_short_text, + is_likely_english_summary, }; static WORD: LazyLock = @@ -955,7 +955,7 @@ fn strip_dotted_identifiers(text: &str) -> String { .unwrap() }); - IDENTIFIER.replace_all(text, " ").into_owned() + IDENTIFIER.replace_all(text, " ").into_owned() } fn strip_command_arguments(text: &str) -> String { @@ -964,7 +964,7 @@ fn strip_command_arguments(text: &str) -> String { Regex::new(r"\[[A-Za-z][A-Za-z0-9_ /|.-]*\]").unwrap() }); - ARGUMENT.replace_all(text, " ").into_owned() + ARGUMENT.replace_all(text, " ").into_owned() } fn strip_technical_syntax(line: &str) -> String { @@ -976,81 +976,84 @@ fn strip_technical_syntax(line: &str) -> String { let without_commands = COMMAND.replace_all(line, "$1"); let without_identifiers = strip_dotted_identifiers(&without_commands); - if COMMAND.is_match(line) { - strip_command_arguments(&without_identifiers) - } else { - without_identifiers - } + if COMMAND.is_match(line) { + strip_command_arguments(&without_identifiers) + } else { + without_identifiers + } } fn strip_loaders(text: &str) -> String { - static LOADER_WORD: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)\b(?:fabric|forge|quilt|neoforge|liteloader)\b").unwrap() - }); + static LOADER_WORD: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:fabric|forge|quilt|neoforge|liteloader)\b") + .unwrap() + }); - LOADER_WORD.replace_all(text, " ").into_owned() + LOADER_WORD.replace_all(text, " ").into_owned() } fn description_language_input(markdown: &str) -> String { - let readable = strip_description_markup(markdown); - let readable = text_without_explicit_links(&readable); - let readable = strip_loaders(&readable); - readable - .lines() - .map(strip_technical_syntax) - .collect::>() - .join("\n") + let readable = strip_description_markup(markdown); + let readable = text_without_explicit_links(&readable); + let readable = strip_loaders(&readable); + readable + .lines() + .map(strip_technical_syntax) + .collect::>() + .join("\n") } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - const MIN_PARAGRAPH_WORDS: usize = 8; - - let readable = description_language_input(markdown); - let mut blocks = Vec::new(); - let mut paragraph = Vec::new(); - let mut previous_line_was_heading = false; - let mut follows_blank_line = false; - - for line in readable.lines() { - let line = line.trim(); - if line.is_empty() { - follows_blank_line = true; - continue; - } - - let is_heading = line.starts_with('#'); - let starts_new_paragraph = follows_blank_line - && !previous_line_was_heading - && (has_minimum_word_count(¶graph.join(" "), MIN_PARAGRAPH_WORDS) - || has_minimum_word_count(line, MIN_PARAGRAPH_WORDS)); - - if is_heading || starts_new_paragraph { - push_description_block(&mut blocks, &mut paragraph); - } - - paragraph.push(line); - previous_line_was_heading = is_heading; - follows_blank_line = false; - } - - push_description_block(&mut blocks, &mut paragraph); - blocks + const MIN_PARAGRAPH_WORDS: usize = 8; + + let readable = description_language_input(markdown); + let mut blocks = Vec::new(); + let mut paragraph = Vec::new(); + let mut previous_line_was_heading = false; + let mut follows_blank_line = false; + + for line in readable.lines() { + let line = line.trim(); + if line.is_empty() { + follows_blank_line = true; + continue; + } + + let is_heading = line.starts_with('#'); + let starts_new_paragraph = follows_blank_line + && !previous_line_was_heading + && (has_minimum_word_count( + ¶graph.join(" "), + MIN_PARAGRAPH_WORDS, + ) || has_minimum_word_count(line, MIN_PARAGRAPH_WORDS)); + + if is_heading || starts_new_paragraph { + push_description_block(&mut blocks, &mut paragraph); + } + + paragraph.push(line); + previous_line_was_heading = is_heading; + follows_blank_line = false; + } + + push_description_block(&mut blocks, &mut paragraph); + blocks } fn has_minimum_word_count(text: &str, minimum: usize) -> bool { - WORD.find_iter(text).take(minimum).count() == minimum + WORD.find_iter(text).take(minimum).count() == minimum } fn push_description_block(blocks: &mut Vec, paragraph: &mut Vec<&str>) { - if paragraph.is_empty() { - return; - } - - let block = extract_description_text(¶graph.join("\n")); - if !block.is_empty() { - blocks.push(block); - } - paragraph.clear(); + if paragraph.is_empty() { + return; + } + + let block = extract_description_text(¶graph.join("\n")); + if !block.is_empty() { + blocks.push(block); + } + paragraph.clear(); } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 3e7373dc4c..e58724a159 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -660,7 +660,7 @@ async fn test_leaving_review_skips_validation() { #[actix_rt::test] async fn test_description_similarity_to_summary() { - with_test_environment( + with_test_environment( None, |test_env: TestEnvironment| async move { let api = &test_env.api; From e55b334d602a1f5e099da3eadcd04f06ca7c686b Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 13:30:20 -0600 Subject: [PATCH 14/42] pnpm prepr --- packages/moderation/src/locales/en-US/index.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/moderation/src/locales/en-US/index.json b/packages/moderation/src/locales/en-US/index.json index be9d0e4a4c..14d6995208 100644 --- a/packages/moderation/src/locales/en-US/index.json +++ b/packages/moderation/src/locales/en-US/index.json @@ -197,6 +197,9 @@ "nags.project-description-banned-link.description": { "defaultMessage": "The link “{fullUrl}” is not allowed in project descriptions." }, + "nags.project-description-content.title": { + "defaultMessage": "Review the description" + }, "nags.project-description-non-english.description": { "defaultMessage": "Your project's description must be written in English or include an English translation." }, @@ -380,6 +383,9 @@ "nags.visit-links-settings.title": { "defaultMessage": "Visit links settings" }, + "project.text-validation.description-matches-summary": { + "defaultMessage": "Your project's description should provide unique information and not repeat the project's summary." + }, "project.text-validation.summary-matches-title": { "defaultMessage": "Your project's summary should provide unique information and not repeat the project's name." }, From 422fbdc57cfbeac1f1c0bb2d712af6a7288ebb2c Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:53:18 +0400 Subject: [PATCH 15/42] performance fix + all tags selected fix --- apps/labrinth/src/validate/project/tags.rs | 88 ++++++++++++++++++---- apps/labrinth/src/validate/project/text.rs | 16 ++-- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/apps/labrinth/src/validate/project/tags.rs b/apps/labrinth/src/validate/project/tags.rs index fda8188387..db5fe8b867 100644 --- a/apps/labrinth/src/validate/project/tags.rs +++ b/apps/labrinth/src/validate/project/tags.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use crate::{ database::models::categories::Category, models::{projects::Project, v2::projects::LegacyProject}, @@ -80,23 +82,77 @@ pub(super) fn validate( ); } - if let Some(available_categories) = available_categories { - let total_available_tags = available_categories - .iter() - .filter(|category| category.project_type == project_type) - .count(); - if total_available_tags > 0 && tag_count == total_available_tags { - nags.push( - ProjectNag::new( - ProjectNagKind::AllTagsSelected, - ProjectNagSeverity::Required, - ) - .with_details(serde_json::json!({ - "total_available_tags": total_available_tags, - })), - ); - } + if let Some(available_categories) = available_categories + && let Some(total_available_tags) = all_available_tags_selected( + project + .categories + .iter() + .chain(&project.additional_categories) + .map(String::as_str), + available_categories + .iter() + .filter(|category| category.project_type == project_type) + .map(|category| category.category.as_str()), + ) + { + nags.push( + ProjectNag::new( + ProjectNagKind::AllTagsSelected, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ + "total_available_tags": total_available_tags, + })), + ); } nags } + +fn all_available_tags_selected<'a, 'b>( + selected_tags: impl Iterator, + available_tags: impl Iterator, +) -> Option { + let selected_tags = selected_tags.collect::>(); + let available_tags = available_tags.collect::>(); + + (!available_tags.is_empty() && available_tags.is_subset(&selected_tags)) + .then_some(available_tags.len()) +} + +#[cfg(test)] +mod tests { + use super::all_available_tags_selected; + + #[test] + fn all_available_tags_are_compared_by_value() { + assert_eq!( + all_available_tags_selected( + ["combat", "magic", "mobs"].into_iter(), + ["combat", "magic", "mobs"].into_iter(), + ), + Some(3) + ); + assert_eq!( + all_available_tags_selected( + ["combat", "magic", "modpack-exclusive"].into_iter(), + ["combat", "magic", "mobs"].into_iter(), + ), + None + ); + assert_eq!( + all_available_tags_selected( + ["combat", "magic", "mobs", "modpack-exclusive"].into_iter(), + ["combat", "magic", "mobs"].into_iter(), + ), + Some(3) + ); + assert_eq!( + all_available_tags_selected( + ["modpack-exclusive"].into_iter(), + std::iter::empty(), + ), + None + ); + } +} diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 710c5365d0..338e43a674 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -1009,6 +1009,7 @@ pub(super) fn extract_description_blocks(markdown: &str) -> Vec { let readable = description_language_input(markdown); let mut blocks = Vec::new(); let mut paragraph = Vec::new(); + let mut paragraph_word_count = 0; let mut previous_line_was_heading = false; let mut follows_blank_line = false; @@ -1020,18 +1021,21 @@ pub(super) fn extract_description_blocks(markdown: &str) -> Vec { } let is_heading = line.starts_with('#'); + let line_word_count = + WORD.find_iter(line).take(MIN_PARAGRAPH_WORDS).count(); let starts_new_paragraph = follows_blank_line && !previous_line_was_heading - && (has_minimum_word_count( - ¶graph.join(" "), - MIN_PARAGRAPH_WORDS, - ) || has_minimum_word_count(line, MIN_PARAGRAPH_WORDS)); + && (paragraph_word_count >= MIN_PARAGRAPH_WORDS + || line_word_count >= MIN_PARAGRAPH_WORDS); if is_heading || starts_new_paragraph { push_description_block(&mut blocks, &mut paragraph); + paragraph_word_count = 0; } paragraph.push(line); + paragraph_word_count = + (paragraph_word_count + line_word_count).min(MIN_PARAGRAPH_WORDS); previous_line_was_heading = is_heading; follows_blank_line = false; } @@ -1040,10 +1044,6 @@ pub(super) fn extract_description_blocks(markdown: &str) -> Vec { blocks } -fn has_minimum_word_count(text: &str, minimum: usize) -> bool { - WORD.find_iter(text).take(minimum).count() == minimum -} - fn push_description_block(blocks: &mut Vec, paragraph: &mut Vec<&str>) { if paragraph.is_empty() { return; From b5137d52a230f1671f45432193de9ef8a499facc Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 15 Sep 2026 13:16:15 -0600 Subject: [PATCH 16/42] fmt --- Cargo.toml | 146 +++++++++++++++++-------------- apps/daedalus_client/Cargo.toml | 6 +- apps/labrinth/Cargo.toml | 27 +++++- packages/app-lib/Cargo.toml | 10 ++- packages/sqlx-tracing/Cargo.toml | 7 +- packages/xredis/Cargo.toml | 7 +- 6 files changed, 126 insertions(+), 77 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aa2fc058bd..108713bb13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,22 +1,22 @@ [workspace] resolver = "2" members = [ - "apps/app", - "apps/app-playground", - "apps/daedalus_client", - "apps/labrinth", - "packages/app-lib", - "packages/ariadne", - "packages/component-derive", - "packages/daedalus", - "packages/modrinth-content-management", - "packages/modrinth-log", - "packages/modrinth-maxmind", - "packages/modrinth-util", - "packages/neverbounce", - "packages/path-util", - "packages/serde-binhum", - "packages/xredis", + "apps/app", + "apps/app-playground", + "apps/daedalus_client", + "apps/labrinth", + "packages/app-lib", + "packages/ariadne", + "packages/component-derive", + "packages/daedalus", + "packages/modrinth-content-management", + "packages/modrinth-log", + "packages/modrinth-maxmind", + "packages/modrinth-util", + "packages/neverbounce", + "packages/path-util", + "packages/serde-binhum", + "packages/xredis", ] [workspace.package] @@ -39,28 +39,35 @@ ariadne = { path = "packages/ariadne" } async-minecraft-ping = { path = "packages/async-minecraft-ping" } async-recursion = "1.1.1" async-stripe = { version = "0.41.0", default-features = false, features = [ - "runtime-tokio-hyper-rustls", + "runtime-tokio-hyper-rustls", ] } async-trait = "0.1.89" -async-tungstenite = { version = "0.31.0", default-features = false, features = ["futures-03-sink"] } +async-tungstenite = { version = "0.31.0", default-features = false, features = [ + "futures-03-sink" +] } async-walkdir = "2.1.0" async_zip = "0.0.18" aws-sdk-s3 = { version = "=1.122.0", default-features = false, features = [ - "default-https-client", - "rt-tokio", - "rustls", + "default-https-client", + "rt-tokio", + "rustls", ] } base64 = "0.22.1" bitflags = "2.9.4" bon = "3.9.3" bytemuck = "1.24.0" bytes = "1.10.1" -cel = { version = "0.14.0", default-features = false, features = ["json", "regex"] } +cel = { version = "0.14.0", default-features = false, features = [ + "json", + "regex" +] } censor = "0.3.0" chardetng = "0.1.17" chrono = "0.4.42" chumsky = "0.9.3" -cidre = { version = "0.15.0", default-features = false, features = ["macos_15_0"] } +cidre = { version = "0.15.0", default-features = false, features = [ + "macos_15_0" +] } clap = "4.5.48" clickhouse = "0.14.0" color-eyre = "0.6.5" @@ -99,10 +106,10 @@ hmac = "0.12.1" httpdate = "1.0.3" hyper = "1.7.0" hyper-rustls = { version = "0.27.7", default-features = false, features = [ - "aws-lc-rs", - "http1", - "native-tokio", - "tls12", + "aws-lc-rs", + "http1", + "native-tokio", + "tls12", ] } hyper-util = "0.1.17" iana-time-zone = "0.1.64" @@ -114,23 +121,23 @@ jemalloc_pprof = "0.8.1" json-patch = { version = "4.1.0", default-features = false } json5 = "1.3.1" lettre = { version = "0.11.19", default-features = false, features = [ - "aws-lc-rs", - "builder", - "hostname", - "pool", - "rustls", - "rustls-native-certs", - "smtp-transport", - "tokio1", - "tokio1-rustls", + "aws-lc-rs", + "builder", + "hostname", + "pool", + "rustls", + "rustls-native-certs", + "smtp-transport", + "tokio1", + "tokio1-rustls", ] } lingua = "=1.8.0" linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ - "checked-decode", - "safe-decode", - "safe-encode", - "std", + "checked-decode", + "safe-decode", + "safe-encode", + "std", ] } maxminddb = "0.26.0" md5 = "0.8.0" @@ -160,19 +167,22 @@ pulldown-cmark = { version = "0.13.4", default-features = false } quartz_nbt = "0.2.9" quick-xml = "0.38.3" quote = { version = "1.0" } -rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 -rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 +rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 +rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 rdkafka = { version = "0.36.2", features = ["cmake-build"] } redis = "1.4.1" regex = "1.12.2" reqwest = { version = "0.12.24", default-features = false } rgb = "0.8.52" -rust_decimal = { version = "1.39.0", features = ["serde-with-float", "serde-with-str"] } +rust_decimal = { version = "1.39.0", features = [ + "serde-with-float", + "serde-with-str" +] } rust_iso3166 = "0.1.14" rust-s3 = { version = "0.37.0", default-features = false, features = [ - "fail-on-err", - "tags", - "tokio-rustls-tls", + "fail-on-err", + "tags", + "tokio-rustls-tls", ] } rustls = "0.23.32" rustrict = { version = "0.7.39", default-features = false, features = ["censor"] } @@ -181,12 +191,12 @@ same-file = "1.0.6" scalar_api_reference = { version = "0.2.2", default-features = false } secrecy = "0.10.3" sentry = { version = "0.45.0", default-features = false, features = [ - "backtrace", - "contexts", - "debug-images", - "panic", - "reqwest", - "rustls", + "backtrace", + "contexts", + "debug-images", + "panic", + "reqwest", + "rustls", ] } serde = "1.0.228" serde-binhum = { path = "packages/serde-binhum" } @@ -217,8 +227,8 @@ tauri-plugin-opener = "2.5.0" tauri-plugin-os = "2.3.1" tauri-plugin-single-instance = "2.3.4" tauri-plugin-updater = { git = "https://github.com/modrinth/plugins-workspace", rev = "0d30f2aa28ec668ce187d527da1c475da3c01cbc", default-features = false, features = [ - "rustls-tls", - "zip", + "rustls-tls", + "zip", ] } tauri-plugin-window-state = "2.4.0" tempfile = "3.23.0" @@ -248,20 +258,20 @@ validator = "0.20.0" webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" webp = { version = "0.3.1", default-features = false } -webview2-com = "0.38.0" # Should be updated in lockstep with wry +webview2-com = "0.38.0" # Should be updated in lockstep with wry whoami = "1.6.1" -windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 -windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 +windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 +windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 winreg = "0.55.0" woothee = "0.13.0" xredis = { path = "packages/xredis" } yaserde = "0.12.0" zbus = "5.11.0" zip = { version = "6.0.0", default-features = false, features = [ - "bzip2", - "deflate", - "deflate64", - "zstd", + "bzip2", + "deflate", + "deflate64", + "zstd", ] } zxcvbn = "3.1.0" @@ -319,15 +329,15 @@ opt-level = 3 # Optimize for speed and reduce size on release builds [profile.release] -opt-level = "s" # Optimize for binary size -strip = true # Remove debug symbols -lto = true # Enables link to optimizations -panic = "abort" # Strip expensive panic clean-up logic +opt-level = "s" # Optimize for binary size +strip = true # Remove debug symbols +lto = true # Enables link to optimizations +panic = "abort" # Strip expensive panic clean-up logic # Specific profile for labrinth production builds [profile.release-labrinth] inherits = "release" opt-level = 2 -strip = false # Keep debug symbols for Sentry -lto = "thin" # Enable LTO but keep compile times reasonable -panic = "unwind" # Don't exit the whole app on panic in production +strip = false # Keep debug symbols for Sentry +lto = "thin" # Enable LTO but keep compile times reasonable +panic = "unwind" # Don't exit the whole app on panic in production diff --git a/apps/daedalus_client/Cargo.toml b/apps/daedalus_client/Cargo.toml index af67313d83..3894856642 100644 --- a/apps/daedalus_client/Cargo.toml +++ b/apps/daedalus_client/Cargo.toml @@ -22,7 +22,11 @@ indexmap = { workspace = true, features = ["serde"] } itertools = { workspace = true } md5 = { workspace = true } quick-xml = { workspace = true } -reqwest = { workspace = true, features = ["json", "rustls-tls-native-roots", "stream"] } +reqwest = { workspace = true, features = [ + "json", + "rustls-tls-native-roots", + "stream" +] } rust-s3 = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 9f8599feb5..78c25a0fd5 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -23,7 +23,12 @@ arc-swap = { workspace = true } argon2 = { workspace = true } ariadne = { workspace = true } async-minecraft-ping = { workspace = true, features = ["srv"] } -async-stripe = { workspace = true, features = ["billing", "checkout", "connect", "webhook-events"] } +async-stripe = { workspace = true, features = [ + "billing", + "checkout", + "connect", + "webhook-events" +] } async-trait = { workspace = true } aws-sdk-s3 = { workspace = true } base64 = { workspace = true } @@ -89,10 +94,24 @@ quick-xml = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } rdkafka = { workspace = true } -redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "r2d2", "tokio-comp"] } +redis = { workspace = true, features = [ + "ahash", + "cluster", + "cluster-async", + "r2d2", + "tokio-comp" +] } regex = { workspace = true } -reqwest = { workspace = true, features = ["http2", "json", "multipart", "rustls-tls-webpki-roots"] } -rust_decimal = { workspace = true, features = ["serde-with-float", "serde-with-str"] } +reqwest = { workspace = true, features = [ + "http2", + "json", + "multipart", + "rustls-tls-webpki-roots" +] } +rust_decimal = { workspace = true, features = [ + "serde-with-float", + "serde-with-str" +] } rust_iso3166 = { workspace = true } rustls.workspace = true rustrict = { workspace = true } diff --git a/packages/app-lib/Cargo.toml b/packages/app-lib/Cargo.toml index 9bb14ce8a4..81b5cc74d4 100644 --- a/packages/app-lib/Cargo.toml +++ b/packages/app-lib/Cargo.toml @@ -13,7 +13,10 @@ ariadne = { workspace = true } async-minecraft-ping = { workspace = true, features = ["srv"] } async-recursion = { workspace = true } async-trait = { workspace = true } -async-tungstenite = { workspace = true, features = ["tokio-runtime", "tokio-rustls-webpki-roots"] } +async-tungstenite = { workspace = true, features = [ + "tokio-runtime", + "tokio-rustls-webpki-roots" +] } async-walkdir = { workspace = true } async_zip = { workspace = true, features = [ "bzip2", @@ -136,7 +139,10 @@ dunce = { workspace = true } cidre = { workspace = true, features = ["blocks", "nw"] } [target."cfg(windows)".dependencies] -windows = { workspace = true, features = ["Networking_Connectivity", "Win32_Storage_FileSystem"] } +windows = { workspace = true, features = [ + "Networking_Connectivity", + "Win32_Storage_FileSystem" +] } windows-core = { workspace = true } winreg = { workspace = true } diff --git a/packages/sqlx-tracing/Cargo.toml b/packages/sqlx-tracing/Cargo.toml index 3ce51c659a..bf89e32aa8 100644 --- a/packages/sqlx-tracing/Cargo.toml +++ b/packages/sqlx-tracing/Cargo.toml @@ -10,7 +10,12 @@ repository = "https://github.com/jdrouet/sqlx-tracing" license = "MIT" # authors = ["Jérémie Drouet "] # deprecated field, Tombi warns keywords = ["database", "observability", "opentelemetry", "sqlx", "tracing"] -categories = ["asynchronous", "database", "development-tools::debugging", "development-tools::profiling"] +categories = [ + "asynchronous", + "database", + "development-tools::debugging", + "development-tools::profiling" +] [dependencies] derive_more = { workspace = true, features = ["deref", "deref_mut"] } diff --git a/packages/xredis/Cargo.toml b/packages/xredis/Cargo.toml index 14a698c047..cc87779098 100644 --- a/packages/xredis/Cargo.toml +++ b/packages/xredis/Cargo.toml @@ -15,7 +15,12 @@ futures = { workspace = true } lz4_flex = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } -redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "tokio-comp"] } +redis = { workspace = true, features = [ + "ahash", + "cluster", + "cluster-async", + "tokio-comp" +] } serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "sync", "time"] } From 5100b9b53bee1f7dc63575bd0357243b7458f10a Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 15 Sep 2026 13:20:31 -0600 Subject: [PATCH 17/42] fmt 2 --- Cargo.toml | 18 ++++-------------- apps/daedalus_client/Cargo.toml | 6 +----- apps/labrinth/Cargo.toml | 27 ++++----------------------- packages/app-lib/Cargo.toml | 10 ++-------- packages/sqlx-tracing/Cargo.toml | 7 +------ packages/xredis/Cargo.toml | 7 +------ 6 files changed, 13 insertions(+), 62 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 108713bb13..ca056543e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,9 +42,7 @@ async-stripe = { version = "0.41.0", default-features = false, features = [ "runtime-tokio-hyper-rustls", ] } async-trait = "0.1.89" -async-tungstenite = { version = "0.31.0", default-features = false, features = [ - "futures-03-sink" -] } +async-tungstenite = { version = "0.31.0", default-features = false, features = ["futures-03-sink"] } async-walkdir = "2.1.0" async_zip = "0.0.18" aws-sdk-s3 = { version = "=1.122.0", default-features = false, features = [ @@ -57,17 +55,12 @@ bitflags = "2.9.4" bon = "3.9.3" bytemuck = "1.24.0" bytes = "1.10.1" -cel = { version = "0.14.0", default-features = false, features = [ - "json", - "regex" -] } +cel = { version = "0.14.0", default-features = false, features = ["json", "regex"] } censor = "0.3.0" chardetng = "0.1.17" chrono = "0.4.42" chumsky = "0.9.3" -cidre = { version = "0.15.0", default-features = false, features = [ - "macos_15_0" -] } +cidre = { version = "0.15.0", default-features = false, features = ["macos_15_0"] } clap = "4.5.48" clickhouse = "0.14.0" color-eyre = "0.6.5" @@ -174,10 +167,7 @@ redis = "1.4.1" regex = "1.12.2" reqwest = { version = "0.12.24", default-features = false } rgb = "0.8.52" -rust_decimal = { version = "1.39.0", features = [ - "serde-with-float", - "serde-with-str" -] } +rust_decimal = { version = "1.39.0", features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = "0.1.14" rust-s3 = { version = "0.37.0", default-features = false, features = [ "fail-on-err", diff --git a/apps/daedalus_client/Cargo.toml b/apps/daedalus_client/Cargo.toml index 3894856642..af67313d83 100644 --- a/apps/daedalus_client/Cargo.toml +++ b/apps/daedalus_client/Cargo.toml @@ -22,11 +22,7 @@ indexmap = { workspace = true, features = ["serde"] } itertools = { workspace = true } md5 = { workspace = true } quick-xml = { workspace = true } -reqwest = { workspace = true, features = [ - "json", - "rustls-tls-native-roots", - "stream" -] } +reqwest = { workspace = true, features = ["json", "rustls-tls-native-roots", "stream"] } rust-s3 = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 78c25a0fd5..9f8599feb5 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -23,12 +23,7 @@ arc-swap = { workspace = true } argon2 = { workspace = true } ariadne = { workspace = true } async-minecraft-ping = { workspace = true, features = ["srv"] } -async-stripe = { workspace = true, features = [ - "billing", - "checkout", - "connect", - "webhook-events" -] } +async-stripe = { workspace = true, features = ["billing", "checkout", "connect", "webhook-events"] } async-trait = { workspace = true } aws-sdk-s3 = { workspace = true } base64 = { workspace = true } @@ -94,24 +89,10 @@ quick-xml = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } rdkafka = { workspace = true } -redis = { workspace = true, features = [ - "ahash", - "cluster", - "cluster-async", - "r2d2", - "tokio-comp" -] } +redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "r2d2", "tokio-comp"] } regex = { workspace = true } -reqwest = { workspace = true, features = [ - "http2", - "json", - "multipart", - "rustls-tls-webpki-roots" -] } -rust_decimal = { workspace = true, features = [ - "serde-with-float", - "serde-with-str" -] } +reqwest = { workspace = true, features = ["http2", "json", "multipart", "rustls-tls-webpki-roots"] } +rust_decimal = { workspace = true, features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = { workspace = true } rustls.workspace = true rustrict = { workspace = true } diff --git a/packages/app-lib/Cargo.toml b/packages/app-lib/Cargo.toml index 81b5cc74d4..9bb14ce8a4 100644 --- a/packages/app-lib/Cargo.toml +++ b/packages/app-lib/Cargo.toml @@ -13,10 +13,7 @@ ariadne = { workspace = true } async-minecraft-ping = { workspace = true, features = ["srv"] } async-recursion = { workspace = true } async-trait = { workspace = true } -async-tungstenite = { workspace = true, features = [ - "tokio-runtime", - "tokio-rustls-webpki-roots" -] } +async-tungstenite = { workspace = true, features = ["tokio-runtime", "tokio-rustls-webpki-roots"] } async-walkdir = { workspace = true } async_zip = { workspace = true, features = [ "bzip2", @@ -139,10 +136,7 @@ dunce = { workspace = true } cidre = { workspace = true, features = ["blocks", "nw"] } [target."cfg(windows)".dependencies] -windows = { workspace = true, features = [ - "Networking_Connectivity", - "Win32_Storage_FileSystem" -] } +windows = { workspace = true, features = ["Networking_Connectivity", "Win32_Storage_FileSystem"] } windows-core = { workspace = true } winreg = { workspace = true } diff --git a/packages/sqlx-tracing/Cargo.toml b/packages/sqlx-tracing/Cargo.toml index bf89e32aa8..3ce51c659a 100644 --- a/packages/sqlx-tracing/Cargo.toml +++ b/packages/sqlx-tracing/Cargo.toml @@ -10,12 +10,7 @@ repository = "https://github.com/jdrouet/sqlx-tracing" license = "MIT" # authors = ["Jérémie Drouet "] # deprecated field, Tombi warns keywords = ["database", "observability", "opentelemetry", "sqlx", "tracing"] -categories = [ - "asynchronous", - "database", - "development-tools::debugging", - "development-tools::profiling" -] +categories = ["asynchronous", "database", "development-tools::debugging", "development-tools::profiling"] [dependencies] derive_more = { workspace = true, features = ["deref", "deref_mut"] } diff --git a/packages/xredis/Cargo.toml b/packages/xredis/Cargo.toml index cc87779098..14a698c047 100644 --- a/packages/xredis/Cargo.toml +++ b/packages/xredis/Cargo.toml @@ -15,12 +15,7 @@ futures = { workspace = true } lz4_flex = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } -redis = { workspace = true, features = [ - "ahash", - "cluster", - "cluster-async", - "tokio-comp" -] } +redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "tokio-comp"] } serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "sync", "time"] } From 6fb84329c653f9c28729849e08548c6db736df64 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:22:07 +0100 Subject: [PATCH 18/42] Ignore some nags inside code blocks --- Cargo.toml | 128 +++++++++++++++++++-------------------- apps/labrinth/Cargo.toml | 1 + 2 files changed, 65 insertions(+), 64 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ca056543e7..aa2fc058bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,22 +1,22 @@ [workspace] resolver = "2" members = [ - "apps/app", - "apps/app-playground", - "apps/daedalus_client", - "apps/labrinth", - "packages/app-lib", - "packages/ariadne", - "packages/component-derive", - "packages/daedalus", - "packages/modrinth-content-management", - "packages/modrinth-log", - "packages/modrinth-maxmind", - "packages/modrinth-util", - "packages/neverbounce", - "packages/path-util", - "packages/serde-binhum", - "packages/xredis", + "apps/app", + "apps/app-playground", + "apps/daedalus_client", + "apps/labrinth", + "packages/app-lib", + "packages/ariadne", + "packages/component-derive", + "packages/daedalus", + "packages/modrinth-content-management", + "packages/modrinth-log", + "packages/modrinth-maxmind", + "packages/modrinth-util", + "packages/neverbounce", + "packages/path-util", + "packages/serde-binhum", + "packages/xredis", ] [workspace.package] @@ -39,16 +39,16 @@ ariadne = { path = "packages/ariadne" } async-minecraft-ping = { path = "packages/async-minecraft-ping" } async-recursion = "1.1.1" async-stripe = { version = "0.41.0", default-features = false, features = [ - "runtime-tokio-hyper-rustls", + "runtime-tokio-hyper-rustls", ] } async-trait = "0.1.89" async-tungstenite = { version = "0.31.0", default-features = false, features = ["futures-03-sink"] } async-walkdir = "2.1.0" async_zip = "0.0.18" aws-sdk-s3 = { version = "=1.122.0", default-features = false, features = [ - "default-https-client", - "rt-tokio", - "rustls", + "default-https-client", + "rt-tokio", + "rustls", ] } base64 = "0.22.1" bitflags = "2.9.4" @@ -99,10 +99,10 @@ hmac = "0.12.1" httpdate = "1.0.3" hyper = "1.7.0" hyper-rustls = { version = "0.27.7", default-features = false, features = [ - "aws-lc-rs", - "http1", - "native-tokio", - "tls12", + "aws-lc-rs", + "http1", + "native-tokio", + "tls12", ] } hyper-util = "0.1.17" iana-time-zone = "0.1.64" @@ -114,23 +114,23 @@ jemalloc_pprof = "0.8.1" json-patch = { version = "4.1.0", default-features = false } json5 = "1.3.1" lettre = { version = "0.11.19", default-features = false, features = [ - "aws-lc-rs", - "builder", - "hostname", - "pool", - "rustls", - "rustls-native-certs", - "smtp-transport", - "tokio1", - "tokio1-rustls", + "aws-lc-rs", + "builder", + "hostname", + "pool", + "rustls", + "rustls-native-certs", + "smtp-transport", + "tokio1", + "tokio1-rustls", ] } lingua = "=1.8.0" linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ - "checked-decode", - "safe-decode", - "safe-encode", - "std", + "checked-decode", + "safe-decode", + "safe-encode", + "std", ] } maxminddb = "0.26.0" md5 = "0.8.0" @@ -160,8 +160,8 @@ pulldown-cmark = { version = "0.13.4", default-features = false } quartz_nbt = "0.2.9" quick-xml = "0.38.3" quote = { version = "1.0" } -rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 -rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 +rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 +rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 rdkafka = { version = "0.36.2", features = ["cmake-build"] } redis = "1.4.1" regex = "1.12.2" @@ -170,9 +170,9 @@ rgb = "0.8.52" rust_decimal = { version = "1.39.0", features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = "0.1.14" rust-s3 = { version = "0.37.0", default-features = false, features = [ - "fail-on-err", - "tags", - "tokio-rustls-tls", + "fail-on-err", + "tags", + "tokio-rustls-tls", ] } rustls = "0.23.32" rustrict = { version = "0.7.39", default-features = false, features = ["censor"] } @@ -181,12 +181,12 @@ same-file = "1.0.6" scalar_api_reference = { version = "0.2.2", default-features = false } secrecy = "0.10.3" sentry = { version = "0.45.0", default-features = false, features = [ - "backtrace", - "contexts", - "debug-images", - "panic", - "reqwest", - "rustls", + "backtrace", + "contexts", + "debug-images", + "panic", + "reqwest", + "rustls", ] } serde = "1.0.228" serde-binhum = { path = "packages/serde-binhum" } @@ -217,8 +217,8 @@ tauri-plugin-opener = "2.5.0" tauri-plugin-os = "2.3.1" tauri-plugin-single-instance = "2.3.4" tauri-plugin-updater = { git = "https://github.com/modrinth/plugins-workspace", rev = "0d30f2aa28ec668ce187d527da1c475da3c01cbc", default-features = false, features = [ - "rustls-tls", - "zip", + "rustls-tls", + "zip", ] } tauri-plugin-window-state = "2.4.0" tempfile = "3.23.0" @@ -248,20 +248,20 @@ validator = "0.20.0" webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" webp = { version = "0.3.1", default-features = false } -webview2-com = "0.38.0" # Should be updated in lockstep with wry +webview2-com = "0.38.0" # Should be updated in lockstep with wry whoami = "1.6.1" -windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 -windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 +windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 +windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 winreg = "0.55.0" woothee = "0.13.0" xredis = { path = "packages/xredis" } yaserde = "0.12.0" zbus = "5.11.0" zip = { version = "6.0.0", default-features = false, features = [ - "bzip2", - "deflate", - "deflate64", - "zstd", + "bzip2", + "deflate", + "deflate64", + "zstd", ] } zxcvbn = "3.1.0" @@ -319,15 +319,15 @@ opt-level = 3 # Optimize for speed and reduce size on release builds [profile.release] -opt-level = "s" # Optimize for binary size -strip = true # Remove debug symbols -lto = true # Enables link to optimizations -panic = "abort" # Strip expensive panic clean-up logic +opt-level = "s" # Optimize for binary size +strip = true # Remove debug symbols +lto = true # Enables link to optimizations +panic = "abort" # Strip expensive panic clean-up logic # Specific profile for labrinth production builds [profile.release-labrinth] inherits = "release" opt-level = 2 -strip = false # Keep debug symbols for Sentry -lto = "thin" # Enable LTO but keep compile times reasonable -panic = "unwind" # Don't exit the whole app on panic in production +strip = false # Keep debug symbols for Sentry +lto = "thin" # Enable LTO but keep compile times reasonable +panic = "unwind" # Don't exit the whole app on panic in production diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 9f8599feb5..456f358a26 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -83,6 +83,7 @@ paste = { workspace = true } path-util = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } +pulldown-cmark = { workspace = true } psl = { workspace = true } pulldown-cmark = { workspace = true } quick-xml = { workspace = true } From 9d13eaed78d1046333ce456e2467f36e01aee4af Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 12:56:13 -0600 Subject: [PATCH 19/42] fmt --- apps/labrinth/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 456f358a26..9f8599feb5 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -83,7 +83,6 @@ paste = { workspace = true } path-util = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } -pulldown-cmark = { workspace = true } psl = { workspace = true } pulldown-cmark = { workspace = true } quick-xml = { workspace = true } From 704b42c6c3917e9258800bf6085af4219778a33b Mon Sep 17 00:00:00 2001 From: tdgao Date: Fri, 11 Sep 2026 16:25:49 -0600 Subject: [PATCH 20/42] feat: external link validation, pass nags in failed project save due to validation, and improve links settings page --- .../ModerationProjectNags.stories.ts | 1 - .../ui/moderation/ModerationProjectNags.vue | 13 +- .../src/composables/project-nag-validation.ts | 13 +- .../composables/project-save-validation.ts | 92 + apps/frontend/src/helpers/donation-links.ts | 55 + apps/frontend/src/helpers/project-url.ts | 5 + apps/frontend/src/locales/de-CH/index.json | 6 - apps/frontend/src/locales/de-DE/index.json | 6 - apps/frontend/src/locales/en-US/index.json | 191 +- apps/frontend/src/locales/es-419/index.json | 6 - apps/frontend/src/locales/es-ES/index.json | 6 - apps/frontend/src/locales/fr-FR/index.json | 6 - apps/frontend/src/locales/hu-HU/index.json | 6 - apps/frontend/src/locales/it-IT/index.json | 6 - apps/frontend/src/locales/nl-NL/index.json | 6 - apps/frontend/src/locales/pl-PL/index.json | 6 - apps/frontend/src/locales/pt-BR/index.json | 6 - apps/frontend/src/locales/ru-RU/index.json | 6 - apps/frontend/src/locales/tr-TR/index.json | 6 - apps/frontend/src/locales/uk-UA/index.json | 6 - apps/frontend/src/locales/zh-CN/index.json | 6 - apps/frontend/src/locales/zh-TW/index.json | 6 - apps/frontend/src/pages/[type]/[project].vue | 78 +- .../[type]/[project]/settings/description.vue | 79 +- .../[type]/[project]/settings/disclosures.vue | 28 +- .../[type]/[project]/settings/gallery.vue | 91 +- .../[type]/[project]/settings/general.vue | 37 +- .../pages/[type]/[project]/settings/index.vue | 37 +- .../[type]/[project]/settings/license.vue | 403 +- .../pages/[type]/[project]/settings/links.vue | 918 +- .../[type]/[project]/settings/members.vue | 16 +- .../[type]/[project]/settings/server.vue | 44 +- .../pages/[type]/[project]/settings/tags.vue | 47 +- apps/labrinth/src/models/v2/projects.rs | 22 + apps/labrinth/src/routes/mod.rs | 9 +- .../src/routes/v2/project_creation.rs | 8 +- apps/labrinth/src/routes/v2/projects.rs | 12 + apps/labrinth/src/routes/v3/disclosures.rs | 23 +- .../src/routes/v3/project_creation.rs | 140 +- apps/labrinth/src/routes/v3/projects/mod.rs | 186 +- .../src/routes/v3/projects/validate.rs | 247 +- .../src/validate/project/description.rs | 9 - apps/labrinth/src/validate/project/license.rs | 168 +- apps/labrinth/src/validate/project/links.rs | 763 +- .../src/validate/project/links/description.rs | 245 + .../src/validate/project/links/network.rs | 611 + .../src/validate/project/links/nsfw.rs | 76895 ++++++++++++++++ apps/labrinth/src/validate/project/mod.rs | 66 +- apps/labrinth/src/validate/project/save.rs | 176 + apps/labrinth/src/validate/project/text.rs | 31 - apps/labrinth/tests/project.rs | 64 +- apps/labrinth/tests/v2/project.rs | 2 +- .../api-client/src/modules/labrinth/types.ts | 6 +- .../moderation/src/data/nags/description.ts | 9 - packages/moderation/src/data/nags/index.ts | 30 +- packages/moderation/src/data/nags/links.ts | 189 +- packages/moderation/src/data/nags/types.ts | 7 +- .../moderation/src/locales/ar-SA/index.json | 19 - .../moderation/src/locales/cs-CZ/index.json | 3 - .../moderation/src/locales/de-CH/index.json | 28 - .../moderation/src/locales/de-DE/index.json | 28 - .../moderation/src/locales/en-US/index.json | 103 +- .../moderation/src/locales/es-419/index.json | 28 - .../moderation/src/locales/es-ES/index.json | 10 - .../moderation/src/locales/fi-FI/index.json | 3 - .../moderation/src/locales/fil-PH/index.json | 4 - .../moderation/src/locales/fr-FR/index.json | 28 - .../moderation/src/locales/he-IL/index.json | 9 - .../moderation/src/locales/hu-HU/index.json | 28 - .../moderation/src/locales/id-ID/index.json | 4 - .../moderation/src/locales/it-IT/index.json | 27 - .../moderation/src/locales/ja-JP/index.json | 4 - .../moderation/src/locales/ko-KR/index.json | 4 - .../moderation/src/locales/ms-MY/index.json | 4 - .../moderation/src/locales/nl-NL/index.json | 28 - .../moderation/src/locales/no-NO/index.json | 4 - .../moderation/src/locales/pl-PL/index.json | 28 - .../moderation/src/locales/pt-BR/index.json | 28 - .../moderation/src/locales/pt-PT/index.json | 4 - .../moderation/src/locales/ro-RO/index.json | 16 - .../moderation/src/locales/ru-RU/index.json | 28 - .../moderation/src/locales/sr-CS/index.json | 10 - .../moderation/src/locales/sv-SE/index.json | 22 - .../moderation/src/locales/tr-TR/index.json | 4 - .../moderation/src/locales/uk-UA/index.json | 28 - .../moderation/src/locales/vi-VN/index.json | 4 - .../moderation/src/locales/zh-CN/index.json | 28 - .../moderation/src/locales/zh-TW/index.json | 28 - packages/ui/src/components/base/Tabs.vue | 20 +- packages/ui/src/providers/project-page.ts | 14 +- 90 files changed, 80936 insertions(+), 1848 deletions(-) create mode 100644 apps/frontend/src/composables/project-save-validation.ts create mode 100644 apps/frontend/src/helpers/donation-links.ts create mode 100644 apps/frontend/src/helpers/project-url.ts create mode 100644 apps/labrinth/src/validate/project/links/description.rs create mode 100644 apps/labrinth/src/validate/project/links/network.rs create mode 100644 apps/labrinth/src/validate/project/links/nsfw.rs create mode 100644 apps/labrinth/src/validate/project/save.rs diff --git a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.stories.ts b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.stories.ts index c44b2bf657..cb13894051 100644 --- a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.stories.ts +++ b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.stories.ts @@ -120,7 +120,6 @@ const suggestionKinds = new Set([ const warningKinds = new Set([ 'missing-alt-text', - 'verify-external-links', 'too-many-languages', 'too-many-tags', 'multiple-resolution-tags', diff --git a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue index 26ff89cf7d..5810757e43 100644 --- a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue +++ b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue @@ -401,16 +401,15 @@ async function submitForReview() { const applicableNags = computed(() => { if (props.nags) return props.nags - const nagsByKind = new Map< - Labrinth.Projects.v3.NormalizedProjectNagKind, - Labrinth.Projects.v3.ProjectNag - >() + const nagsById = new Map() for (const nag of props.validationNags) { const kind = normalizeProjectNagKind(nag.kind) - if (kind && !nagsByKind.has(kind)) nagsByKind.set(kind, nag) + if (!kind) continue + const mapped = toProjectNag(nag, props.project.project_type) + if (!nagsById.has(mapped.id)) nagsById.set(mapped.id, mapped) } - return [...nagsByKind.values()].map((nag) => toProjectNag(nag, props.project.project_type)) + return [...nagsById.values()] }) function isNagComplete(nag: Nag): boolean { @@ -490,7 +489,7 @@ watch( const actionableNagKeys = new Set( validationNags .filter((nag) => nag.severity === 'required' || nag.severity === 'warning') - .map((nag) => `${nag.severity}:${nag.kind}`), + .map((nag) => `${nag.severity}:${nag.kind}:${JSON.stringify(nag.details)}`), ) const previousNagKeys = previousActionableNagKeys const hasNewActionableNag = diff --git a/apps/frontend/src/composables/project-nag-validation.ts b/apps/frontend/src/composables/project-nag-validation.ts index 2bf81e6edf..171f834683 100644 --- a/apps/frontend/src/composables/project-nag-validation.ts +++ b/apps/frontend/src/composables/project-nag-validation.ts @@ -14,8 +14,7 @@ export type ProjectSettingsField = | 'custom-license' | 'license-url' | 'external-links' - | 'source-issues-discord-links' - | 'non-discord-link-fields' + | 'link-field' | 'source-availability' | 'permissions' | 'server-region' @@ -50,14 +49,15 @@ export const projectNagFields = { ], icon: ['add-icon'], description: [ + 'link-validation', 'project-description-slur', 'project-description-profanity', 'project-description-non-standard-text', 'project-description-non-english', + 'project-description-matches-summary', 'add-description', 'description-too-short', 'project-description-spam', - 'project-description-banned-link', 'long-headers', 'description-ends-with-header', 'adjacent-headers', @@ -67,10 +67,9 @@ export const projectNagFields = { 'gallery-images': ['upload-gallery-image', 'feature-gallery-image'], license: ['select-license'], 'custom-license': ['add-custom-license-details'], - 'license-url': ['invalid-license-url'], - 'external-links': ['add-links', 'add-links-server', 'identical-links', 'banned-link-usage'], - 'source-issues-discord-links': ['verify-external-links'], - 'non-discord-link-fields': ['misused-discord-link'], + 'license-url': ['invalid-license-url', 'link-validation'], + 'link-field': ['link-validation'], + 'external-links': ['add-links', 'add-links-server'], 'source-availability': ['gpl-license-source-required'], permissions: ['review-permissions'], 'server-region': ['select-country'], diff --git a/apps/frontend/src/composables/project-save-validation.ts b/apps/frontend/src/composables/project-save-validation.ts new file mode 100644 index 0000000000..66ecca9716 --- /dev/null +++ b/apps/frontend/src/composables/project-save-validation.ts @@ -0,0 +1,92 @@ +import type { Labrinth } from '@modrinth/api-client' +import { normalizeProjectNagKind, toProjectFieldMessage } from '@modrinth/moderation' +import { injectProjectPageContext } from '@modrinth/ui' +import { computed, ref } from 'vue' + +import { projectNagFields, type ProjectSettingsField } from './project-nag-validation' + +function matchesField(nag: Labrinth.Projects.v3.ProjectNag, field: string, detailField = field) { + const kinds: readonly string[] | undefined = Object.hasOwn(projectNagFields, field) + ? projectNagFields[field as ProjectSettingsField] + : undefined + const kind = normalizeProjectNagKind(nag.kind) + if (kinds && (!kind || !kinds.includes(kind))) return false + if (typeof nag.details.field === 'string') return nag.details.field === detailField + if (Array.isArray(nag.details.fields)) return nag.details.fields.includes(detailField) + return kinds !== undefined +} + +/** Keeps rejected-save messages attached to the exact values that were submitted. */ +export function useProjectSaveValidation(state: () => unknown) { + const { projectV2 } = injectProjectPageContext() + const rejected = ref([]) + const rejectedState = ref('') + const showMessages = computed( + () => + projectV2.value.status === 'processing' && rejectedState.value === JSON.stringify(state()), + ) + const messages = computed(() => + showMessages.value ? rejected.value.map((nag) => toProjectFieldMessage(nag)) : [], + ) + + const hasErrors = computed(() => messages.value.some((message) => message.severity === 'error')) + + function snapshot() { + return JSON.stringify(state()) ?? '' + } + + function capture(error: unknown, submittedState: string): boolean { + if (projectV2.value.status !== 'processing') return false + let value = error + for (let depth = 0; depth < 5; depth++) { + if (typeof value !== 'object' || value === null) return false + const data = value as Record + const details = data.details + if (typeof details === 'object' && details !== null && 'nags' in details) { + const nags = details.nags + if (!Array.isArray(nags)) return false + const recognized = nags.filter( + (nag): nag is Labrinth.Projects.v3.ProjectNag => + typeof nag === 'object' && + nag !== null && + typeof nag.kind === 'string' && + normalizeProjectNagKind(nag.kind) !== null && + ['required', 'warning', 'suggestion'].includes(nag.severity) && + typeof nag.details === 'object' && + nag.details !== null, + ) + rejected.value = recognized.filter((nag) => nag.severity !== 'suggestion') + rejectedState.value = submittedState + return recognized.length > 0 + } + value = data.responseData ?? data.data ?? data.originalError ?? data.cause + } + return false + } + + function forField(field: string, detailField = field) { + if (!showMessages.value) return [] + return rejected.value + .filter((nag) => matchesField(nag, field, detailField)) + .map((nag) => toProjectFieldMessage(nag)) + } + + function withoutFields(fields: (string | [field: string, detailField: string])[]) { + if (!showMessages.value) return [] + return rejected.value + .filter( + (nag) => + !fields.some((field) => + Array.isArray(field) ? matchesField(nag, ...field) : matchesField(nag, field), + ), + ) + .map((nag) => toProjectFieldMessage(nag)) + } + + function clear() { + rejected.value = [] + rejectedState.value = '' + } + + return { capture, clear, messages, hasErrors, forField, withoutFields, snapshot } +} diff --git a/apps/frontend/src/helpers/donation-links.ts b/apps/frontend/src/helpers/donation-links.ts new file mode 100644 index 0000000000..15788a5cb7 --- /dev/null +++ b/apps/frontend/src/helpers/donation-links.ts @@ -0,0 +1,55 @@ +export const donationUsernamePrefixes: Record = { + patreon: 'https://www.patreon.com/', + bmac: 'https://buymeacoffee.com/', + paypal: 'https://www.paypal.me/', + github: 'https://github.com/sponsors/', + 'ko-fi': 'https://ko-fi.com/', +} + +export interface DonationInput { + id?: string + url: string + input: string + mode: 'username' | 'url' +} + +export function donationUsernameFromUrl( + platform: string | undefined, + raw: string, +): string | undefined { + const prefix = platform ? donationUsernamePrefixes[platform] : undefined + if (!prefix || !raw.startsWith(prefix)) return undefined + const username = raw.slice(prefix.length) + if (!username || /[/?#\s]/.test(username)) return undefined + try { + return decodeURIComponent(username) + } catch { + return undefined + } +} + +export function donationInput(id?: string, url = ''): DonationInput { + const username = donationUsernameFromUrl(id, url) + return { + id, + url, + input: username ?? url, + mode: + username !== undefined || (!url && id && donationUsernamePrefixes[id]) ? 'username' : 'url', + } +} + +export function setDonationInput(row: DonationInput, value: string | number, detectUrl = true) { + row.input = String(value) + if (detectUrl && /^https?:\/\//i.test(row.input.trim())) row.mode = 'url' + const prefix = row.id ? donationUsernamePrefixes[row.id] : undefined + row.url = + row.mode === 'username' && prefix && row.input + ? prefix + encodeURIComponent(row.input) + : row.input +} + +export function toggleDonationInput(row: DonationInput) { + row.mode = row.mode === 'username' ? 'url' : 'username' + setDonationInput(row, row.input, false) +} diff --git a/apps/frontend/src/helpers/project-url.ts b/apps/frontend/src/helpers/project-url.ts new file mode 100644 index 0000000000..c4d87e6a8c --- /dev/null +++ b/apps/frontend/src/helpers/project-url.ts @@ -0,0 +1,5 @@ +export function normalizeProjectUrl(value: string): string { + const url = value.trim() + if (!url || /^[a-z][a-z\d+.-]*:/i.test(url)) return url + return `https://${url}` +} diff --git a/apps/frontend/src/locales/de-CH/index.json b/apps/frontend/src/locales/de-CH/index.json index fd3911ebe5..29a6c93a28 100644 --- a/apps/frontend/src/locales/de-CH/index.json +++ b/apps/frontend/src/locales/de-CH/index.json @@ -4115,12 +4115,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Du hast bereits einen anderen {platform}-Link." - }, - "project.settings.links.donation.no-type": { - "message": "Bitte wählen eine Plattform für diesen Spenden-Link aus." - }, "project.settings.monetization.description": { "message": "Projekte auf Modrinth nehmen automatisch am Belohnungsprogramm teil. Falls du mit diesem Projekt keine Einnahmen erzielen möchtest (oder aus rechtlichen Gründen nicht darfst), kannst du dies hier deaktivieren." }, diff --git a/apps/frontend/src/locales/de-DE/index.json b/apps/frontend/src/locales/de-DE/index.json index 7be56beef9..36abfdcc9a 100644 --- a/apps/frontend/src/locales/de-DE/index.json +++ b/apps/frontend/src/locales/de-DE/index.json @@ -4115,12 +4115,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Du hast bereits einen anderen {platform}-Link." - }, - "project.settings.links.donation.no-type": { - "message": "Bitte wählen eine Plattform für diesen Spenden-Link aus." - }, "project.settings.monetization.description": { "message": "Projekte auf Modrinth nehmen automatisch am Belohnungsprogramm teil. Falls du mit diesem Projekt keine Einnahmen erzielen möchtest (oder aus rechtlichen Gründen nicht darfst), kannst du dies hier deaktivieren." }, diff --git a/apps/frontend/src/locales/en-US/index.json b/apps/frontend/src/locales/en-US/index.json index e87cc6c93c..e027225a3f 100644 --- a/apps/frontend/src/locales/en-US/index.json +++ b/apps/frontend/src/locales/en-US/index.json @@ -3884,6 +3884,21 @@ "project.settings.delete-project.title": { "message": "Delete project" }, + "project.settings.description.failed": { + "message": "Failed to update description" + }, + "project.settings.description.intro": { + "message": "You can type an extended description of your project here. The description must clearly and honestly describe the purpose and function of the project. See section 2.1 of the Content Rules for the full requirements." + }, + "project.settings.description.title": { + "message": "Description" + }, + "project.settings.description.updated": { + "message": "Description updated" + }, + "project.settings.description.updated-text": { + "message": "Your description has been updated." + }, "project.settings.disclosures.advertising.description.1": { "message": "Must be enabled if this project contains advertisements, sponsorships, or promotions of other works." }, @@ -4115,11 +4130,179 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "You already have another {platform} link." + "project.settings.license.all-rights": { + "message": "All Rights Reserved/No License" + }, + "project.settings.license.allow-later": { + "message": "Allow later editions" + }, + "project.settings.license.custom": { + "message": "Custom" + }, + "project.settings.license.custom-url-description": { + "message": "The web location of the full license text. You have to provide a link since this is a custom license." + }, + "project.settings.license.failed": { + "message": "Failed to update license" + }, + "project.settings.license.has-spdx": { + "message": "Use SPDX identifier" + }, + "project.settings.license.intro": { + "message": "It is important to choose a proper license for your {type}. You may choose one from our list or provide a custom license. You may also provide a custom URL to your chosen license; otherwise, the license text will be displayed. See our licensing guide for more information." + }, + "project.settings.license.later": { + "message": "Later editions" + }, + "project.settings.license.later-description": { + "message": "The license you selected has an \"or later\" clause. If you check this box, users may use your project under later editions of the license." + }, + "project.settings.license.missing-name": { + "message": "Enter a name or SPDX identifier for your custom license." + }, + "project.settings.license.missing-url": { + "message": "Enter a URL to the full text of your custom license." + }, + "project.settings.license.name": { + "message": "License name" + }, + "project.settings.license.name-description": { + "message": "The full name of the license. If the license has a SPDX identifier, please check the checkbox and use the identifier instead." + }, + "project.settings.license.name-placeholder": { + "message": "License name" + }, + "project.settings.license.optional-url": { + "message": "License URL (optional)" + }, + "project.settings.license.select": { + "message": "Select a license" + }, + "project.settings.license.select-description": { + "message": "How users are and aren't allowed to use your project." + }, + "project.settings.license.select-placeholder": { + "message": "Select license..." + }, + "project.settings.license.spdx": { + "message": "SPDX identifier" + }, + "project.settings.license.spdx-description": { + "message": "If your license does not have an official SPDX license identifier, uncheck the box and enter the name of the license instead." + }, + "project.settings.license.spdx-placeholder": { + "message": "SPDX identifier" + }, + "project.settings.license.title": { + "message": "License" + }, + "project.settings.license.updated": { + "message": "License updated" + }, + "project.settings.license.updated-text": { + "message": "Your license has been updated." + }, + "project.settings.license.url": { + "message": "License URL" + }, + "project.settings.license.url-description": { + "message": "The web location of the full license text. If you don't provide a link, the license text will be displayed instead." + }, + "project.settings.links.add-link": { + "message": "Add link" + }, + "project.settings.links.discord": { + "message": "Discord invite" + }, + "project.settings.links.discord-description": { + "message": "An invitation link to your Discord server." + }, + "project.settings.links.donation-link": { + "message": "Link" + }, + "project.settings.links.donation-platform": { + "message": "Platform" + }, + "project.settings.links.donation-url": { + "message": "URL" + }, + "project.settings.links.donation-username": { + "message": "Username" + }, + "project.settings.links.donation-username-placeholder": { + "message": "Enter your {platform} username" + }, + "project.settings.links.donations": { + "message": "Donation links" + }, + "project.settings.links.donations-description": { + "message": "Add donation links for users to support you directly." + }, + "project.settings.links.failed": { + "message": "Failed to update links" + }, + "project.settings.links.issues": { + "message": "Issue tracker" + }, + "project.settings.links.issues-description": { + "message": "A place for users to report bugs, issues, and concerns about your project." + }, + "project.settings.links.link-type": { + "message": "Link type" + }, + "project.settings.links.no-donation-links": { + "message": "No donation links added" + }, + "project.settings.links.remove-donation-link": { + "message": "Remove donation link" + }, + "project.settings.links.server-discord": { + "message": "Discord" + }, + "project.settings.links.server-updated": { + "message": "Your server links have been updated." + }, + "project.settings.links.server-wiki-description": { + "message": "A page containing information, documentation, and help for the server." + }, + "project.settings.links.site": { + "message": "Website" + }, + "project.settings.links.site-description": { + "message": "Your server's official website." + }, + "project.settings.links.source": { + "message": "Source code" + }, + "project.settings.links.source-description": { + "message": "A page/repository containing the source code for your project" + }, + "project.settings.links.store": { + "message": "Store" + }, + "project.settings.links.store-description": { + "message": "A link to your server's store or shop." + }, + "project.settings.links.title": { + "message": "Links" + }, + "project.settings.links.updated": { + "message": "Your links have been updated." + }, + "project.settings.links.updated-title": { + "message": "Links updated" + }, + "project.settings.links.url-placeholder": { + "message": "Enter a valid URL" + }, + "project.settings.links.visit-link": { + "message": "Visit {url}" + }, + "project.settings.links.wiki": { + "message": "Wiki page" }, - "project.settings.links.donation.no-type": { - "message": "Please select a platform for this Donation link." + "project.settings.links.wiki-description": { + "message": "A page containing information, documentation, and help for the project." }, "project.settings.monetization.description": { "message": "Projects on Modrinth are automatically enrolled in the Rewards Program. If you don't want to (or can't for legal reasons) earn revenue from this project, you can turn it off here." diff --git a/apps/frontend/src/locales/es-419/index.json b/apps/frontend/src/locales/es-419/index.json index 6f6d8a6aa8..ee483f7397 100644 --- a/apps/frontend/src/locales/es-419/index.json +++ b/apps/frontend/src/locales/es-419/index.json @@ -4115,12 +4115,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Ya tienes otro enlace de {platform}." - }, - "project.settings.links.donation.no-type": { - "message": "Por favor, seleccione una plataforma para este enlace de donación." - }, "project.settings.monetization.description": { "message": "Los proyectos en Modrinth están inscritos automáticamente en el Rewards Program. Si usted no quiere (o no puede por razones legales) obtener ingresos de este proyecto, puede desactivarlo aquí." }, diff --git a/apps/frontend/src/locales/es-ES/index.json b/apps/frontend/src/locales/es-ES/index.json index 53fd869504..8bbd2479c7 100644 --- a/apps/frontend/src/locales/es-ES/index.json +++ b/apps/frontend/src/locales/es-ES/index.json @@ -4109,12 +4109,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Ya tienes otro enlace de {platform}." - }, - "project.settings.links.donation.no-type": { - "message": "Por favor, seleccione una plataforma para este enlace de donación." - }, "project.settings.monetization.description": { "message": "Los proyectos en Modrinth están inscritos automáticamente en el Rewards Program (Programa de Recompensas). Si usted no quiere (o no puede por razones legales) obtener ingresos de este proyecto, puede desactivarlo aquí." }, diff --git a/apps/frontend/src/locales/fr-FR/index.json b/apps/frontend/src/locales/fr-FR/index.json index 3ead5573b4..f272700a21 100644 --- a/apps/frontend/src/locales/fr-FR/index.json +++ b/apps/frontend/src/locales/fr-FR/index.json @@ -4091,12 +4091,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Vous avez déjà un autre lien {platform}." - }, - "project.settings.links.donation.no-type": { - "message": "Veuillez sélectionner une plateforme pour ce lien de donation." - }, "project.settings.monetization.description": { "message": "Les projets sur Modrinth sont automatiquement inscrits au programme de récompenses. Si vous ne souhaitez pas (ou ne pouvez pas, pour des raisons légales) générer des revenus avec ce projet, vous pouvez désactiver cette option ici." }, diff --git a/apps/frontend/src/locales/hu-HU/index.json b/apps/frontend/src/locales/hu-HU/index.json index 10d4fec792..7675e9d2be 100644 --- a/apps/frontend/src/locales/hu-HU/index.json +++ b/apps/frontend/src/locales/hu-HU/index.json @@ -4103,12 +4103,6 @@ "project.settings.general.url.title": { "message": "Link" }, - "project.settings.links.donation.duplicate-type": { - "message": "Már rendelkezel egy másik {platform} hivatkozással." - }, - "project.settings.links.donation.no-type": { - "message": "Válassz egy platformot ehhez az adományozási hivatkozáshoz." - }, "project.settings.monetization.description": { "message": "A Modrinthon található projektek automatikusan részt vesznek a Jutalmazási programban. Ha nem szeretnél (vagy jogi okokból nem tudsz) bevételt szerezni ebből a projektből, itt kikapcsolhatod." }, diff --git a/apps/frontend/src/locales/it-IT/index.json b/apps/frontend/src/locales/it-IT/index.json index 2be753b97e..8e43504934 100644 --- a/apps/frontend/src/locales/it-IT/index.json +++ b/apps/frontend/src/locales/it-IT/index.json @@ -4106,12 +4106,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Hai già un link per {platform}." - }, - "project.settings.links.donation.no-type": { - "message": "Seleziona una piattaforma per il link di donazione." - }, "project.settings.monetization.description": { "message": "I progetti su Modrinth vengono automaticamente iscritti al Programma Premi. Se non vuoi guadagnare da questo progetto (o non puoi per motivi legali), puoi disattivarlo qui." }, diff --git a/apps/frontend/src/locales/nl-NL/index.json b/apps/frontend/src/locales/nl-NL/index.json index 42f3a27cb6..87a6e8da3a 100644 --- a/apps/frontend/src/locales/nl-NL/index.json +++ b/apps/frontend/src/locales/nl-NL/index.json @@ -4040,12 +4040,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Je hebt al een andere {platform} link." - }, - "project.settings.links.donation.no-type": { - "message": "Selecteer een platform voor deze donatie link." - }, "project.settings.monetization.description": { "message": "Projecten op Modrinth worden automatisch ingeschreven in het Beloningen Programma. Als je geen inkomsten wilt (of niet kunt verdienen om juridische redenen) van dit project, kun je het hier uitschakelen." }, diff --git a/apps/frontend/src/locales/pl-PL/index.json b/apps/frontend/src/locales/pl-PL/index.json index 8031abc964..d4588b0998 100644 --- a/apps/frontend/src/locales/pl-PL/index.json +++ b/apps/frontend/src/locales/pl-PL/index.json @@ -4058,12 +4058,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Masz już inny link do platformy {platform}." - }, - "project.settings.links.donation.no-type": { - "message": "Wybierz platformę dla tego linku do darowizn." - }, "project.settings.monetization.description": { "message": "Projekty na Modrinth są automatycznie przyjęte w Programie Nagród. Jeśli ty nie chcesz (lub nie możesz z powodów prawnych) zarabiać z tego projektu, możesz wyłączyć to tutaj." }, diff --git a/apps/frontend/src/locales/pt-BR/index.json b/apps/frontend/src/locales/pt-BR/index.json index eaba876186..0d180bc1a8 100644 --- a/apps/frontend/src/locales/pt-BR/index.json +++ b/apps/frontend/src/locales/pt-BR/index.json @@ -4115,12 +4115,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Você já possui outro link de {platform}." - }, - "project.settings.links.donation.no-type": { - "message": "Por favor, selecione uma plataforma para este link de Doação." - }, "project.settings.monetization.description": { "message": "Projetos do Modrinth são automaticamente inscritos no Programa de Recompensas. Se você não quer (ou não pode por motivos legais) lucrar com o projeto, pode desativar aqui." }, diff --git a/apps/frontend/src/locales/ru-RU/index.json b/apps/frontend/src/locales/ru-RU/index.json index 57cea928cf..3ef8c879fb 100644 --- a/apps/frontend/src/locales/ru-RU/index.json +++ b/apps/frontend/src/locales/ru-RU/index.json @@ -4106,12 +4106,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Ссылка на {platform} повторяется." - }, - "project.settings.links.donation.no-type": { - "message": "Выберите платформу для этой ссылки." - }, "project.settings.monetization.description": { "message": "Проекты на Modrinth автоматически зачисляются в программу наград. Если вы не хотите (или не можете по юридическим причинам) получать доход от этого проекта, вы можете отключить его здесь." }, diff --git a/apps/frontend/src/locales/tr-TR/index.json b/apps/frontend/src/locales/tr-TR/index.json index 4c73767f8c..c84fe1d7ea 100644 --- a/apps/frontend/src/locales/tr-TR/index.json +++ b/apps/frontend/src/locales/tr-TR/index.json @@ -4043,12 +4043,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Zaten başka bir {platform} bağlantınız var." - }, - "project.settings.links.donation.no-type": { - "message": "Lütfen bu bağış bağlantısı için bir platform seçin." - }, "project.settings.monetization.description": { "message": "Modrinth’teki projeler otomatik olarak Ödül Programı’na kaydedilir. Bu projeden gelir elde etmek istemiyorsanız (veya yasal nedenlerle edemiyorsanız), buradan kapatabilirsiniz." }, diff --git a/apps/frontend/src/locales/uk-UA/index.json b/apps/frontend/src/locales/uk-UA/index.json index 07683debfd..c60e09aeb7 100644 --- a/apps/frontend/src/locales/uk-UA/index.json +++ b/apps/frontend/src/locales/uk-UA/index.json @@ -4112,12 +4112,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "Ви вже маєте ще одне посилання {platform}." - }, - "project.settings.links.donation.no-type": { - "message": "Будь ласка, оберіть платформу для цього посилання для пожертви." - }, "project.settings.monetization.description": { "message": "Проєкти на Modrinth автоматично реєструються в Програмі винагород. Якщо ви не хочете (або не можете з юридичних причин) отримувати дохід від цього проєкту, ви можете вимкнути його тут." }, diff --git a/apps/frontend/src/locales/zh-CN/index.json b/apps/frontend/src/locales/zh-CN/index.json index b75ed71c4f..28c15bbf80 100644 --- a/apps/frontend/src/locales/zh-CN/index.json +++ b/apps/frontend/src/locales/zh-CN/index.json @@ -4115,12 +4115,6 @@ "project.settings.general.url.title": { "message": "URL" }, - "project.settings.links.donation.duplicate-type": { - "message": "你已经有了一个{platform}链接了。" - }, - "project.settings.links.donation.no-type": { - "message": "请为此捐赠链接选择一个平台。" - }, "project.settings.monetization.description": { "message": "Modrinth上的项目会自动加入激励计划。如果你不想（或由于法律原因不能）从这个项目赚取收入，你可以在这里关闭它。" }, diff --git a/apps/frontend/src/locales/zh-TW/index.json b/apps/frontend/src/locales/zh-TW/index.json index 44496a4464..1f88c74435 100644 --- a/apps/frontend/src/locales/zh-TW/index.json +++ b/apps/frontend/src/locales/zh-TW/index.json @@ -4115,12 +4115,6 @@ "project.settings.general.url.title": { "message": "網址" }, - "project.settings.links.donation.duplicate-type": { - "message": "你已經有一個 {platform} 連結了。" - }, - "project.settings.links.donation.no-type": { - "message": "請為這個贊助連結選擇平台。" - }, "project.settings.monetization.description": { "message": "Modrinth 上的專案會自動加入獎勵計畫。如果你不想（或因法律原因無法）從專案獲取收益，可在這裡關閉營利功能。" }, diff --git a/apps/frontend/src/pages/[type]/[project].vue b/apps/frontend/src/pages/[type]/[project].vue index 6d83e1152b..5b6e5662c3 100644 --- a/apps/frontend/src/pages/[type]/[project].vue +++ b/apps/frontend/src/pages/[type]/[project].vue @@ -1386,15 +1386,17 @@ function addProjectMutationErrorNotification(error) { error?.responseData?.description ?? error?.data?.description ?? error?.message - const isProjectReviewValidationError = description === PROJECT_REVIEW_VALIDATION_ERROR + const response = error?.responseData ?? error?.data ?? error?.v1Error + const isProjectValidationError = + Array.isArray(response?.details?.nags) || description === PROJECT_REVIEW_VALIDATION_ERROR addNotification({ title: formatMessage( - isProjectReviewValidationError + isProjectValidationError && project.value.status === 'processing' ? messages.projectReviewSaveFailed : commonMessages.errorNotificationTitle, ), - text: isProjectReviewValidationError + text: isProjectValidationError ? formatMessage(messages.projectReviewSaveFailedDescription) : description, type: 'error', @@ -1409,7 +1411,8 @@ const patchProjectMutation = useMutation({ return data }, - onMutate: async ({ projectId, data }) => { + onMutate: async ({ projectId, data, optimistic = true }) => { + if (!optimistic) return await queryClient.cancelQueries({ queryKey: ['project', 'v2', projectId] }) await queryClient.cancelQueries({ queryKey: ['project', 'v3', projectId] }) @@ -1489,7 +1492,8 @@ const patchProjectV3Mutation = useMutation({ return data }, - onMutate: async ({ projectId, data }) => { + onMutate: async ({ projectId, data, optimistic = true }) => { + if (!optimistic) return await queryClient.cancelQueries({ queryKey: ['project', 'v3', projectId] }) await queryClient.cancelQueries({ queryKey: ['project', 'v2', projectId] }) @@ -1516,8 +1520,8 @@ const patchProjectV3Mutation = useMutation({ addProjectMutationErrorNotification(err) }, - onSettled: () => { - void invalidateProject() + onSettled: async () => { + await invalidateProject() }, }) @@ -2168,12 +2172,12 @@ async function setProcessing() { ) } -async function patchProject(resData, quiet = false) { +async function patchProject(resData, quiet = false, throwOnError = false) { startLoading() - return new Promise((resolve) => { + return new Promise((resolve, reject) => { patchProjectMutation.mutate( - { projectId: project.value.id, data: resData }, + { projectId: project.value.id, data: resData, optimistic: !throwOnError }, { onSuccess: async () => { if (!quiet) { @@ -2185,19 +2189,19 @@ async function patchProject(resData, quiet = false) { } resolve(true) }, - onError: () => resolve(false), + onError: (error) => (throwOnError ? reject(error) : resolve(false)), onSettled: () => stopLoading(), }, ) }) } -async function patchProjectV3(resData, quiet = false) { +async function patchProjectV3(resData, quiet = false, throwOnError = false) { startLoading() - return new Promise((resolve) => { + return new Promise((resolve, reject) => { patchProjectV3Mutation.mutate( - { projectId: project.value.id, data: resData }, + { projectId: project.value.id, data: resData, optimistic: !throwOnError }, { onSuccess: async () => { if (!quiet) { @@ -2209,7 +2213,7 @@ async function patchProjectV3(resData, quiet = false) { } resolve(true) }, - onError: () => resolve(false), + onError: (error) => (throwOnError ? reject(error) : resolve(false)), onSettled: () => stopLoading(), }, ) @@ -2231,30 +2235,58 @@ async function patchIcon(icon) { }) } -async function createGalleryItem(file, title, description, featured, ordering) { +async function createGalleryItem( + file, + title, + description, + featured, + ordering, + throwOnError = false, +) { startLoading() - return new Promise((resolve) => { + return new Promise((resolve, reject) => { createGalleryItemMutation.mutate( - { projectId: project.value.id, file, title, description, featured, ordering }, + { + projectId: project.value.id, + file, + title, + description, + featured, + ordering, + }, { onSuccess: () => resolve(true), - onError: () => resolve(false), + onError: (error) => (throwOnError ? reject(error) : resolve(false)), onSettled: () => stopLoading(), }, ) }) } -async function editGalleryItem(imageUrl, title, description, featured, ordering) { +async function editGalleryItem( + imageUrl, + title, + description, + featured, + ordering, + throwOnError = false, +) { startLoading() - return new Promise((resolve) => { + return new Promise((resolve, reject) => { editGalleryItemMutation.mutate( - { projectId: project.value.id, imageUrl, title, description, featured, ordering }, + { + projectId: project.value.id, + imageUrl, + title, + description, + featured, + ordering, + }, { onSuccess: () => resolve(true), - onError: () => resolve(false), + onError: (error) => (throwOnError ? reject(error) : resolve(false)), onSettled: () => stopLoading(), }, ) diff --git a/apps/frontend/src/pages/[type]/[project]/settings/description.vue b/apps/frontend/src/pages/[type]/[project]/settings/description.vue index 147d7c6ded..df0ecc9489 100644 --- a/apps/frontend/src/pages/[type]/[project]/settings/description.vue +++ b/apps/frontend/src/pages/[type]/[project]/settings/description.vue @@ -4,20 +4,20 @@
-

Description

+

{{ formatMessage(messages.title) }}

- You can type an extended description of your project here. - - The description must clearly and honestly describe the purpose and function of the - project. See section 2.1 of the - Content Rules - for the full requirements. - + + +
+
Content Rules for the full requirements.', + }, + updated: { id: 'project.settings.description.updated', defaultMessage: 'Description updated' }, + updatedText: { + id: 'project.settings.description.updated-text', + defaultMessage: 'Your description has been updated.', + }, + failed: { + id: 'project.settings.description.failed', + defaultMessage: 'Failed to update description', + }, +}) const aiImageWarningModal = useTemplateRef('aiImageWarningModal') useProjectSettingsHeadTitle(commonProjectSettingsMessages.description) @@ -72,7 +99,8 @@ const { } = useSavable( () => ({ description: project.value.body }), async ({ description }) => { - await patchProject({ body: description }) + await labrinth.projects_v3.edit(project.value.id, { description }) + await invalidate() }, ) @@ -86,12 +114,33 @@ const hasPermission = computed( (currentMember.value.permissions & TeamMemberPermission.EDIT_BODY) === TeamMemberPermission.EDIT_BODY), ) -const descriptionValidation = useProjectNagMessages('description') -const canSave = computed(() => hasPermission.value) +const descriptionValidation = useProjectNagMessages('description', 'description') +const saveValidation = useProjectSaveValidation(() => current.value) +const canSave = computed( + () => + hasPermission.value && + !saveValidation.messages.value.some((message) => message.severity === 'error'), +) async function save() { - if (!canSave.value) return - await saveForm() + if (!canSave.value || saving.value) return + const submittedState = saveValidation.snapshot() + try { + await saveForm() + saveValidation.clear() + addNotification({ + title: formatMessage(messages.updated), + text: formatMessage(messages.updatedText), + type: 'success', + }) + } catch (error) { + saveValidation.capture(error, submittedState) + addNotification({ + title: formatMessage(messages.failed), + text: error instanceof Error ? error.message : String(error), + type: 'error', + }) + } } async function onUploadHandler(file: File) { diff --git a/apps/frontend/src/pages/[type]/[project]/settings/disclosures.vue b/apps/frontend/src/pages/[type]/[project]/settings/disclosures.vue index 86853fa80d..cf0e3d20ba 100644 --- a/apps/frontend/src/pages/[type]/[project]/settings/disclosures.vue +++ b/apps/frontend/src/pages/[type]/[project]/settings/disclosures.vue @@ -42,6 +42,7 @@ import { import ValidationMessage from '~/components/ValidationMessage.vue' import { useAuth } from '~/composables/auth' import { useProjectNagMessages } from '~/composables/project-nag-validation' +import { useProjectSaveValidation } from '~/composables/project-save-validation' const DISCLOSURE_QUERY_STALE_TIME = 1000 * 60 * 5 @@ -195,7 +196,7 @@ const { saved, current, saving, - reset, + reset: resetForm, save: saveForm, } = useSavable( () => disclosuresToForm(disclosuresResponse.value?.disclosures ?? []), @@ -221,10 +222,23 @@ const hasChanges = computed( () => JSON.stringify(savedSnapshot.value) !== JSON.stringify(currentSnapshot.value), ) +const saveValidation = useProjectSaveValidation(() => currentSnapshot.value) + async function save() { - if (!hasChanges.value) return - await saveForm() - await refreshProjectValidation() + if (!hasChanges.value || !canSave.value || saving.value) return + const submittedState = saveValidation.snapshot() + try { + await saveForm() + saveValidation.clear() + await refreshProjectValidation() + } catch (error) { + if (!saveValidation.capture(error, submittedState)) throw error + } +} + +function reset() { + resetForm() + saveValidation.clear() } function disclosureUpdateProps(type: DisclosureType) { @@ -268,7 +282,10 @@ const disclosureTextValidation = useProjectNagMessages('disclosure-text') const disclosureValidation = useProjectNagMessages('disclosures') const canSave = computed( - () => hasPermission.value && (isAdminUser.value || issues.value.length === 0), + () => + !saveValidation.hasErrors.value && + hasPermission.value && + (isAdminUser.value || issues.value.length === 0), ) const saveDisabledReason = computed(() => { @@ -398,6 +415,7 @@ const { confirmLeaveModal } = usePageLeaveSafety(hasChanges) " /> + -