diff --git a/apps/frontend/src/components/ui/moderation/ModerationTechRevCard.vue b/apps/frontend/src/components/ui/moderation/ModerationTechRevCard.vue index 40e62bf1292..31bf6134769 100644 --- a/apps/frontend/src/components/ui/moderation/ModerationTechRevCard.vue +++ b/apps/frontend/src/components/ui/moderation/ModerationTechRevCard.vue @@ -52,7 +52,7 @@ const emit = defineEmits<{ showMaliciousSummary: [unsafeFiles: UnsafeFile[]] }>() -const showHiddenTraces = ref(false) +const showHiddenTraces = ref(props.allowShowingHiddenTraces ?? false) const hiddenTraceCount = computed(() => props.item.reports.reduce( (reportCount, report) => diff --git a/apps/frontend/src/pages/moderation/technical-review/rules.vue b/apps/frontend/src/pages/moderation/technical-review/rules.vue index b541151d4e6..8d7f8656693 100644 --- a/apps/frontend/src/pages/moderation/technical-review/rules.vue +++ b/apps/frontend/src/pages/moderation/technical-review/rules.vue @@ -377,26 +377,22 @@ IS_MATCH ? "low" : null -
+
-
-

Scanning Delphi rule effects

-

- {{ scanProgress.scanned.toLocaleString() }} of - {{ scanProgress.total.toLocaleString() }} details scanned · - {{ scanProgress.effects.toLocaleString() }} effects -

-
- +

Scanning Delphi rule effects

+ {{ scanProgress.phase }} revision {{ scanProgress.revision }}
@@ -467,9 +463,20 @@ IS_MATCH ? "low" : null >{{ rule.rule }}
-

- Affected details ({{ getAffectedDetailsTotal(rule).toLocaleString() }}) -

+
+

+ Affected details ({{ getAffectedDetailsTotal(rule).toLocaleString() }}) +

+ +

No details are affected in the current revision.

@@ -597,6 +604,7 @@ import { type TabsTab, TagItem, Textarea, + Toggle, } from '@modrinth/ui' import { useQuery } from '@tanstack/vue-query' import { useDebounceFn } from '@vueuse/core' @@ -792,6 +800,8 @@ const traceDataError = ref(null) const testTraceMode = ref('fields') const rawTestTraceInput = ref('') const scanProgress = ref(null) +const scanEstimatedSecondsRemaining = ref(null) +const scanProgressSamples: Array<{ scanned: number; recordedAt: number }> = [] const affectedDetailsPages = reactive( new Map< number, @@ -803,7 +813,9 @@ const affectedDetailsPages = reactive( >(), ) const loadingAffectedRuleIds = reactive(new Set()) +const processingOnlyAffectedRuleIds = reactive(new Set()) const AFFECTED_DETAILS_PAGE_SIZE = 3 +const SCAN_RATE_SAMPLE_COUNT = 8 const form = reactive({ name: '', priority: 0 as number | undefined, @@ -824,6 +836,31 @@ onMounted(async () => { }) const modalTitle = computed(() => (editingRuleId.value === null ? 'Create rule' : 'Edit rule')) +const scanEstimatedTimeRemaining = computed(() => { + const seconds = scanEstimatedSecondsRemaining.value + if (seconds === null) return null + + if (seconds < 60) { + const roundedSeconds = Math.max(1, Math.ceil(seconds)) + return `${roundedSeconds} second${roundedSeconds === 1 ? '' : 's'}` + } + + const minutes = Math.ceil(seconds / 60) + if (minutes < 60) return `${minutes} minute${minutes === 1 ? '' : 's'}` + + const hours = Math.floor(minutes / 60) + const remainingMinutes = minutes % 60 + return remainingMinutes === 0 ? `${hours}h` : `${hours}h ${remainingMinutes}m` +}) +const scanProgressLabel = computed(() => { + const progress = scanProgress.value + if (!progress) return 'Preparing scan...' + + const label = `${progress.scanned.toLocaleString()} of ${progress.total.toLocaleString()} details scanned · ${progress.effects.toLocaleString()} effects` + return scanEstimatedTimeRemaining.value + ? `${label} · About ${scanEstimatedTimeRemaining.value} remaining` + : label +}) const { data: issueTypeSchema, isPending: isLoadingIssueTypes, @@ -1245,6 +1282,7 @@ async function loadRules() { try { rules.value = await client.labrinth.tech_review_internal.getRules() affectedDetailsPages.clear() + processingOnlyAffectedRuleIds.clear() } catch (error) { console.error('Failed to load Delphi rules', error) loadFailed.value = true @@ -1313,27 +1351,58 @@ function getVersionLink(detail: Labrinth.TechReview.Internal.DelphiRuleAffectedD return `/project/${detail.project_id}/version/${detail.version_id}` } +function isShowingProcessingAffectedDetailsOnly( + rule: Labrinth.TechReview.Internal.DelphiRule, +): boolean { + return processingOnlyAffectedRuleIds.has(rule.id) +} + +async function setProcessingAffectedDetailsOnly( + rule: Labrinth.TechReview.Internal.DelphiRule, + processingOnly: boolean, +) { + if (loadingAffectedRuleIds.has(rule.id)) return + + if (processingOnly) { + processingOnlyAffectedRuleIds.add(rule.id) + const loaded = await loadAffectedDetailsPage(rule, 1) + if (!loaded) processingOnlyAffectedRuleIds.delete(rule.id) + } else { + processingOnlyAffectedRuleIds.delete(rule.id) + affectedDetailsPages.delete(rule.id) + } +} + async function switchAffectedDetailsPage( rule: Labrinth.TechReview.Internal.DelphiRule, page: number, ) { if (loadingAffectedRuleIds.has(rule.id)) return - if (page === 1) { + if (page === 1 && !isShowingProcessingAffectedDetailsOnly(rule)) { affectedDetailsPages.delete(rule.id) return } + await loadAffectedDetailsPage(rule, page) +} + +async function loadAffectedDetailsPage( + rule: Labrinth.TechReview.Internal.DelphiRule, + page: number, +): Promise { loadingAffectedRuleIds.add(rule.id) try { const response = await client.labrinth.tech_review_internal.getRuleAffectedDetails(rule.id, { limit: AFFECTED_DETAILS_PAGE_SIZE, page: page - 1, + processing_only: isShowingProcessingAffectedDetailsOnly(rule), }) affectedDetailsPages.set(rule.id, { page, total: response.total, details: response.details, }) + return true } catch (error) { console.error('Failed to load details affected by Delphi rule', error) addNotification({ @@ -1341,6 +1410,7 @@ async function switchAffectedDetailsPage( title: 'Failed to load affected details', text: 'The requested page of affected details could not be loaded.', }) + return false } finally { loadingAffectedRuleIds.delete(rule.id) } @@ -1475,11 +1545,45 @@ async function deleteRule() { } } +function updateScanProgress(progress: Labrinth.TechReview.Internal.DelphiRuleScanEvent) { + scanProgress.value = progress + + if (progress.phase !== 'scanning' || progress.total === 0 || progress.scanned >= progress.total) { + scanEstimatedSecondsRemaining.value = null + return + } + + const previousSample = scanProgressSamples.at(-1) + if (!previousSample || progress.scanned > previousSample.scanned) { + scanProgressSamples.push({ scanned: progress.scanned, recordedAt: Date.now() }) + if (scanProgressSamples.length > SCAN_RATE_SAMPLE_COUNT) scanProgressSamples.shift() + } + + const firstSample = scanProgressSamples[0] + const latestSample = scanProgressSamples.at(-1) + if (!firstSample || !latestSample || firstSample === latestSample) { + scanEstimatedSecondsRemaining.value = null + return + } + + const elapsedSeconds = (latestSample.recordedAt - firstSample.recordedAt) / 1000 + const scannedDetails = latestSample.scanned - firstSample.scanned + if (elapsedSeconds <= 0 || scannedDetails <= 0) { + scanEstimatedSecondsRemaining.value = null + return + } + + const detailsPerSecond = scannedDetails / elapsedSeconds + scanEstimatedSecondsRemaining.value = (progress.total - progress.scanned) / detailsPerSecond +} + async function runFullScan() { if (isScanning.value) return isScanning.value = true scanProgress.value = null + scanEstimatedSecondsRemaining.value = null + scanProgressSamples.length = 0 scanAbortController = new AbortController() let completed = false @@ -1490,6 +1594,7 @@ async function runFullScan() { const parser = new SseParser() const processItems = (items: ReturnType) => { + let latestProgress: Labrinth.TechReview.Internal.DelphiRuleScanEvent | null = null for (const item of items) { if (item.kind !== 'event') continue @@ -1501,32 +1606,44 @@ async function runFullScan() { } if (item.event === 'progress' || item.event === 'complete') { - scanProgress.value = JSON.parse( + latestProgress = JSON.parse( item.data, ) as Labrinth.TechReview.Internal.DelphiRuleScanEvent completed ||= item.event === 'complete' } } + + if (!latestProgress) return false + updateScanProgress(latestProgress) + return true } while (true) { const { done, value } = await reader.read() if (done) break - processItems(parser.feed(decoder.decode(value, { stream: true }))) + const progressUpdated = processItems( + parser.feed(decoder.decode(value, { stream: true })), + ) + if (progressUpdated) { + await new Promise((resolve) => setTimeout(resolve, 0)) + } } const finalChunk = decoder.decode() if (finalChunk) processItems(parser.feed(finalChunk)) processItems(parser.end()) - if (!completed || !scanProgress.value) { + const completedProgress = scanProgress.value as + | Labrinth.TechReview.Internal.DelphiRuleScanEvent + | null + if (!completed || !completedProgress) { throw new Error('The scan stream ended before the new revision was published.') } addNotification({ type: 'success', title: 'Rule scan complete', - text: `${scanProgress.value.scanned.toLocaleString()} details were scanned for revision ${scanProgress.value.revision}.`, + text: `${completedProgress.scanned.toLocaleString()} details were scanned for revision ${completedProgress.revision}.`, }) await loadRules() } catch (error) { diff --git a/apps/labrinth/.sqlx/query-bfc21c97440520573234e266dd28b6007451b8c2d993a33b90941aec06136486.json b/apps/labrinth/.sqlx/query-081706789d719f01ef1f86cceadbd42a2cb409e19d2f50059e292dbe7e0fd5ee.json similarity index 60% rename from apps/labrinth/.sqlx/query-bfc21c97440520573234e266dd28b6007451b8c2d993a33b90941aec06136486.json rename to apps/labrinth/.sqlx/query-081706789d719f01ef1f86cceadbd42a2cb409e19d2f50059e292dbe7e0fd5ee.json index 9abbc799a38..c032d1f89b7 100644 --- a/apps/labrinth/.sqlx/query-bfc21c97440520573234e266dd28b6007451b8c2d993a33b90941aec06136486.json +++ b/apps/labrinth/.sqlx/query-081706789d719f01ef1f86cceadbd42a2cb409e19d2f50059e292dbe7e0fd5ee.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n\t\tSELECT\n\t\t\tdelphi_rule.id AS \"id!: DelphiRuleId\",\n\t\t\tdelphi_rule.name,\n\t\t\tdelphi_rule.rule,\n\t\t\tdelphi_rule.priority,\n\t\t\tdelphi_rule.on_issue_types,\n\t\t\tdelphi_rule.revision,\n\t\t\t(\n\t\t\t\tSELECT revision FROM delphi_rule_revisions LIMIT 1\n\t\t\t) AS \"current_revision!\",\n\t\t\tdelphi_rule.created_at,\n\t\t\tdelphi_rule.updated_at,\n\t\t\tdelphi_rule.created_by,\n\t\t\tdelphi_rule.updated_by,\n\t\t\tCOALESCE(preview.affected_details_count, 0)\n\t\t\t\tAS \"affected_details_count!\",\n\t\t\tpreview.detail_id AS \"detail_id?: DelphiReportIssueDetailsId\",\n\t\t\tpreview.issue_id AS \"issue_id?: DelphiReportIssueId\",\n\t\t\tpreview.project_id AS \"project_id?: DBProjectId\",\n\t\t\tpreview.project_name AS \"project_name?\",\n\t\t\tpreview.project_icon_url AS \"project_icon_url?\",\n\t\t\tpreview.version_id AS \"version_id?: DBVersionId\",\n\t\t\tpreview.version_name AS \"version_name?\",\n\t\t\tpreview.version_number AS \"version_number?\",\n\t\t\tpreview.issue_type AS \"issue_type?\",\n\t\t\tpreview.key AS \"key?\",\n\t\t\tpreview.jar AS \"jar?\",\n\t\t\tpreview.file_path AS \"file_path?\",\n\t\t\tpreview.original_severity AS \"original_severity?: DelphiSeverity\",\n\t\t\tpreview.severity AS \"effect_severity?: DelphiSeverity\"\n\t\tFROM delphi_rules delphi_rule\n\t\tLEFT JOIN LATERAL (\n\t\t\tSELECT\n\t\t\t\teffect.detail_id,\n\t\t\t\tdetail.issue_id,\n\t\t\t\tversion.mod_id AS project_id,\n\t\t\t\tproject.name AS project_name,\n\t\t\t\tproject.icon_url AS project_icon_url,\n\t\t\t\tversion.id AS version_id,\n\t\t\t\tversion.name AS version_name,\n\t\t\t\tversion.version_number,\n\t\t\t\tissue.issue_type,\n\t\t\t\tdetail.key,\n\t\t\t\tdetail.jar,\n\t\t\t\tdetail.file_path,\n\t\t\t\tdetail.severity AS original_severity,\n\t\t\t\teffect.severity,\n\t\t\t\tCOUNT(*) OVER () AS affected_details_count\n\t\t\tFROM delphi_rule_effects effect\n\t\t\tINNER JOIN delphi_rule_revisions published\n\t\t\t\tON published.revision = effect.revision\n\t\t\tINNER JOIN delphi_report_issue_details detail\n\t\t\t\tON detail.id = effect.detail_id\n\t\t\tINNER JOIN delphi_report_issues issue\n\t\t\t\tON issue.id = detail.issue_id\n\t\t\tINNER JOIN delphi_reports report\n\t\t\t\tON report.id = issue.report_id\n\t\t\tLEFT JOIN files file ON file.id = report.file_id\n\t\t\tLEFT JOIN versions version ON version.id = file.version_id\n\t\t\tLEFT JOIN mods project ON project.id = version.mod_id\n\t\t\tWHERE effect.rule_id = delphi_rule.id\n\t\t\tORDER BY effect.detail_id DESC\n\t\t\tLIMIT 3\n\t\t) preview ON TRUE\n\t\tWHERE NOT delphi_rule.delete_on_next_revision\n\t\tORDER BY\n\t\t\tdelphi_rule.priority DESC,\n\t\t\tdelphi_rule.id,\n\t\t\tpreview.detail_id DESC\n\t\t", + "query": "\n\t\tSELECT\n\t\t\tdelphi_rule.id AS \"id!: DelphiRuleId\",\n\t\t\tdelphi_rule.name,\n\t\t\tdelphi_rule.rule,\n\t\t\tdelphi_rule.priority,\n\t\t\tdelphi_rule.on_issue_types,\n\t\t\tdelphi_rule.revision,\n\t\t\t(\n\t\t\t\tSELECT revision FROM delphi_rule_revisions LIMIT 1\n\t\t\t) AS \"current_revision!\",\n\t\t\tdelphi_rule.created_at,\n\t\t\tdelphi_rule.updated_at,\n\t\t\tdelphi_rule.created_by,\n\t\t\tdelphi_rule.updated_by,\n\t\t\tCOALESCE(affected.affected_details_count, 0)\n\t\t\t\tAS \"affected_details_count!\",\n\t\t\tpreview.detail_id AS \"detail_id?: DelphiReportIssueDetailsId\",\n\t\t\tpreview.issue_id AS \"issue_id?: DelphiReportIssueId\",\n\t\t\tpreview.project_id AS \"project_id?: DBProjectId\",\n\t\t\tpreview.project_name AS \"project_name?\",\n\t\t\tpreview.project_icon_url AS \"project_icon_url?\",\n\t\t\tpreview.version_id AS \"version_id?: DBVersionId\",\n\t\t\tpreview.version_name AS \"version_name?\",\n\t\t\tpreview.version_number AS \"version_number?\",\n\t\t\tpreview.issue_type AS \"issue_type?\",\n\t\t\tpreview.key AS \"key?\",\n\t\t\tpreview.jar AS \"jar?\",\n\t\t\tpreview.file_path AS \"file_path?\",\n\t\t\tpreview.original_severity AS \"original_severity?: DelphiSeverity\",\n\t\t\tpreview.severity AS \"effect_severity?: DelphiSeverity\"\n\t\tFROM delphi_rules delphi_rule\n\t\tLEFT JOIN (\n\t\t\tSELECT\n\t\t\t\teffect.rule_id,\n\t\t\t\tCOUNT(*) AS affected_details_count\n\t\t\tFROM delphi_rule_effects effect\n\t\t\tINNER JOIN delphi_rule_revisions published\n\t\t\t\tON published.revision = effect.revision\n\t\t\tGROUP BY effect.rule_id\n\t\t) affected ON affected.rule_id = delphi_rule.id\n\t\tLEFT JOIN LATERAL (\n\t\t\tSELECT\n\t\t\t\teffect.detail_id,\n\t\t\t\tdetail.issue_id,\n\t\t\t\tversion.mod_id AS project_id,\n\t\t\t\tproject.name AS project_name,\n\t\t\t\tproject.icon_url AS project_icon_url,\n\t\t\t\tversion.id AS version_id,\n\t\t\t\tversion.name AS version_name,\n\t\t\t\tversion.version_number,\n\t\t\t\tissue.issue_type,\n\t\t\t\tdetail.key,\n\t\t\t\tdetail.jar,\n\t\t\t\tdetail.file_path,\n\t\t\t\tdetail.severity AS original_severity,\n\t\t\t\teffect.severity\n\t\t\tFROM (\n\t\t\t\tSELECT effect.detail_id, effect.severity\n\t\t\t\tFROM delphi_rule_effects effect\n\t\t\t\tINNER JOIN delphi_rule_revisions published\n\t\t\t\t\tON published.revision = effect.revision\n\t\t\t\tWHERE effect.rule_id = delphi_rule.id\n\t\t\t\tORDER BY effect.detail_id DESC\n\t\t\t\tLIMIT 3\n\t\t\t) effect\n\t\t\tINNER JOIN delphi_report_issue_details detail\n\t\t\t\tON detail.id = effect.detail_id\n\t\t\tINNER JOIN delphi_report_issues issue\n\t\t\t\tON issue.id = detail.issue_id\n\t\t\tINNER JOIN delphi_reports report\n\t\t\t\tON report.id = issue.report_id\n\t\t\tLEFT JOIN files file ON file.id = report.file_id\n\t\t\tLEFT JOIN versions version ON version.id = file.version_id\n\t\t\tLEFT JOIN mods project ON project.id = version.mod_id\n\t\t\tORDER BY effect.detail_id DESC\n\t\t) preview ON TRUE\n\t\tWHERE NOT delphi_rule.delete_on_next_revision\n\t\tORDER BY\n\t\t\tdelphi_rule.priority DESC,\n\t\t\tdelphi_rule.id,\n\t\t\tpreview.detail_id DESC\n\t\t", "describe": { "columns": [ { @@ -194,5 +194,5 @@ false ] }, - "hash": "bfc21c97440520573234e266dd28b6007451b8c2d993a33b90941aec06136486" + "hash": "081706789d719f01ef1f86cceadbd42a2cb409e19d2f50059e292dbe7e0fd5ee" } diff --git a/apps/labrinth/.sqlx/query-20ebdaf894827f6084af18aedab2f6ce511a659c11e348f30afeb43007a90ee2.json b/apps/labrinth/.sqlx/query-20ebdaf894827f6084af18aedab2f6ce511a659c11e348f30afeb43007a90ee2.json new file mode 100644 index 00000000000..c65ae0b32cb --- /dev/null +++ b/apps/labrinth/.sqlx/query-20ebdaf894827f6084af18aedab2f6ce511a659c11e348f30afeb43007a90ee2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tSELECT COUNT(*) AS \"count!\"\n\t\tFROM delphi_rule_effects effect\n\t\tINNER JOIN delphi_rule_revisions published\n\t\t\tON published.revision = effect.revision\n\t\tWHERE\n\t\t\teffect.rule_id = $1\n\t\t\tAND (\n\t\t\t\tNOT $2\n\t\t\t\tOR EXISTS (\n\t\t\t\t\tSELECT 1\n\t\t\t\t\tFROM delphi_report_issue_details detail\n\t\t\t\t\tINNER JOIN delphi_report_issues issue\n\t\t\t\t\t\tON issue.id = detail.issue_id\n\t\t\t\t\tINNER JOIN delphi_reports report\n\t\t\t\t\t\tON report.id = issue.report_id\n\t\t\t\t\tINNER JOIN files file ON file.id = report.file_id\n\t\t\t\t\tINNER JOIN versions version ON version.id = file.version_id\n\t\t\t\t\tINNER JOIN mods project ON project.id = version.mod_id\n\t\t\t\t\tWHERE\n\t\t\t\t\t\tdetail.id = effect.detail_id\n\t\t\t\t\t\tAND project.status = 'processing'\n\t\t\t\t)\n\t\t\t)\n\t\t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Bool" + ] + }, + "nullable": [ + null + ] + }, + "hash": "20ebdaf894827f6084af18aedab2f6ce511a659c11e348f30afeb43007a90ee2" +} diff --git a/apps/labrinth/.sqlx/query-285c66152a506e79c3c8996b4f44b8f63dcf79a2d02dcfeac9655d89bdb2750e.json b/apps/labrinth/.sqlx/query-285c66152a506e79c3c8996b4f44b8f63dcf79a2d02dcfeac9655d89bdb2750e.json deleted file mode 100644 index 5d0c4829060..00000000000 --- a/apps/labrinth/.sqlx/query-285c66152a506e79c3c8996b4f44b8f63dcf79a2d02dcfeac9655d89bdb2750e.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM delphi_tech_review_queue\n WHERE project_id = $1\n RETURNING project_id AS \"project_id: DBProjectId\"\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "project_id: DBProjectId", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "285c66152a506e79c3c8996b4f44b8f63dcf79a2d02dcfeac9655d89bdb2750e" -} diff --git a/apps/labrinth/.sqlx/query-348f71a746d67cbbbcaf48c42f2228eff41da7c2846d0f2be5c06830a1189ad4.json b/apps/labrinth/.sqlx/query-348f71a746d67cbbbcaf48c42f2228eff41da7c2846d0f2be5c06830a1189ad4.json deleted file mode 100644 index c6a72838f03..00000000000 --- a/apps/labrinth/.sqlx/query-348f71a746d67cbbbcaf48c42f2228eff41da7c2846d0f2be5c06830a1189ad4.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n\t\tSELECT COUNT(*) AS \"count!\"\n\t\tFROM delphi_rule_effects effect\n\t\tINNER JOIN delphi_rule_revisions published\n\t\t\tON published.revision = effect.revision\n\t\tWHERE effect.rule_id = $1\n\t\t", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "348f71a746d67cbbbcaf48c42f2228eff41da7c2846d0f2be5c06830a1189ad4" -} diff --git a/apps/labrinth/.sqlx/query-91dafadcd637b3f8bfc78f0ea06b8a2a0a0d0d37636e98191e2a9d7453686391.json b/apps/labrinth/.sqlx/query-91dafadcd637b3f8bfc78f0ea06b8a2a0a0d0d37636e98191e2a9d7453686391.json new file mode 100644 index 00000000000..e7e3ed7f4a1 --- /dev/null +++ b/apps/labrinth/.sqlx/query-91dafadcd637b3f8bfc78f0ea06b8a2a0a0d0d37636e98191e2a9d7453686391.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM delphi_tech_review_queue\n WHERE project_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "91dafadcd637b3f8bfc78f0ea06b8a2a0a0d0d37636e98191e2a9d7453686391" +} diff --git a/apps/labrinth/.sqlx/query-f40f23ca7f9d3c42a6f4c505afc7dd596c9829b5f0b42ca1b2830b0f68e33882.json b/apps/labrinth/.sqlx/query-abfe1f90c9d311e21107f994c241526d473aae2cab2106e05947b816a4d1c0d0.json similarity index 93% rename from apps/labrinth/.sqlx/query-f40f23ca7f9d3c42a6f4c505afc7dd596c9829b5f0b42ca1b2830b0f68e33882.json rename to apps/labrinth/.sqlx/query-abfe1f90c9d311e21107f994c241526d473aae2cab2106e05947b816a4d1c0d0.json index bc9858831e5..5ea700c3dab 100644 --- a/apps/labrinth/.sqlx/query-f40f23ca7f9d3c42a6f4c505afc7dd596c9829b5f0b42ca1b2830b0f68e33882.json +++ b/apps/labrinth/.sqlx/query-abfe1f90c9d311e21107f994c241526d473aae2cab2106e05947b816a4d1c0d0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n\t\tSELECT\n\t\t\teffect.detail_id AS \"detail_id!: DelphiReportIssueDetailsId\",\n\t\t\tdetail.issue_id AS \"issue_id!: DelphiReportIssueId\",\n\t\t\tversion.mod_id AS \"project_id?: DBProjectId\",\n\t\t\tproject.name AS \"project_name?\",\n\t\t\tproject.icon_url AS \"project_icon_url?\",\n\t\t\tversion.id AS \"version_id?: DBVersionId\",\n\t\t\tversion.name AS \"version_name?\",\n\t\t\tversion.version_number AS \"version_number?\",\n\t\t\tissue.issue_type,\n\t\t\tdetail.key,\n\t\t\tdetail.jar,\n\t\t\tdetail.file_path,\n\t\t\tdetail.severity AS \"original_severity!: DelphiSeverity\",\n\t\t\teffect.severity AS \"effect_severity!: DelphiSeverity\"\n\t\tFROM delphi_rule_effects effect\n\t\tINNER JOIN delphi_rule_revisions published\n\t\t\tON published.revision = effect.revision\n\t\tINNER JOIN delphi_report_issue_details detail\n\t\t\tON detail.id = effect.detail_id\n\t\tINNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id\n\t\tINNER JOIN delphi_reports report ON report.id = issue.report_id\n\t\tLEFT JOIN files file ON file.id = report.file_id\n\t\tLEFT JOIN versions version ON version.id = file.version_id\n\t\tLEFT JOIN mods project ON project.id = version.mod_id\n\t\tWHERE effect.rule_id = $1\n\t\tORDER BY effect.detail_id DESC\n\t\tLIMIT $2 OFFSET $3\n\t\t", + "query": "\n\t\tSELECT\n\t\t\teffect.detail_id AS \"detail_id!: DelphiReportIssueDetailsId\",\n\t\t\tdetail.issue_id AS \"issue_id!: DelphiReportIssueId\",\n\t\t\tversion.mod_id AS \"project_id?: DBProjectId\",\n\t\t\tproject.name AS \"project_name?\",\n\t\t\tproject.icon_url AS \"project_icon_url?\",\n\t\t\tversion.id AS \"version_id?: DBVersionId\",\n\t\t\tversion.name AS \"version_name?\",\n\t\t\tversion.version_number AS \"version_number?\",\n\t\t\tissue.issue_type,\n\t\t\tdetail.key,\n\t\t\tdetail.jar,\n\t\t\tdetail.file_path,\n\t\t\tdetail.severity AS \"original_severity!: DelphiSeverity\",\n\t\t\teffect.severity AS \"effect_severity!: DelphiSeverity\"\n\t\tFROM delphi_rule_effects effect\n\t\tINNER JOIN delphi_rule_revisions published\n\t\t\tON published.revision = effect.revision\n\t\tINNER JOIN delphi_report_issue_details detail\n\t\t\tON detail.id = effect.detail_id\n\t\tINNER JOIN delphi_report_issues issue ON issue.id = detail.issue_id\n\t\tINNER JOIN delphi_reports report ON report.id = issue.report_id\n\t\tLEFT JOIN files file ON file.id = report.file_id\n\t\tLEFT JOIN versions version ON version.id = file.version_id\n\t\tLEFT JOIN mods project ON project.id = version.mod_id\n\t\tWHERE\n\t\t\teffect.rule_id = $1\n\t\t\tAND (NOT $2 OR project.status = 'processing')\n\t\tORDER BY effect.detail_id DESC\n\t\tLIMIT $3 OFFSET $4\n\t\t", "describe": { "columns": [ { @@ -105,6 +105,7 @@ "parameters": { "Left": [ "Int8", + "Bool", "Int8", "Int8" ] @@ -126,5 +127,5 @@ false ] }, - "hash": "f40f23ca7f9d3c42a6f4c505afc7dd596c9829b5f0b42ca1b2830b0f68e33882" + "hash": "abfe1f90c9d311e21107f994c241526d473aae2cab2106e05947b816a4d1c0d0" } diff --git a/apps/labrinth/migrations/20260917120000_delphi_rule_effects_pagination.sql b/apps/labrinth/migrations/20260917120000_delphi_rule_effects_pagination.sql new file mode 100644 index 00000000000..2ebbde37c3b --- /dev/null +++ b/apps/labrinth/migrations/20260917120000_delphi_rule_effects_pagination.sql @@ -0,0 +1,5 @@ +DROP INDEX IF EXISTS delphi_rule_effects_rule_id; + +CREATE INDEX IF NOT EXISTS delphi_rule_effects_rule_id_detail_id +ON delphi_rule_effects (rule_id, detail_id DESC) +INCLUDE (revision, severity); diff --git a/apps/labrinth/src/routes/internal/moderation/tech_review.rs b/apps/labrinth/src/routes/internal/moderation/tech_review.rs index 2b49c0c9ba3..1394c5feaed 100644 --- a/apps/labrinth/src/routes/internal/moderation/tech_review.rs +++ b/apps/labrinth/src/routes/internal/moderation/tech_review.rs @@ -1161,18 +1161,18 @@ pub async fn submit_report( ))); } - sqlx::query_scalar!( + sqlx::query!( r#" DELETE FROM delphi_tech_review_queue WHERE project_id = $1 - RETURNING project_id AS "project_id: DBProjectId" "#, project_id as DBProjectId, ) - .fetch_optional(&mut txn) + .execute(&mut txn) .await - .wrap_internal_err("failed to remove project from technical review queue")? - .wrap_not_found_err("project not found in technical review queue")?; + .wrap_internal_err( + "failed to remove project from technical review queue", + )?; let record = sqlx::query!( r#" diff --git a/apps/labrinth/src/routes/internal/moderation/tech_review/rules.rs b/apps/labrinth/src/routes/internal/moderation/tech_review/rules.rs index 08480650451..e6fa468c8da 100644 --- a/apps/labrinth/src/routes/internal/moderation/tech_review/rules.rs +++ b/apps/labrinth/src/routes/internal/moderation/tech_review/rules.rs @@ -82,6 +82,8 @@ pub struct GetRuleAffectedDetailsRequest { #[serde(default)] #[schema(default = 0)] pub page: u64, + #[serde(default)] + pub processing_only: bool, } #[derive(Debug, Serialize, utoipa::ToSchema)] @@ -269,7 +271,7 @@ pub async fn get_rules( delphi_rule.updated_at, delphi_rule.created_by, delphi_rule.updated_by, - COALESCE(preview.affected_details_count, 0) + COALESCE(affected.affected_details_count, 0) AS "affected_details_count!", preview.detail_id AS "detail_id?: DelphiReportIssueDetailsId", preview.issue_id AS "issue_id?: DelphiReportIssueId", @@ -286,6 +288,15 @@ pub async fn get_rules( preview.original_severity AS "original_severity?: DelphiSeverity", preview.severity AS "effect_severity?: DelphiSeverity" FROM delphi_rules delphi_rule + LEFT JOIN ( + SELECT + effect.rule_id, + COUNT(*) AS affected_details_count + FROM delphi_rule_effects effect + INNER JOIN delphi_rule_revisions published + ON published.revision = effect.revision + GROUP BY effect.rule_id + ) affected ON affected.rule_id = delphi_rule.id LEFT JOIN LATERAL ( SELECT effect.detail_id, @@ -301,11 +312,16 @@ pub async fn get_rules( detail.jar, detail.file_path, detail.severity AS original_severity, - effect.severity, - COUNT(*) OVER () AS affected_details_count - FROM delphi_rule_effects effect - INNER JOIN delphi_rule_revisions published - ON published.revision = effect.revision + effect.severity + FROM ( + SELECT effect.detail_id, effect.severity + FROM delphi_rule_effects effect + INNER JOIN delphi_rule_revisions published + ON published.revision = effect.revision + WHERE effect.rule_id = delphi_rule.id + ORDER BY effect.detail_id DESC + LIMIT 3 + ) effect INNER JOIN delphi_report_issue_details detail ON detail.id = effect.detail_id INNER JOIN delphi_report_issues issue @@ -315,9 +331,7 @@ pub async fn get_rules( LEFT JOIN files file ON file.id = report.file_id LEFT JOIN versions version ON version.id = file.version_id LEFT JOIN mods project ON project.id = version.mod_id - WHERE effect.rule_id = delphi_rule.id ORDER BY effect.detail_id DESC - LIMIT 3 ) preview ON TRUE WHERE NOT delphi_rule.delete_on_next_revision ORDER BY @@ -406,7 +420,8 @@ pub async fn get_rules( params( ("id" = DelphiRuleId, Path), ("limit" = Option, Query), - ("page" = Option, Query) + ("page" = Option, Query), + ("processing_only" = Option, Query) ), responses((status = OK, body = GetRuleAffectedDetailsResponse)) )] @@ -443,9 +458,28 @@ pub async fn get_rule_affected_details( FROM delphi_rule_effects effect INNER JOIN delphi_rule_revisions published ON published.revision = effect.revision - WHERE effect.rule_id = $1 + WHERE + effect.rule_id = $1 + AND ( + NOT $2 + OR EXISTS ( + SELECT 1 + FROM delphi_report_issue_details detail + INNER JOIN delphi_report_issues issue + ON issue.id = detail.issue_id + INNER JOIN delphi_reports report + ON report.id = issue.report_id + INNER JOIN files file ON file.id = report.file_id + INNER JOIN versions version ON version.id = file.version_id + INNER JOIN mods project ON project.id = version.mod_id + WHERE + detail.id = effect.detail_id + AND project.status = 'processing' + ) + ) "#, rule_id as DelphiRuleId, + query.processing_only, ) .fetch_one(&***ro_pool) .await @@ -478,11 +512,14 @@ pub async fn get_rule_affected_details( LEFT JOIN files file ON file.id = report.file_id LEFT JOIN versions version ON version.id = file.version_id LEFT JOIN mods project ON project.id = version.mod_id - WHERE effect.rule_id = $1 + WHERE + effect.rule_id = $1 + AND (NOT $2 OR project.status = 'processing') ORDER BY effect.detail_id DESC - LIMIT $2 OFFSET $3 + LIMIT $3 OFFSET $4 "#, rule_id as DelphiRuleId, + query.processing_only, limit, offset, ) diff --git a/apps/labrinth/src/routes/internal/moderation/tech_review/rules_scan.rs b/apps/labrinth/src/routes/internal/moderation/tech_review/rules_scan.rs index d9edcd43b68..0250e0457c1 100644 --- a/apps/labrinth/src/routes/internal/moderation/tech_review/rules_scan.rs +++ b/apps/labrinth/src/routes/internal/moderation/tech_review/rules_scan.rs @@ -1,14 +1,18 @@ -use std::collections::{BTreeMap, HashMap}; +use std::{ + collections::{BTreeMap, HashMap}, + time::Duration, +}; -use actix_web::{HttpRequest, HttpResponse, get, post, web}; +use actix_web::{ + HttpRequest, HttpResponse, get, http::header::ContentEncoding, post, web, +}; use ariadne::ids::base62_impl::to_base62; use bytes::Bytes; use eyre::{Result, eyre}; -use futures_util::{StreamExt, TryStreamExt}; +use futures_util::TryStreamExt; use serde::{Deserialize, Serialize}; use sqlx::types::Json; use tokio::sync::mpsc; -use tokio_stream::wrappers::UnboundedReceiverStream; use utoipa::{PartialSchema, ToSchema}; use xredis::RedisPool; @@ -33,6 +37,7 @@ use crate::{ const RULE_SCAN_LOCK_ID: i64 = 0x6465_6c70_6869_7275; const PROGRESS_INTERVAL: usize = 50; +const SSE_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); pub fn config(cfg: &mut actix_web::web::ServiceConfig) { cfg.service(get_rule_schema) .service(get_detail_rule_input) @@ -393,6 +398,8 @@ pub async fn scan_rules( } let (sender, receiver) = mpsc::unbounded_channel(); + let _ = sender.send(Bytes::from_static(b": connected\n\n")); + actix_web::rt::spawn(async move { match run_scan(transaction, &sender).await { Ok(summary) => { @@ -422,12 +429,31 @@ pub async fn scan_rules( } }); - let stream = - UnboundedReceiverStream::new(receiver).map(Ok::<_, std::io::Error>); + let mut heartbeat = tokio::time::interval_at( + tokio::time::Instant::now() + SSE_HEARTBEAT_INTERVAL, + SSE_HEARTBEAT_INTERVAL, + ); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let stream = futures_util::stream::unfold( + (receiver, heartbeat), + |(mut receiver, mut heartbeat)| async move { + tokio::select! { + event = receiver.recv() => event.map(|event| ( + Ok::<_, std::io::Error>(event), + (receiver, heartbeat), + )), + _ = heartbeat.tick() => Some(( + Ok(Bytes::from_static(b": keepalive\n\n")), + (receiver, heartbeat), + )), + } + }, + ); Ok(HttpResponse::Ok() .insert_header(("Content-Type", "text/event-stream")) - .insert_header(("Cache-Control", "no-cache")) + .insert_header(ContentEncoding::Identity) + .insert_header(("Cache-Control", "no-cache, no-transform")) .insert_header(("X-Accel-Buffering", "no")) .streaming(stream)) } diff --git a/apps/labrinth/src/routes/mod.rs b/apps/labrinth/src/routes/mod.rs index 37130663421..3e6a98236e6 100644 --- a/apps/labrinth/src/routes/mod.rs +++ b/apps/labrinth/src/routes/mod.rs @@ -136,7 +136,10 @@ pub fn root_config(cfg: &mut web::ServiceConfig) { .wrap(default_cors()) .service(index::index_get) .service(index::build_get) - .service(Files::new("/", "assets/")), + .service(Files::new( + "/", + concat!(env!("CARGO_MANIFEST_DIR"), "/assets"), + )), ); } diff --git a/packages/api-client/src/modules/labrinth/types.ts b/packages/api-client/src/modules/labrinth/types.ts index e105386f05d..47ed9aa8626 100644 --- a/packages/api-client/src/modules/labrinth/types.ts +++ b/packages/api-client/src/modules/labrinth/types.ts @@ -2540,6 +2540,7 @@ export namespace Labrinth { export type GetRuleAffectedDetailsRequest = { limit?: number page?: number + processing_only?: boolean } export type GetRuleAffectedDetailsResponse = { diff --git a/packages/ui/src/components/base/ProgressBar.vue b/packages/ui/src/components/base/ProgressBar.vue index c47a493d509..7ff4aad832a 100644 --- a/packages/ui/src/components/base/ProgressBar.vue +++ b/packages/ui/src/components/base/ProgressBar.vue @@ -11,6 +11,7 @@ const props = withDefaults( fullWidth?: boolean striped?: boolean gradientBorder?: boolean + animated?: boolean label?: string labelClass?: string showProgress?: boolean @@ -22,6 +23,7 @@ const props = withDefaults( fullWidth: false, striped: false, gradientBorder: true, + animated: true, showProgress: false, }, ) @@ -83,6 +85,7 @@ const percent = computed(() => props.progress / props.max) class="rounded-full progress-bar" :class="[ colors[props.color].fg, + { 'progress-bar--animated': animated }, { 'progress-bar--waiting': waiting }, { 'progress-bar--gradient-border': gradientBorder }, striped ? `progress-bar--striped--${color}` : '', @@ -93,7 +96,7 @@ const percent = computed(() => props.progress / props.max)