diff --git a/.github/workflows/frontend-deploy.yml b/.github/workflows/frontend-deploy.yml index 0d9d80be621..6e260251e57 100644 --- a/.github/workflows/frontend-deploy.yml +++ b/.github/workflows/frontend-deploy.yml @@ -126,6 +126,12 @@ jobs: BUILD_ENV: ${{ steps.meta.outputs.env }} PREVIEW: ${{ inputs.preview && 'true' || 'false' }} + - name: Disable server source map uploads for previews + if: ${{ inputs.preview }} + working-directory: ./apps/frontend + run: | + sed -i 's/"upload_source_maps": true/"upload_source_maps": false/g' wrangler.jsonc + - name: Install Wrangler CLI uses: cloudflare/wrangler-action@953926a2e2182532811c01a25e53647d93bf07c0 # v4.1.3 with: diff --git a/Cargo.lock b/Cargo.lock index 43dae65372f..ffaf74a0c49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -396,6 +396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -2945,6 +2946,20 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "serde", + "static_assertions", + "zmij", +] + [[package]] name = "component-derive" version = "0.0.0" @@ -6785,6 +6800,7 @@ dependencies = [ "redis", "regex", "reqwest 0.13.5", + "rhai", "rust_decimal", "rust_iso3166", "rustls 0.23.46", @@ -6818,6 +6834,7 @@ dependencies = [ "url", "urlencoding", "utoipa", + "utoipa-actix-web", "uuid 1.28.0", "validator", "webauthn-rs", @@ -8524,6 +8541,15 @@ dependencies = [ "libc", ] +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +dependencies = [ + "spin 0.5.2", +] + [[package]] name = "no_std_io2" version = "0.9.4" @@ -11137,6 +11163,37 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "rhai" +version = "1.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3925b784c0bee6542637e425ed7511a63257bf11b7ecf51ffd845e2d24d9163" +dependencies = [ + "ahash", + "bitflags 2.13.2", + "compact_str 0.10.0", + "no-std-compat", + "num-traits", + "once_cell", + "rhai_codegen", + "serde", + "smallvec", + "smartstring", + "thin-vec", + "web-time", +] + +[[package]] +name = "rhai_codegen" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d9661edb4a4b8145b017582d634c5eb88ff1a5284a5aa701f83a28f0657a826" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "ring" version = "0.17.14" @@ -11539,7 +11596,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826e80413b9a35e9d33217b3dcac04cf95f6559d15944b93887a08be5496c4a4" dependencies = [ - "compact_str", + "compact_str 0.7.1", ] [[package]] @@ -12399,6 +12456,17 @@ dependencies = [ "serde", ] +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "smol_str" version = "0.3.7" @@ -12492,6 +12560,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.9" @@ -13877,6 +13951,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -15405,6 +15485,17 @@ dependencies = [ "utoipa-gen", ] +[[package]] +name = "utoipa-actix-web" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3af2eb471d6791cfead324a893342eb73530d48bd37c485dd6296bc9fd8101ed" +dependencies = [ + "actix-service", + "actix-web", + "utoipa", +] + [[package]] name = "utoipa-gen" version = "6.0.1" diff --git a/Cargo.toml b/Cargo.toml index 24154a871e9..ca462e9d00d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -183,6 +183,7 @@ redis = "1.7.1" regex = "1.13.1" reqwest = { version = "0.13.5", default-features = false, features = ["form", "query"] } rgb = "0.8.53" +rhai = "1.27.0" rust_decimal = { version = "1.43.0", features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = "0.2.0" rust-s3 = { version = "0.38.0", default-features = false, features = [ @@ -264,6 +265,7 @@ unicode-segmentation = "1.13.3" url = "2.5.8" urlencoding = "2.1.3" utoipa = { version = "6.0.0", features = ["actix_extras", "chrono", "decimal"] } +utoipa-actix-web = "0.2.0" uuid = "1.28.0" validator = "0.21.0" webauthn-rs = "0.5.5" diff --git a/apps/frontend/src/components/ui/moderation/project-flag/FileScanPatternsEditor.vue b/apps/frontend/src/components/ui/moderation/project-flag/FileScanPatternsEditor.vue new file mode 100644 index 00000000000..48402bc3d91 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/FileScanPatternsEditor.vue @@ -0,0 +1,78 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagInputSchema.vue b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagInputSchema.vue new file mode 100644 index 00000000000..bf0266403e0 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagInputSchema.vue @@ -0,0 +1,52 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue new file mode 100644 index 00000000000..0239d5c6ce0 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue @@ -0,0 +1,131 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuns.vue b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuns.vue new file mode 100644 index 00000000000..f8adbada8c8 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuns.vue @@ -0,0 +1,501 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/project-flag/RhaiExpressionEditor.vue b/apps/frontend/src/components/ui/moderation/project-flag/RhaiExpressionEditor.vue new file mode 100644 index 00000000000..b90a01fd832 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/RhaiExpressionEditor.vue @@ -0,0 +1,75 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/project-flag/ace-rhai.ts b/apps/frontend/src/components/ui/moderation/project-flag/ace-rhai.ts new file mode 100644 index 00000000000..457bcf37458 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/ace-rhai.ts @@ -0,0 +1,153 @@ +import 'ace-builds/src-noconflict/mode-rust' + +import type { Ace } from 'ace-builds' +import ace from 'ace-builds' + +const RHAI_KEYWORDS = + 'let const fn private if else switch for in while loop do until break continue return throw try catch import export as is this' + +const RHAI_SNIPPETS = [ + { caption: 'let', snippet: 'let ${1:name} = ${2:value};' }, + { caption: 'const', snippet: 'const ${1:NAME} = ${2:value};' }, + { caption: 'fn', snippet: 'fn ${1:name}(${2:args}) {\n\t${0}\n}' }, + { caption: 'if', snippet: 'if ${1:condition} {\n\t${0}\n}' }, + { caption: 'for', snippet: 'for ${1:item} in ${2:items} {\n\t${0}\n}' }, + { caption: 'while', snippet: 'while ${1:condition} {\n\t${0}\n}' }, + { caption: 'try', snippet: 'try {\n\t${1}\n} catch (${2:error}) {\n\t${0}\n}' }, +] + +const RHAI_COMPLETER: Ace.Completer = { + getCompletions(_editor, _session, _position, _prefix, callback) { + callback(null, [ + ...`${RHAI_KEYWORDS} true false`.split(' ').map((value) => ({ + caption: value, + value, + meta: 'Rhai keyword', + score: 100, + })), + ...RHAI_SNIPPETS.map((snippet) => ({ ...snippet, meta: 'Rhai snippet', score: 90 })), + ]) + }, +} + +export const RHAI_EDITOR_OPTIONS: Partial = { + useWorker: false, + tabSize: 4, + useSoftTabs: false, + enableBasicAutocompletion: [RHAI_COMPLETER], + enableLiveAutocompletion: [RHAI_COMPLETER], + enableSnippets: true, +} + +ace['define']( + 'ace/mode/rhai_highlight_rules', + ['require', 'exports', 'ace/lib/oop', 'ace/mode/text_highlight_rules'], + function (require, exports) { + const oop = require('ace/lib/oop') + const TextHighlightRules = require('ace/mode/text_highlight_rules').TextHighlightRules + + const RhaiHighlightRules = function (this: any) { + const keywords = this.createKeywordMapper( + { + keyword: RHAI_KEYWORDS.replaceAll(' ', '|'), + 'constant.language': 'true|false', + }, + 'identifier', + false, + ) + + this.$rules = { + start: [ + { token: 'comment', regex: /\/\/.*$/.source }, + { token: 'comment', regex: /\/\*/.source, push: 'comment' }, + { token: 'string', regex: /"/.source, push: 'string' }, + { token: 'string', regex: /`/.source, push: 'template' }, + { + token: 'string', + regex: /'(?:\\(?:u\{[\da-fA-F]+\}|u[\da-fA-F]{4}|x[\da-fA-F]{2}|.)|[^'\\])'/.source, + }, + { token: 'constant.language', regex: /\(\s*\)/.source }, + { + token: 'constant.numeric', + regex: + /\b(?:0x[\da-fA-F][\da-fA-F_]*|0o[0-7][0-7_]*|0b[01][01_]*|\d[\d_]*(?:\.(?!\.)\d[\d_]*)?(?:[eE][+-]?\d[\d_]*)?)\b/ + .source, + }, + { + token: (value: string) => { + const token = keywords(value) + return token === 'identifier' ? 'entity.name.function' : token + }, + regex: /[a-zA-Z_][\w]*(?=\s*\()/.source, + }, + { token: keywords, regex: /[a-zA-Z_][\w]*/.source }, + { token: 'paren.lparen', regex: /#\{|[[({]/.source }, + { token: 'paren.rparen', regex: /[\])}]/.source }, + { + token: 'keyword.operator', + regex: + /\*\*=|<<=|>>=|\.\.=|==|!=|<=|>=|&&|\|\||\?\?|\?\.|=>|::|\*\*|<<|>>|\.\.|[+\-*/%&|^]=?|[!~<>=?:.]/ + .source, + }, + { token: 'punctuation.operator', regex: /[,;]/.source }, + { token: 'text', regex: /\s+/.source }, + ], + comment: [ + { token: 'comment', regex: /\/\*/.source, push: 'comment' }, + { token: 'comment', regex: /\*\//.source, next: 'pop' }, + { defaultToken: 'comment' }, + ], + string: [ + { token: 'constant.language.escape', regex: /\\./.source }, + { token: 'string', regex: /"/.source, next: 'pop' }, + { defaultToken: 'string' }, + ], + template: [ + { token: 'constant.language.escape', regex: /\\./.source }, + { token: 'paren.lparen', regex: /\$\{/.source, push: 'interpolation' }, + { token: 'string', regex: /`/.source, next: 'pop' }, + { defaultToken: 'string' }, + ], + interpolation: [ + { token: 'paren.lparen', regex: /#?\{/.source, push: 'interpolation' }, + { token: 'paren.rparen', regex: /\}/.source, next: 'pop' }, + { include: 'start' }, + ], + } + this.normalizeRules() + } + + oop.inherits(RhaiHighlightRules, TextHighlightRules) + exports.RhaiHighlightRules = RhaiHighlightRules + }, +) + +ace['define']( + 'ace/mode/rhai', + [ + 'require', + 'exports', + 'ace/lib/oop', + 'ace/mode/rust', + 'ace/mode/rhai_highlight_rules', + 'ace/mode/folding/cstyle', + ], + function (require, exports) { + const oop = require('ace/lib/oop') + const RustMode = require('ace/mode/rust').Mode + const RhaiHighlightRules = require('ace/mode/rhai_highlight_rules').RhaiHighlightRules + const FoldMode = require('ace/mode/folding/cstyle').FoldMode + + const Mode = function (this: any) { + RustMode.call(this) + this.HighlightRules = RhaiHighlightRules + this.foldingRules = new FoldMode() + this.lineCommentStart = '//' + this.blockComment = { start: '/*', end: '*/', nestable: true } + this.$id = 'ace/mode/rhai' + } + + oop.inherits(Mode, RustMode) + exports.Mode = Mode + }, +) diff --git a/apps/frontend/src/pages/moderation.vue b/apps/frontend/src/pages/moderation.vue index 7919b16054d..2a47c8a85e1 100644 --- a/apps/frontend/src/pages/moderation.vue +++ b/apps/frontend/src/pages/moderation.vue @@ -75,6 +75,7 @@ const moderationLinks = [ href: '/moderation/technical-review/rules', icon: SettingsIcon, }, + { label: 'Project flag rules', href: '/moderation/project-flag', icon: SettingsIcon }, ] const mobileNavOptions = [ @@ -83,6 +84,7 @@ const mobileNavOptions = [ formatMessage(messages.externalFilesTitle), formatMessage(messages.globalDetailTracesTitle), formatMessage(messages.delphiRulesTitle), + 'Project flag rules', ] const selectedChip = computed({ @@ -90,6 +92,8 @@ const selectedChip = computed({ const path = route.path if (path.startsWith('/moderation/technical-review/rules')) { return formatMessage(messages.delphiRulesTitle) + } else if (path.startsWith('/moderation/project-flag')) { + return 'Project flag rules' } else if (path.startsWith('/moderation/reports')) { return formatMessage(messages.reportsTitle) } else if (path.startsWith('/moderation/external-projects')) { @@ -114,6 +118,8 @@ function navigateToPage(selectedOption: string) { router.push('/moderation/global-traces') } else if (selectedOption === formatMessage(messages.delphiRulesTitle)) { router.push('/moderation/technical-review/rules') + } else if (selectedOption === 'Project flag rules') { + router.push('/moderation/project-flag') } else { router.push('/moderation') } diff --git a/apps/frontend/src/pages/moderation/project-flag.vue b/apps/frontend/src/pages/moderation/project-flag.vue new file mode 100644 index 00000000000..96996887388 --- /dev/null +++ b/apps/frontend/src/pages/moderation/project-flag.vue @@ -0,0 +1,413 @@ + + + diff --git a/apps/labrinth/.sqlx/query-052ac329820a9e2fa9951f4b39a10dee885caa54ae8082b6cf9417ce8dd17597.json b/apps/labrinth/.sqlx/query-052ac329820a9e2fa9951f4b39a10dee885caa54ae8082b6cf9417ce8dd17597.json new file mode 100644 index 00000000000..967f55f5dcd --- /dev/null +++ b/apps/labrinth/.sqlx/query-052ac329820a9e2fa9951f4b39a10dee885caa54ae8082b6cf9417ce8dd17597.json @@ -0,0 +1,104 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE project_flag_rules\n SET name = $2, params = $3, updated_by = $4, updated_at = $5\n WHERE id = $1\n RETURNING id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Varchar", + "Jsonb", + "Int8", + "Timestamptz" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "052ac329820a9e2fa9951f4b39a10dee885caa54ae8082b6cf9417ce8dd17597" +} diff --git a/apps/labrinth/.sqlx/query-059765ac84ce39c47f99bcf415bd307f5bf46989fdbae4d0829141301f61b36d.json b/apps/labrinth/.sqlx/query-059765ac84ce39c47f99bcf415bd307f5bf46989fdbae4d0829141301f61b36d.json new file mode 100644 index 00000000000..dd7871e2484 --- /dev/null +++ b/apps/labrinth/.sqlx/query-059765ac84ce39c47f99bcf415bd307f5bf46989fdbae4d0829141301f61b36d.json @@ -0,0 +1,98 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n FROM project_flag_rules\n ORDER BY id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "059765ac84ce39c47f99bcf415bd307f5bf46989fdbae4d0829141301f61b36d" +} diff --git a/apps/labrinth/.sqlx/query-0a1106008e49501cc33f418974c0dce39f0d8deaca7775e9d5ff8f0d41a044b0.json b/apps/labrinth/.sqlx/query-0a1106008e49501cc33f418974c0dce39f0d8deaca7775e9d5ff8f0d41a044b0.json new file mode 100644 index 00000000000..d970933f6e5 --- /dev/null +++ b/apps/labrinth/.sqlx/query-0a1106008e49501cc33f418974c0dce39f0d8deaca7775e9d5ff8f0d41a044b0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO project_flag_run_results\n (id, run_id, project_id, rules, error)\n SELECT results.id, results.run_id, results.project_id, results.rules, results.error\n FROM jsonb_to_recordset($1) AS results(\n id UUID, run_id UUID, project_id BIGINT, rules JSONB, error TEXT\n )\n LEFT JOIN project_flag_runs AS runs ON runs.id = results.run_id\n WHERE results.run_id IS NULL OR runs.status = 'running'\n ORDER BY results.run_id, results.project_id\n ON CONFLICT (run_id, project_id) DO UPDATE\n SET rules = EXCLUDED.rules,\n error = EXCLUDED.error, evaluated_at = CURRENT_TIMESTAMP\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "0a1106008e49501cc33f418974c0dce39f0d8deaca7775e9d5ff8f0d41a044b0" +} diff --git a/apps/labrinth/.sqlx/query-0fe059fc4a9430c807795fd7b909723b6889d7e462ca1c7e87320caed28f9a7c.json b/apps/labrinth/.sqlx/query-0fe059fc4a9430c807795fd7b909723b6889d7e462ca1c7e87320caed28f9a7c.json new file mode 100644 index 00000000000..176c2abd4b8 --- /dev/null +++ b/apps/labrinth/.sqlx/query-0fe059fc4a9430c807795fd7b909723b6889d7e462ca1c7e87320caed28f9a7c.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, name, version_number, date_published\n FROM versions\n WHERE id = ANY($1)\n ORDER BY date_published DESC, id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "versions", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "version_number", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "versions", + "name": "version_number" + } + } + }, + { + "ordinal": 3, + "name": "date_published", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "versions", + "name": "date_published" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "0fe059fc4a9430c807795fd7b909723b6889d7e462ca1c7e87320caed28f9a7c" +} diff --git a/apps/labrinth/.sqlx/query-0fe6c80b9d078cf9457ba72716c793f4eeb9fedaa2cfd6ac7bdd99f10b14c511.json b/apps/labrinth/.sqlx/query-0fe6c80b9d078cf9457ba72716c793f4eeb9fedaa2cfd6ac7bdd99f10b14c511.json new file mode 100644 index 00000000000..355c8aec4d3 --- /dev/null +++ b/apps/labrinth/.sqlx/query-0fe6c80b9d078cf9457ba72716c793f4eeb9fedaa2cfd6ac7bdd99f10b14c511.json @@ -0,0 +1,148 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, title, status, link, exceptions, proof, flame_project_id,\n inserted_at, inserted_by, updated_at, updated_by\n FROM moderation_external_licenses\n WHERE id = ANY($1)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "title", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "title" + } + } + }, + { + "ordinal": 2, + "name": "status", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "status" + } + } + }, + { + "ordinal": 3, + "name": "link", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "link" + } + } + }, + { + "ordinal": 4, + "name": "exceptions", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "exceptions" + } + } + }, + { + "ordinal": 5, + "name": "proof", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "proof" + } + } + }, + { + "ordinal": 6, + "name": "flame_project_id", + "type_info": "Int4", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "flame_project_id" + } + } + }, + { + "ordinal": 7, + "name": "inserted_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "inserted_at" + } + } + }, + { + "ordinal": 8, + "name": "inserted_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "inserted_by" + } + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "updated_at" + } + } + }, + { + "ordinal": 10, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "updated_by" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "0fe6c80b9d078cf9457ba72716c793f4eeb9fedaa2cfd6ac7bdd99f10b14c511" +} diff --git a/apps/labrinth/.sqlx/query-11769eb1c8764857579505a52cd9ab18ca959bce216193d8c1aa7cb61d9c1ffc.json b/apps/labrinth/.sqlx/query-11769eb1c8764857579505a52cd9ab18ca959bce216193d8c1aa7cb61d9c1ffc.json new file mode 100644 index 00000000000..071a15182d6 --- /dev/null +++ b/apps/labrinth/.sqlx/query-11769eb1c8764857579505a52cd9ab18ca959bce216193d8c1aa7cb61d9c1ffc.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE project_flags SET run_id = NULL WHERE run_id = $1 AND active", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "11769eb1c8764857579505a52cd9ab18ca959bce216193d8c1aa7cb61d9c1ffc" +} diff --git a/apps/labrinth/.sqlx/query-13d3731070096afa00698998459ea32ead13b547476e3900ee767d4fedd3cb68.json b/apps/labrinth/.sqlx/query-13d3731070096afa00698998459ea32ead13b547476e3900ee767d4fedd3cb68.json new file mode 100644 index 00000000000..e0d73a701f2 --- /dev/null +++ b/apps/labrinth/.sqlx/query-13d3731070096afa00698998459ea32ead13b547476e3900ee767d4fedd3cb68.json @@ -0,0 +1,101 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n FROM project_flag_rules\n ORDER BY created_at DESC, id DESC\n LIMIT $1 OFFSET $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "13d3731070096afa00698998459ea32ead13b547476e3900ee767d4fedd3cb68" +} diff --git a/apps/labrinth/.sqlx/query-15ca266ff7c1f2f7f6dfe10defafacb4b5b6b6eeb3519ac614e2aea07dad5dfb.json b/apps/labrinth/.sqlx/query-15ca266ff7c1f2f7f6dfe10defafacb4b5b6b6eeb3519ac614e2aea07dad5dfb.json new file mode 100644 index 00000000000..8ab51c69893 --- /dev/null +++ b/apps/labrinth/.sqlx/query-15ca266ff7c1f2f7f6dfe10defafacb4b5b6b6eeb3519ac614e2aea07dad5dfb.json @@ -0,0 +1,100 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n FROM project_flag_rules\n WHERE id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "15ca266ff7c1f2f7f6dfe10defafacb4b5b6b6eeb3519ac614e2aea07dad5dfb" +} diff --git a/apps/labrinth/.sqlx/query-18b4cd2bb06995aebdd157900b14d2a0476843c808a03f54a7e628dfa479ec0b.json b/apps/labrinth/.sqlx/query-18b4cd2bb06995aebdd157900b14d2a0476843c808a03f54a7e628dfa479ec0b.json new file mode 100644 index 00000000000..b09439b73cb --- /dev/null +++ b/apps/labrinth/.sqlx/query-18b4cd2bb06995aebdd157900b14d2a0476843c808a03f54a7e628dfa479ec0b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT COUNT(*)\n FROM project_flag_run_results\n WHERE run_id = runs.id\n ) AS \"total!\"\n FROM project_flag_runs AS runs\n WHERE runs.id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "18b4cd2bb06995aebdd157900b14d2a0476843c808a03f54a7e628dfa479ec0b" +} diff --git a/apps/labrinth/.sqlx/query-2026e6f5ceac9e6591acf88fc741a24d798a699f43f9c2bef8198244efc57038.json b/apps/labrinth/.sqlx/query-2026e6f5ceac9e6591acf88fc741a24d798a699f43f9c2bef8198244efc57038.json new file mode 100644 index 00000000000..3d0255b806a --- /dev/null +++ b/apps/labrinth/.sqlx/query-2026e6f5ceac9e6591acf88fc741a24d798a699f43f9c2bef8198244efc57038.json @@ -0,0 +1,83 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT source_version.mod_id AS project_id,\n ofs.file_path AS \"file_path!\", CONVERT_FROM(ofs.sha1, 'UTF8') AS \"sha1!\",\n source_file.version_id, platform_file.version_id AS platform_version_id,\n platform_version.mod_id AS platform_project_id\n FROM files source_file\n INNER JOIN versions source_version ON source_version.id = source_file.version_id\n INNER JOIN override_file_sources ofs ON ofs.file_id = source_file.id\n INNER JOIN hashes h ON h.algorithm = 'sha1' AND h.hash = ofs.sha1\n INNER JOIN files platform_file ON platform_file.id = h.file_id\n INNER JOIN versions platform_version ON platform_version.id = platform_file.version_id\n WHERE source_version.mod_id = ANY($1)\n ORDER BY source_version.mod_id, source_file.version_id, ofs.file_path,\n platform_file.version_id, platform_version.mod_id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "project_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "mod_id" + } + } + }, + { + "ordinal": 1, + "name": "file_path!", + "type_info": "Text", + "origin": { + "Table": { + "table": "override_file_sources", + "name": "file_path" + } + } + }, + { + "ordinal": 2, + "name": "sha1!", + "type_info": "Text", + "origin": "Expression" + }, + { + "ordinal": 3, + "name": "version_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "files", + "name": "version_id" + } + } + }, + { + "ordinal": 4, + "name": "platform_version_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "files", + "name": "version_id" + } + } + }, + { + "ordinal": 5, + "name": "platform_project_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "mod_id" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + null, + false, + false, + false + ] + }, + "hash": "2026e6f5ceac9e6591acf88fc741a24d798a699f43f9c2bef8198244efc57038" +} diff --git a/apps/labrinth/.sqlx/query-2c5eaf5fe4a303de0dc848e84b28770dc4d19ce2968a4a97be5f4705710191b1.json b/apps/labrinth/.sqlx/query-2c5eaf5fe4a303de0dc848e84b28770dc4d19ce2968a4a97be5f4705710191b1.json new file mode 100644 index 00000000000..1edaa099c14 --- /dev/null +++ b/apps/labrinth/.sqlx/query-2c5eaf5fe4a303de0dc848e84b28770dc4d19ce2968a4a97be5f4705710191b1.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM project_flag_runs WHERE id = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "2c5eaf5fe4a303de0dc848e84b28770dc4d19ce2968a4a97be5f4705710191b1" +} diff --git a/apps/labrinth/.sqlx/query-325c48f77ec92f11c104c5c4353c04dd48751a393877b9923e047eef5bb1d9e4.json b/apps/labrinth/.sqlx/query-325c48f77ec92f11c104c5c4353c04dd48751a393877b9923e047eef5bb1d9e4.json new file mode 100644 index 00000000000..9bd2e62deb2 --- /dev/null +++ b/apps/labrinth/.sqlx/query-325c48f77ec92f11c104c5c4353c04dd48751a393877b9923e047eef5bb1d9e4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM project_flag_runs\n WHERE id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "325c48f77ec92f11c104c5c4353c04dd48751a393877b9923e047eef5bb1d9e4" +} diff --git a/apps/labrinth/.sqlx/query-3ea3b382fd37491079984f165801d969860f2758c6ef2354997d8451abdc4468.json b/apps/labrinth/.sqlx/query-3ea3b382fd37491079984f165801d969860f2758c6ef2354997d8451abdc4468.json new file mode 100644 index 00000000000..89161cdf027 --- /dev/null +++ b/apps/labrinth/.sqlx/query-3ea3b382fd37491079984f165801d969860f2758c6ef2354997d8451abdc4468.json @@ -0,0 +1,104 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO project_flag_rules\n (id, name, params, created_by, created_at, updated_by, updated_at)\n VALUES ($1, $2, $3, $4, $5, $4, $5)\n RETURNING id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Varchar", + "Jsonb", + "Int8", + "Timestamptz" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "3ea3b382fd37491079984f165801d969860f2758c6ef2354997d8451abdc4468" +} diff --git a/apps/labrinth/.sqlx/query-43a668954d69d7d450f73077d6fa89fd9d47eedd1e627079ddc3cb89bc9ec4f2.json b/apps/labrinth/.sqlx/query-4b5ae530df1467e490fb7c2c02ef35f18d45db9476dcdeeeb6ffef851f30a582.json similarity index 92% rename from apps/labrinth/.sqlx/query-43a668954d69d7d450f73077d6fa89fd9d47eedd1e627079ddc3cb89bc9ec4f2.json rename to apps/labrinth/.sqlx/query-4b5ae530df1467e490fb7c2c02ef35f18d45db9476dcdeeeb6ffef851f30a582.json index b11e945aafa..b95ad779930 100644 --- a/apps/labrinth/.sqlx/query-43a668954d69d7d450f73077d6fa89fd9d47eedd1e627079ddc3cb89bc9ec4f2.json +++ b/apps/labrinth/.sqlx/query-4b5ae530df1467e490fb7c2c02ef35f18d45db9476dcdeeeb6ffef851f30a582.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n\t\t\tSELECT project_id, type AS \"disclosure_type!\", metadata, updated_at, updated_by, set_by_moderator, deleted_at, lock_status\n\t\t\tFROM project_disclosures\n\t\t\tWHERE project_id = $1 AND ($2 OR deleted_at IS NULL)\n\t\t\tORDER BY updated_at DESC\n\t\t\t", + "query": "\n\t\t\tSELECT project_id, type AS \"disclosure_type!\", metadata, updated_at, updated_by, set_by_moderator, deleted_at, lock_status\n\t\t\tFROM project_disclosures\n\t\t\tWHERE project_id = ANY($1) AND ($2 OR deleted_at IS NULL)\n\t\t\tORDER BY updated_at DESC\n\t\t\t", "describe": { "columns": [ { @@ -94,7 +94,7 @@ ], "parameters": { "Left": [ - "Int8", + "Int8Array", "Bool" ] }, @@ -109,5 +109,5 @@ false ] }, - "hash": "43a668954d69d7d450f73077d6fa89fd9d47eedd1e627079ddc3cb89bc9ec4f2" + "hash": "4b5ae530df1467e490fb7c2c02ef35f18d45db9476dcdeeeb6ffef851f30a582" } diff --git a/apps/labrinth/.sqlx/query-4c8bbeb507e28b1f6418429c7d676d39e66b4cf70fce3f5c79a8204b30150248.json b/apps/labrinth/.sqlx/query-4c8bbeb507e28b1f6418429c7d676d39e66b4cf70fce3f5c79a8204b30150248.json new file mode 100644 index 00000000000..c1c295735a1 --- /dev/null +++ b/apps/labrinth/.sqlx/query-4c8bbeb507e28b1f6418429c7d676d39e66b4cf70fce3f5c79a8204b30150248.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO project_flag_run_pending_jobs (run_id, project_id)\n SELECT $1, project_id FROM UNNEST($2::BIGINT[]) AS jobs(project_id)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "4c8bbeb507e28b1f6418429c7d676d39e66b4cf70fce3f5c79a8204b30150248" +} diff --git a/apps/labrinth/.sqlx/query-4d434f2ec68539ec88460fb325be96bcee32af66a8a754a5e1dcac03929a224f.json b/apps/labrinth/.sqlx/query-4d434f2ec68539ec88460fb325be96bcee32af66a8a754a5e1dcac03929a224f.json new file mode 100644 index 00000000000..d03ee52bcf2 --- /dev/null +++ b/apps/labrinth/.sqlx/query-4d434f2ec68539ec88460fb325be96bcee32af66a8a754a5e1dcac03929a224f.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id FROM mods\n WHERE id IN (SELECT project_id FROM project_flags WHERE run_id = $1)\n ORDER BY id\n FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "mods", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4d434f2ec68539ec88460fb325be96bcee32af66a8a754a5e1dcac03929a224f" +} diff --git a/apps/labrinth/.sqlx/query-4f71596f731158bdf72079623b147944e4d9193d8fbd43696f958e21018fb376.json b/apps/labrinth/.sqlx/query-4f71596f731158bdf72079623b147944e4d9193d8fbd43696f958e21018fb376.json new file mode 100644 index 00000000000..25f0d323031 --- /dev/null +++ b/apps/labrinth/.sqlx/query-4f71596f731158bdf72079623b147944e4d9193d8fbd43696f958e21018fb376.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name FROM project_flag_rules WHERE id = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "4f71596f731158bdf72079623b147944e4d9193d8fbd43696f958e21018fb376" +} diff --git a/apps/labrinth/.sqlx/query-500eb2a79f8c8eac49dbfb4181cbad1b653ad58a4ad8242892c461dcd2ba0283.json b/apps/labrinth/.sqlx/query-500eb2a79f8c8eac49dbfb4181cbad1b653ad58a4ad8242892c461dcd2ba0283.json new file mode 100644 index 00000000000..a094b05ad97 --- /dev/null +++ b/apps/labrinth/.sqlx/query-500eb2a79f8c8eac49dbfb4181cbad1b653ad58a4ad8242892c461dcd2ba0283.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT COUNT(*) AS \"num_scanning!\"\n FROM project_flag_run_pending_jobs AS pending\n JOIN project_flag_runs AS runs ON runs.id = pending.run_id\n WHERE runs.status = 'running'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "num_scanning!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "500eb2a79f8c8eac49dbfb4181cbad1b653ad58a4ad8242892c461dcd2ba0283" +} diff --git a/apps/labrinth/.sqlx/query-5037929f9ef2c47da0890a08443e2bd431b199cb238859619ab499e71cbee727.json b/apps/labrinth/.sqlx/query-5037929f9ef2c47da0890a08443e2bd431b199cb238859619ab499e71cbee727.json new file mode 100644 index 00000000000..aa14d66ba71 --- /dev/null +++ b/apps/labrinth/.sqlx/query-5037929f9ef2c47da0890a08443e2bd431b199cb238859619ab499e71cbee727.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT flags.what AS \"what?: sqlx::types::Json\"\n FROM mods AS projects\n LEFT JOIN project_flags AS flags ON flags.project_id = projects.id\n AND (flags.active OR (flags.what->>'retain_if_inactive')::bool = TRUE)\n WHERE projects.id = $1\n ORDER BY flags.what->>'key', flags.id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "what?: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flags", + "name": "what" + } + } + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5037929f9ef2c47da0890a08443e2bd431b199cb238859619ab499e71cbee727" +} diff --git a/apps/labrinth/.sqlx/query-63c0675524e625a090cfb56b1cb8d561f635ffcc0891f4c2e8dbfadedc8fe651.json b/apps/labrinth/.sqlx/query-63c0675524e625a090cfb56b1cb8d561f635ffcc0891f4c2e8dbfadedc8fe651.json new file mode 100644 index 00000000000..742b503facb --- /dev/null +++ b/apps/labrinth/.sqlx/query-63c0675524e625a090cfb56b1cb8d561f635ffcc0891f4c2e8dbfadedc8fe651.json @@ -0,0 +1,88 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, project_id, flame_project, attribution, attributed_at, attributed_by\n FROM project_attribution_groups\n WHERE project_id = ANY($1)\n ORDER BY project_id, id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "project_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "project_id" + } + } + }, + { + "ordinal": 2, + "name": "flame_project", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "flame_project" + } + } + }, + { + "ordinal": 3, + "name": "attribution", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "attribution" + } + } + }, + { + "ordinal": 4, + "name": "attributed_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "attributed_at" + } + } + }, + { + "ordinal": 5, + "name": "attributed_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "attributed_by" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + true, + true, + true, + true + ] + }, + "hash": "63c0675524e625a090cfb56b1cb8d561f635ffcc0891f4c2e8dbfadedc8fe651" +} diff --git a/apps/labrinth/.sqlx/query-7285d8fc34604cc1364decbbc5dca1260cc250ea4455402f84e9f1530c31df87.json b/apps/labrinth/.sqlx/query-7285d8fc34604cc1364decbbc5dca1260cc250ea4455402f84e9f1530c31df87.json new file mode 100644 index 00000000000..f8095f9d8bf --- /dev/null +++ b/apps/labrinth/.sqlx/query-7285d8fc34604cc1364decbbc5dca1260cc250ea4455402f84e9f1530c31df87.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM mods WHERE status = 'processing'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "mods", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "7285d8fc34604cc1364decbbc5dca1260cc250ea4455402f84e9f1530c31df87" +} diff --git a/apps/labrinth/.sqlx/query-7f46fbe0d71881039fff48a46daaa1df4277617ca79a16d4335036aaca435f6c.json b/apps/labrinth/.sqlx/query-7f46fbe0d71881039fff48a46daaa1df4277617ca79a16d4335036aaca435f6c.json new file mode 100644 index 00000000000..620bcf82f2b --- /dev/null +++ b/apps/labrinth/.sqlx/query-7f46fbe0d71881039fff48a46daaa1df4277617ca79a16d4335036aaca435f6c.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id FROM mods\n WHERE id = ANY($1)\n ORDER BY id\n FOR KEY SHARE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "mods", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7f46fbe0d71881039fff48a46daaa1df4277617ca79a16d4335036aaca435f6c" +} diff --git a/apps/labrinth/.sqlx/query-8d75fb4aa2c9f79ed7054ac4b63090e66bfc8fe6679e49c34a30d3b0343ecf9a.json b/apps/labrinth/.sqlx/query-8d75fb4aa2c9f79ed7054ac4b63090e66bfc8fe6679e49c34a30d3b0343ecf9a.json new file mode 100644 index 00000000000..a33d4d32429 --- /dev/null +++ b/apps/labrinth/.sqlx/query-8d75fb4aa2c9f79ed7054ac4b63090e66bfc8fe6679e49c34a30d3b0343ecf9a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM project_flag_run_pending_jobs AS pending\n USING UNNEST($1::UUID[], $2::BIGINT[]) AS consumed(run_id, project_id)\n WHERE pending.run_id = consumed.run_id\n AND pending.project_id = consumed.project_id\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "8d75fb4aa2c9f79ed7054ac4b63090e66bfc8fe6679e49c34a30d3b0343ecf9a" +} diff --git a/apps/labrinth/.sqlx/query-99a623492d62c03e0c9f4dfe5358d4c2c10d4c95bdc084a9e87288b87d37fd1a.json b/apps/labrinth/.sqlx/query-99a623492d62c03e0c9f4dfe5358d4c2c10d4c95bdc084a9e87288b87d37fd1a.json new file mode 100644 index 00000000000..1576fce8d11 --- /dev/null +++ b/apps/labrinth/.sqlx/query-99a623492d62c03e0c9f4dfe5358d4c2c10d4c95bdc084a9e87288b87d37fd1a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM project_flag_rules\n WHERE id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "99a623492d62c03e0c9f4dfe5358d4c2c10d4c95bdc084a9e87288b87d37fd1a" +} diff --git a/apps/labrinth/.sqlx/query-a9450319c2576b9a3076fed184e0e0b2f1028be4ef44d663c0ea09127ad9f1b7.json b/apps/labrinth/.sqlx/query-a9450319c2576b9a3076fed184e0e0b2f1028be4ef44d663c0ea09127ad9f1b7.json new file mode 100644 index 00000000000..9f83cb088cc --- /dev/null +++ b/apps/labrinth/.sqlx/query-a9450319c2576b9a3076fed184e0e0b2f1028be4ef44d663c0ea09127ad9f1b7.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT paf.group_id AS \"group_id!\", paf.name AS \"name!\",\n CONVERT_FROM(paf.sha1, 'UTF8') AS \"sha1!\",\n paf.moderation_external_license_id,\n COALESCE(ARRAY_AGG(DISTINCT aev.id) FILTER (WHERE aev.id IS NOT NULL), '{}') AS \"version_ids!: Vec\"\n FROM project_attribution_files paf\n INNER JOIN project_attribution_groups pag ON pag.id = paf.group_id\n LEFT JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1\n LEFT JOIN files f ON f.id = ofs.file_id\n LEFT JOIN versions v ON v.id = f.version_id AND v.mod_id = pag.project_id\n LEFT JOIN attribution_enforced_versions aev ON aev.id = v.id\n WHERE paf.group_id = ANY($1)\n GROUP BY paf.group_id, paf.name, paf.sha1, paf.moderation_external_license_id\n ORDER BY paf.group_id, paf.name, paf.sha1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "group_id!", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_files", + "name": "group_id" + } + } + }, + { + "ordinal": 1, + "name": "name!", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_attribution_files", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "sha1!", + "type_info": "Text", + "origin": "Expression" + }, + { + "ordinal": 3, + "name": "moderation_external_license_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_files", + "name": "moderation_external_license_id" + } + } + }, + { + "ordinal": 4, + "name": "version_ids!: Vec", + "type_info": "Int8Array", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + null, + true, + null + ] + }, + "hash": "a9450319c2576b9a3076fed184e0e0b2f1028be4ef44d663c0ea09127ad9f1b7" +} diff --git a/apps/labrinth/.sqlx/query-b29282c47f6768fdb629d1add77bac85b1c4cec7216aa11ded2032c5a80b889e.json b/apps/labrinth/.sqlx/query-b29282c47f6768fdb629d1add77bac85b1c4cec7216aa11ded2032c5a80b889e.json new file mode 100644 index 00000000000..e5a95a69e7b --- /dev/null +++ b/apps/labrinth/.sqlx/query-b29282c47f6768fdb629d1add77bac85b1c4cec7216aa11ded2032c5a80b889e.json @@ -0,0 +1,88 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT v.mod_id AS project_id, f.id, f.version_id, f.filename, f.url, f.size\n FROM files f\n INNER JOIN versions v ON v.id = f.version_id\n WHERE v.mod_id = ANY($1)\n ORDER BY v.mod_id, v.date_published DESC, v.id DESC, f.id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "project_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "mod_id" + } + } + }, + { + "ordinal": 1, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "files", + "name": "id" + } + } + }, + { + "ordinal": 2, + "name": "version_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "files", + "name": "version_id" + } + } + }, + { + "ordinal": 3, + "name": "filename", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "files", + "name": "filename" + } + } + }, + { + "ordinal": 4, + "name": "url", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "files", + "name": "url" + } + } + }, + { + "ordinal": 5, + "name": "size", + "type_info": "Int4", + "origin": { + "Table": { + "table": "files", + "name": "size" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false + ] + }, + "hash": "b29282c47f6768fdb629d1add77bac85b1c4cec7216aa11ded2032c5a80b889e" +} diff --git a/apps/labrinth/.sqlx/query-b69bb764f5b415b75e830fb11d272c89491848279a33298d20218cfeea9dc2b4.json b/apps/labrinth/.sqlx/query-b69bb764f5b415b75e830fb11d272c89491848279a33298d20218cfeea9dc2b4.json new file mode 100644 index 00000000000..54f20ff89c4 --- /dev/null +++ b/apps/labrinth/.sqlx/query-b69bb764f5b415b75e830fb11d272c89491848279a33298d20218cfeea9dc2b4.json @@ -0,0 +1,102 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT projects.id, projects.name AS title, projects.slug, projects.icon_url,\n results.rules AS \"rules: sqlx::types::Json\",\n results.error, results.evaluated_at\n FROM project_flag_run_results AS results\n INNER JOIN mods AS projects ON projects.id = results.project_id\n WHERE results.run_id = $1\n ORDER BY results.project_id\n LIMIT $2 OFFSET $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "mods", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "title", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "mods", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "slug", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "mods", + "name": "slug" + } + } + }, + { + "ordinal": 3, + "name": "icon_url", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "mods", + "name": "icon_url" + } + } + }, + { + "ordinal": 4, + "name": "rules: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_run_results", + "name": "rules" + } + } + }, + { + "ordinal": 5, + "name": "error", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_flag_run_results", + "name": "error" + } + } + }, + { + "ordinal": 6, + "name": "evaluated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_run_results", + "name": "evaluated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + true, + false, + true, + false + ] + }, + "hash": "b69bb764f5b415b75e830fb11d272c89491848279a33298d20218cfeea9dc2b4" +} diff --git a/apps/labrinth/.sqlx/query-bcfe1a5dc34a9f65e2a8de9b1b00ae5a1ffad6e9af8e0435180f743bcbfb2a18.json b/apps/labrinth/.sqlx/query-bcfe1a5dc34a9f65e2a8de9b1b00ae5a1ffad6e9af8e0435180f743bcbfb2a18.json new file mode 100644 index 00000000000..1e7c7c44920 --- /dev/null +++ b/apps/labrinth/.sqlx/query-bcfe1a5dc34a9f65e2a8de9b1b00ae5a1ffad6e9af8e0435180f743bcbfb2a18.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE project_flag_runs\n SET status = $2, completed_at = $3, error = $4\n WHERE id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Timestamptz", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bcfe1a5dc34a9f65e2a8de9b1b00ae5a1ffad6e9af8e0435180f743bcbfb2a18" +} diff --git a/apps/labrinth/.sqlx/query-c1e150a9a958dcb9fc06ee3b997bb9e44c2a1d6b167904e6b8b6af3800440fd2.json b/apps/labrinth/.sqlx/query-c1e150a9a958dcb9fc06ee3b997bb9e44c2a1d6b167904e6b8b6af3800440fd2.json new file mode 100644 index 00000000000..3bdddf8099d --- /dev/null +++ b/apps/labrinth/.sqlx/query-c1e150a9a958dcb9fc06ee3b997bb9e44c2a1d6b167904e6b8b6af3800440fd2.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM versions WHERE mod_id = ANY($1) ORDER BY date_published DESC, id DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c1e150a9a958dcb9fc06ee3b997bb9e44c2a1d6b167904e6b8b6af3800440fd2" +} diff --git a/apps/labrinth/.sqlx/query-cd998091abca3723b3c356ecff256c8f4e7e97b41de37fa6c205eee1e7d0092e.json b/apps/labrinth/.sqlx/query-cd998091abca3723b3c356ecff256c8f4e7e97b41de37fa6c205eee1e7d0092e.json new file mode 100644 index 00000000000..567be933fdc --- /dev/null +++ b/apps/labrinth/.sqlx/query-cd998091abca3723b3c356ecff256c8f4e7e97b41de37fa6c205eee1e7d0092e.json @@ -0,0 +1,168 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT runs.id, runs.status AS \"status: ProjectFlagRunStatus\", runs.total_projects,\n starter.id AS \"started_by_id?\", starter.username AS \"started_by_username?\",\n applier.id AS \"applied_by_id?\", applier.username AS \"applied_by_username?\",\n runs.started_at, runs.completed_at, runs.applied_at,\n runs.output AS \"output: sqlx::types::Json\", runs.error,\n (SELECT COUNT(*) FROM project_flag_run_results AS results\n WHERE results.run_id = runs.id AND results.error IS NOT NULL\n ) AS \"num_errors!\"\n FROM project_flag_runs AS runs\n LEFT JOIN users AS starter ON starter.id = runs.started_by\n LEFT JOIN users AS applier ON applier.id = runs.applied_by\n ORDER BY runs.started_at DESC, runs.id DESC\n LIMIT $1 OFFSET $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "status: ProjectFlagRunStatus", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "status" + } + } + }, + { + "ordinal": 2, + "name": "total_projects", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "total_projects" + } + } + }, + { + "ordinal": 3, + "name": "started_by_id?", + "type_info": "Int8", + "origin": { + "Table": { + "table": "users", + "name": "id" + } + } + }, + { + "ordinal": 4, + "name": "started_by_username?", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "users", + "name": "username" + } + } + }, + { + "ordinal": 5, + "name": "applied_by_id?", + "type_info": "Int8", + "origin": { + "Table": { + "table": "users", + "name": "id" + } + } + }, + { + "ordinal": 6, + "name": "applied_by_username?", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "users", + "name": "username" + } + } + }, + { + "ordinal": 7, + "name": "started_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "started_at" + } + } + }, + { + "ordinal": 8, + "name": "completed_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "completed_at" + } + } + }, + { + "ordinal": 9, + "name": "applied_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "applied_at" + } + } + }, + { + "ordinal": 10, + "name": "output: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "output" + } + } + }, + { + "ordinal": 11, + "name": "error", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "error" + } + } + }, + { + "ordinal": 12, + "name": "num_errors!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + null + ] + }, + "hash": "cd998091abca3723b3c356ecff256c8f4e7e97b41de37fa6c205eee1e7d0092e" +} diff --git a/apps/labrinth/.sqlx/query-de3230de507ca1e11d2ca40bef8a5b8470628ddbaa454af4f49f6fe6953f9014.json b/apps/labrinth/.sqlx/query-de3230de507ca1e11d2ca40bef8a5b8470628ddbaa454af4f49f6fe6953f9014.json new file mode 100644 index 00000000000..1b72c3ca9f0 --- /dev/null +++ b/apps/labrinth/.sqlx/query-de3230de507ca1e11d2ca40bef8a5b8470628ddbaa454af4f49f6fe6953f9014.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT username FROM users WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "users", + "name": "username" + } + } + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "de3230de507ca1e11d2ca40bef8a5b8470628ddbaa454af4f49f6fe6953f9014" +} diff --git a/apps/labrinth/.sqlx/query-e19110b3c4230a260b0007d8d35edf55b6de4c4a124216006512dc05572ed78e.json b/apps/labrinth/.sqlx/query-e19110b3c4230a260b0007d8d35edf55b6de4c4a124216006512dc05572ed78e.json new file mode 100644 index 00000000000..5ac24edd279 --- /dev/null +++ b/apps/labrinth/.sqlx/query-e19110b3c4230a260b0007d8d35edf55b6de4c4a124216006512dc05572ed78e.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id\n FROM project_flag_runs\n WHERE id = ANY($1)\n ORDER BY id\n FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e19110b3c4230a260b0007d8d35edf55b6de4c4a124216006512dc05572ed78e" +} diff --git a/apps/labrinth/.sqlx/query-e8b537da45864ca001d74485a118b6ac860ceece6a3cd45ef26860cd78737dad.json b/apps/labrinth/.sqlx/query-e8b537da45864ca001d74485a118b6ac860ceece6a3cd45ef26860cd78737dad.json new file mode 100644 index 00000000000..c38b111f1b1 --- /dev/null +++ b/apps/labrinth/.sqlx/query-e8b537da45864ca001d74485a118b6ac860ceece6a3cd45ef26860cd78737dad.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE project_flag_runs AS runs\n SET status = 'done', completed_at = CURRENT_TIMESTAMP\n WHERE runs.id = ANY($1)\n AND runs.status = 'running'\n AND NOT EXISTS (\n SELECT 1\n FROM project_flag_run_pending_jobs AS pending\n WHERE pending.run_id = runs.id\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "e8b537da45864ca001d74485a118b6ac860ceece6a3cd45ef26860cd78737dad" +} diff --git a/apps/labrinth/.sqlx/query-fe94b8e26665ba77421fdb4d436d9a40ea223bf3407fb1f84b5263198f67c32b.json b/apps/labrinth/.sqlx/query-fe94b8e26665ba77421fdb4d436d9a40ea223bf3407fb1f84b5263198f67c32b.json new file mode 100644 index 00000000000..5840bd9b02a --- /dev/null +++ b/apps/labrinth/.sqlx/query-fe94b8e26665ba77421fdb4d436d9a40ea223bf3407fb1f84b5263198f67c32b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO project_flag_runs (id, status, started_by, started_at, total_projects)\n VALUES ($1, 'running', $2, $3, $4)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int8", + "Timestamptz", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "fe94b8e26665ba77421fdb4d436d9a40ea223bf3407fb1f84b5263198f67c32b" +} diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index c87688202f1..0f0ab6158bf 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -111,6 +111,7 @@ rdkafka = { workspace = true } redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "r2d2", "tokio-comp"] } regex = { workspace = true } reqwest = { workspace = true, features = ["http2", "json", "multipart", "rustls"] } +rhai = { workspace = true, features = ["serde", "sync"] } rust_decimal = { workspace = true, features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = { workspace = true } rustls = { workspace = true, features = [ @@ -158,6 +159,7 @@ unicode-segmentation = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } utoipa = { workspace = true, features = ["url"] } +utoipa-actix-web = { workspace = true } uuid = { workspace = true, features = ["fast-rng", "serde", "v4", "v7"] } validator = { workspace = true, features = ["derive"] } webauthn-rs = { workspace = true, features = [ diff --git a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql new file mode 100644 index 00000000000..3ffb9e6cd27 --- /dev/null +++ b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql @@ -0,0 +1,53 @@ +CREATE TABLE project_flag_rules ( + id BIGINT PRIMARY KEY, + name VARCHAR(1023) NOT NULL, + created_by BIGINT REFERENCES users(id) ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_by BIGINT REFERENCES users(id) ON DELETE SET NULL, + updated_at TIMESTAMPTZ NOT NULL, + params JSONB NOT NULL +); + +CREATE TABLE project_flag_runs ( + id UUID PRIMARY KEY, + status TEXT NOT NULL, + total_projects BIGINT NOT NULL, + started_by BIGINT REFERENCES users(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL, + completed_at TIMESTAMPTZ, + applied_at TIMESTAMPTZ, + output JSONB, + error TEXT +); + +CREATE TABLE project_flags ( + id BIGINT PRIMARY KEY, + run_id UUID REFERENCES project_flag_runs(id) ON DELETE CASCADE, + project_id BIGINT NOT NULL REFERENCES mods(id) ON DELETE CASCADE, + what JSONB NOT NULL, + active BOOLEAN NOT NULL DEFAULT FALSE +); + +CREATE INDEX project_flags_run_id ON project_flags (run_id); + +CREATE INDEX project_flags_project_id ON project_flags (project_id); + +CREATE UNIQUE INDEX project_flags_identity + ON project_flags (project_id, (what->>'key')); + +CREATE TABLE project_flag_run_results ( + id UUID PRIMARY KEY, + run_id UUID REFERENCES project_flag_runs(id) ON DELETE CASCADE, + project_id BIGINT NOT NULL REFERENCES mods(id) ON DELETE CASCADE, + rules JSONB NOT NULL DEFAULT '{}'::jsonb, + error TEXT, + evaluated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (run_id, project_id) +); + +CREATE INDEX project_flag_run_results_project_id + ON project_flag_run_results (project_id); + +CREATE INDEX project_flag_run_results_errors + ON project_flag_run_results (run_id) + WHERE error IS NOT NULL; diff --git a/apps/labrinth/migrations/20261107120000_project_flag_run_applied_by.sql b/apps/labrinth/migrations/20261107120000_project_flag_run_applied_by.sql new file mode 100644 index 00000000000..b18a6a7fbef --- /dev/null +++ b/apps/labrinth/migrations/20261107120000_project_flag_run_applied_by.sql @@ -0,0 +1,2 @@ +ALTER TABLE project_flag_runs + ADD COLUMN applied_by BIGINT REFERENCES users(id) ON DELETE SET NULL; diff --git a/apps/labrinth/migrations/20261110120000_project_flag_pending_jobs.sql b/apps/labrinth/migrations/20261110120000_project_flag_pending_jobs.sql new file mode 100644 index 00000000000..a5884a2e2ba --- /dev/null +++ b/apps/labrinth/migrations/20261110120000_project_flag_pending_jobs.sql @@ -0,0 +1,5 @@ +CREATE TABLE project_flag_run_pending_jobs ( + run_id UUID NOT NULL REFERENCES project_flag_runs(id) ON DELETE CASCADE, + project_id BIGINT NOT NULL, + PRIMARY KEY (run_id, project_id) +); diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs index 05d9ae1c23d..5852e360fef 100644 --- a/apps/labrinth/src/background_task.rs +++ b/apps/labrinth/src/background_task.rs @@ -24,6 +24,8 @@ pub enum BackgroundTask { IndexBilling, IndexSubscriptions, IncrementalIndexSearch, + /// Consumes project flag run jobs from Kafka. + ProjectFlagRuns, Migrations, Mail, /// Queries server project analytics (e.g. number of verified plays in last @@ -87,6 +89,11 @@ impl BackgroundTask { ) .await } + ProjectFlagRuns => { + crate::queue::moderation::project_flag_run::consume_project_flag_runs( + pool, ro_pool.into(), redis_pool, file_host.into_inner(), + ).await + } Mail => run_email(email_queue).await, CacheAnalytics => { cache_analytics(&pool, &redis_pool, &clickhouse).await diff --git a/apps/labrinth/src/database/models/ids.rs b/apps/labrinth/src/database/models/ids.rs index e3e573b24ab..ae050ac2f82 100644 --- a/apps/labrinth/src/database/models/ids.rs +++ b/apps/labrinth/src/database/models/ids.rs @@ -306,3 +306,6 @@ id_type!(DelphiReportIssueId as i64); id_type!(DelphiReportIssueDetailsId as i64); id_type!(DelphiRuleId as i64); id_type!(SharedInstanceId as i64); +id_type!(ProjectFlagRuleId as i64); +id_type!(ProjectFlagId as i64); +id_type!(ProjectFlagRunId as uuid::Uuid); diff --git a/apps/labrinth/src/database/models/mod.rs b/apps/labrinth/src/database/models/mod.rs index 66c6d8104f8..750aa073be6 100644 --- a/apps/labrinth/src/database/models/mod.rs +++ b/apps/labrinth/src/database/models/mod.rs @@ -29,6 +29,7 @@ pub mod payouts_values_notifications; pub mod product_item; pub mod products_tax_identifier_item; pub mod project_disclosure_item; +pub mod project_flag_item; pub mod project_item; pub mod report_item; pub mod session_item; @@ -68,3 +69,7 @@ pub use version_item::DBVersion; pub use moderation_lock_item::{DBModerationLock, ModerationLockWithUser}; pub use moderation_note_item::DBModerationNote; +pub use project_flag_item::{ + DBProjectFlag, DBProjectFlagRule, DBProjectFlagRun, ProjectFlagRunOutput, + ProjectFlagRunStatus, +}; diff --git a/apps/labrinth/src/database/models/project_disclosure_item.rs b/apps/labrinth/src/database/models/project_disclosure_item.rs index d8df955312b..3f935b49a10 100644 --- a/apps/labrinth/src/database/models/project_disclosure_item.rs +++ b/apps/labrinth/src/database/models/project_disclosure_item.rs @@ -64,14 +64,23 @@ impl DBProjectDisclosure { include_deleted: bool, exec: impl crate::database::Executor<'_, Database = sqlx::Postgres>, ) -> Result> { + Self::get_many_for_projects(&[project_id], include_deleted, exec).await + } + + pub async fn get_many_for_projects( + project_ids: &[DBProjectId], + include_deleted: bool, + exec: impl crate::database::Executor<'_, Database = sqlx::Postgres>, + ) -> Result> { + let ids = project_ids.iter().map(|id| id.0).collect::>(); let rows = sqlx::query!( r#" SELECT project_id, type AS "disclosure_type!", metadata, updated_at, updated_by, set_by_moderator, deleted_at, lock_status FROM project_disclosures - WHERE project_id = $1 AND ($2 OR deleted_at IS NULL) + WHERE project_id = ANY($1) AND ($2 OR deleted_at IS NULL) ORDER BY updated_at DESC "#, - project_id as DBProjectId, + &ids, include_deleted, ) .fetch_all(exec) diff --git a/apps/labrinth/src/database/models/project_flag_item.rs b/apps/labrinth/src/database/models/project_flag_item.rs new file mode 100644 index 00000000000..87ebc61df6a --- /dev/null +++ b/apps/labrinth/src/database/models/project_flag_item.rs @@ -0,0 +1,849 @@ +use std::collections::BTreeMap; + +use ariadne::ids::random_base62; +use chrono::{DateTime, Utc}; +use eyre::{Result, WrapErr}; + +use crate::database::{PgPool, ReadOnlyPgPool}; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, serde_as}; +use utoipa::ToSchema; +use validator::Validate; + +use super::{ + DBProjectId, DBUserId, ProjectFlagId, ProjectFlagRuleId, ProjectFlagRunId, +}; + +#[derive( + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + ToSchema, +)] +#[serde(transparent)] +#[schema(value_type = String, example = "suspicious_url")] +pub struct PatternName(pub String); + +#[serde_as] +#[derive(Clone, Debug, Serialize, Deserialize, Validate, ToSchema)] +pub struct ProjectRuleParams { + #[validate(length(min = 1, max = 65535))] + pub expression: String, + #[serde(default)] + #[serde_as(as = "BTreeMap<_, DisplayFromStr>")] + #[schema(value_type = BTreeMap)] + pub scan_files_for: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct DBProjectFlagRule { + pub id: ProjectFlagRuleId, + pub name: String, + pub created_by: Option, + pub created_at: DateTime, + pub updated_by: Option, + pub updated_at: DateTime, + pub params: ProjectRuleParams, +} + +impl DBProjectFlagRule { + pub async fn all(pool: &PgPool) -> Result> { + let rows = sqlx::query!( + r#" + SELECT id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at + FROM project_flag_rules + ORDER BY id + "#, + ) + .fetch_all(pool) + .await + .wrap_err("snapshotting project flag rules")?; + + Ok(rows + .into_iter() + .map(|row| Self { + id: ProjectFlagRuleId(row.id), + name: row.name, + params: row.params.0, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + }) + .collect()) + } + + pub async fn list( + limit: i64, + offset: i64, + pool: &ReadOnlyPgPool, + ) -> Result> { + let rows = sqlx::query!( + r#" + SELECT id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at + FROM project_flag_rules + ORDER BY created_at DESC, id DESC + LIMIT $1 OFFSET $2 + "#, + limit, + offset, + ) + .fetch_all(&**pool) + .await + .wrap_err("listing project flag rules")?; + + Ok(rows + .into_iter() + .map(|row| Self { + id: ProjectFlagRuleId(row.id), + name: row.name, + params: row.params.0, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + }) + .collect()) + } + + pub async fn get( + id: ProjectFlagRuleId, + pool: &ReadOnlyPgPool, + ) -> Result> { + let row = sqlx::query!( + r#" + SELECT id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at + FROM project_flag_rules + WHERE id = $1 + "#, + id.0, + ) + .fetch_optional(&**pool) + .await + .wrap_err("fetching project flag rule")?; + + Ok(row.map(|row| Self { + id: ProjectFlagRuleId(row.id), + name: row.name, + params: row.params.0, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + })) + } + + pub async fn create( + name: &str, + params: &ProjectRuleParams, + user_id: DBUserId, + pool: &PgPool, + ) -> Result { + let id = random_base62(8) as i64; + let now = Utc::now(); + let row = sqlx::query!( + r#" + INSERT INTO project_flag_rules + (id, name, params, created_by, created_at, updated_by, updated_at) + VALUES ($1, $2, $3, $4, $5, $4, $5) + RETURNING id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at + "#, + id, + name, + sqlx::types::Json(params) as _, + user_id.0, + now, + ) + .fetch_one(pool) + .await + .wrap_err("creating project flag rule")?; + + Ok(Self { + id: ProjectFlagRuleId(row.id), + name: row.name, + params: row.params.0, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + }) + } + + pub async fn update( + id: ProjectFlagRuleId, + name: &str, + params: &ProjectRuleParams, + user_id: DBUserId, + pool: &PgPool, + ) -> Result> { + let now = Utc::now(); + let row = sqlx::query!( + r#" + UPDATE project_flag_rules + SET name = $2, params = $3, updated_by = $4, updated_at = $5 + WHERE id = $1 + RETURNING id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at + "#, + id.0, + name, + sqlx::types::Json(params) as _, + user_id.0, + now, + ) + .fetch_optional(pool) + .await + .wrap_err("updating project flag rule")?; + + Ok(row.map(|row| Self { + id: ProjectFlagRuleId(row.id), + name: row.name, + params: row.params.0, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + })) + } + + pub async fn delete(id: ProjectFlagRuleId, pool: &PgPool) -> Result { + let result = sqlx::query!( + r#" + DELETE FROM project_flag_rules + WHERE id = $1 + "#, + id.0, + ) + .execute(pool) + .await + .wrap_err("deleting project flag rule")?; + + Ok(result.rows_affected() > 0) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProjectFlag { + pub key: String, + #[serde(default)] + pub retain_if_inactive: bool, + #[serde(flatten)] + pub value: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DBProjectFlag { + pub id: ProjectFlagId, + pub run_id: Option, + pub project_id: DBProjectId, + pub what: ProjectFlag, + pub active: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProjectFlagRuleResult { + pub flags: Vec, + pub log: Vec, + pub error: Option, +} + +type RuleResults = BTreeMap; + +#[derive(Debug, Serialize)] +pub struct DBProjectFlagRunResult { + pub run_id: Option, + pub project_id: DBProjectId, + pub rules: BTreeMap, + pub error: Option, +} + +impl DBProjectFlag { + pub async fn get_project_flags( + project_id: DBProjectId, + pool: &ReadOnlyPgPool, + ) -> Result>> { + let rows = sqlx::query!( + r#" + SELECT flags.what AS "what?: sqlx::types::Json" + FROM mods AS projects + LEFT JOIN project_flags AS flags ON flags.project_id = projects.id + AND (flags.active OR (flags.what->>'retain_if_inactive')::bool = TRUE) + WHERE projects.id = $1 + ORDER BY flags.what->>'key', flags.id + "#, + project_id.0, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching project flags")?; + + if rows.is_empty() { + return Ok(None); + } + Ok(Some( + rows.into_iter() + .filter_map(|row| row.what.map(|what| what.0)) + .collect(), + )) + } +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ProjectFlagRunResultProject { + pub id: crate::models::ids::ProjectId, + pub title: String, + pub slug: Option, + pub icon_url: Option, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ProjectFlagRunRuleResult { + pub name: Option, + #[serde(flatten)] + pub result: ProjectFlagRuleResult, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ProjectFlagRunResult { + pub project: ProjectFlagRunResultProject, + pub rules: BTreeMap, + pub error: Option, + pub evaluated_at: DateTime, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ProjectFlagRunResults { + pub results: Vec, + pub total: i64, +} + +impl DBProjectFlagRunResult { + pub async fn list( + run_id: ProjectFlagRunId, + limit: i64, + offset: i64, + pool: &ReadOnlyPgPool, + ) -> Result> { + let run = sqlx::query!( + r#" + SELECT ( + SELECT COUNT(*) + FROM project_flag_run_results + WHERE run_id = runs.id + ) AS "total!" + FROM project_flag_runs AS runs + WHERE runs.id = $1 + "#, + run_id.0, + ) + .fetch_optional(&**pool) + .await + .wrap_err("counting project flag run results")?; + let Some(run) = run else { + return Ok(None); + }; + let rows = sqlx::query!( + r#" + SELECT projects.id, projects.name AS title, projects.slug, projects.icon_url, + results.rules AS "rules: sqlx::types::Json", + results.error, results.evaluated_at + FROM project_flag_run_results AS results + INNER JOIN mods AS projects ON projects.id = results.project_id + WHERE results.run_id = $1 + ORDER BY results.project_id + LIMIT $2 OFFSET $3 + "#, + run_id.0, + limit, + offset, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching project flag run results")?; + let mut rule_ids = rows + .iter() + .flat_map(|row| row.rules.0.keys()) + .map(|id| id.parse::()) + .collect::, _>>() + .wrap_err("parsing project flag result rule IDs")?; + rule_ids.sort_unstable(); + rule_ids.dedup(); + let names = sqlx::query!( + r#"SELECT id, name FROM project_flag_rules WHERE id = ANY($1)"#, + &rule_ids, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching project flag rule names")? + .into_iter() + .map(|row| (row.id.to_string(), row.name)) + .collect::>(); + Ok(Some(ProjectFlagRunResults { + results: rows + .into_iter() + .map(|row| ProjectFlagRunResult { + project: ProjectFlagRunResultProject { + id: DBProjectId(row.id).into(), + title: row.title, + slug: row.slug, + icon_url: row.icon_url, + }, + rules: row + .rules + .0 + .into_iter() + .map(|(id, result)| { + let name = names.get(&id).cloned(); + (id, ProjectFlagRunRuleResult { name, result }) + }) + .collect(), + error: row.error, + evaluated_at: row.evaluated_at, + }) + .collect(), + total: run.total, + })) + } + + pub async fn upsert_many( + results: Vec, + consumed_jobs: &[(ProjectFlagRunId, DBProjectId)], + pool: &PgPool, + ) -> Result<()> { + if results.is_empty() && consumed_jobs.is_empty() { + return Ok(()); + } + let mut run_ids = results + .iter() + .filter_map(|result| result.run_id.map(|id| id.0)) + .chain(consumed_jobs.iter().map(|(run_id, _)| run_id.0)) + .collect::>(); + run_ids.sort_unstable(); + run_ids.dedup(); + #[derive(Serialize)] + struct ResultPayload { + id: uuid::Uuid, + #[serde(flatten)] + result: DBProjectFlagRunResult, + } + + let mut run_results = std::collections::HashMap::new(); + let mut standalone_results = Vec::new(); + for mut result in results { + if result.error.is_none() { + let errors = result + .rules + .iter() + .filter_map(|(rule_id, rule)| { + rule.error + .as_ref() + .map(|error| format!("{rule_id}: {error}")) + }) + .collect::>(); + if !errors.is_empty() { + result.error = Some(errors.join("\n")); + } + } + if let Some(run_id) = result.run_id { + run_results.insert((run_id, result.project_id), result); + } else { + standalone_results.push(result); + } + } + let results = run_results + .into_values() + .chain(standalone_results) + .map(|result| ResultPayload { + id: uuid::Uuid::now_v7(), + result, + }) + .collect::>(); + let mut project_ids = results + .iter() + .map(|payload| payload.result.project_id.0) + .collect::>(); + project_ids.sort_unstable(); + project_ids.dedup(); + let mut transaction = pool + .begin() + .await + .wrap_err("starting project flag results transaction")?; + sqlx::query!( + r#" + SELECT id + FROM project_flag_runs + WHERE id = ANY($1) + ORDER BY id + FOR UPDATE + "#, + &run_ids, + ) + .fetch_all(&mut transaction) + .await + .wrap_err("locking project flag runs")?; + let surviving_projects = sqlx::query_scalar!( + r#" + SELECT id FROM mods + WHERE id = ANY($1) + ORDER BY id + FOR KEY SHARE + "#, + &project_ids, + ) + .fetch_all(&mut transaction) + .await + .wrap_err("locking projects for project flag results")?; + let results = results + .into_iter() + .filter(|payload| { + surviving_projects + .binary_search(&payload.result.project_id.0) + .is_ok() + }) + .collect::>(); + let results = serde_json::to_value(results) + .wrap_err("serializing project flag run results")?; + sqlx::query!( + r#" + INSERT INTO project_flag_run_results + (id, run_id, project_id, rules, error) + SELECT results.id, results.run_id, results.project_id, results.rules, results.error + FROM jsonb_to_recordset($1) AS results( + id UUID, run_id UUID, project_id BIGINT, rules JSONB, error TEXT + ) + LEFT JOIN project_flag_runs AS runs ON runs.id = results.run_id + WHERE results.run_id IS NULL OR runs.status = 'running' + ORDER BY results.run_id, results.project_id + ON CONFLICT (run_id, project_id) DO UPDATE + SET rules = EXCLUDED.rules, + error = EXCLUDED.error, evaluated_at = CURRENT_TIMESTAMP + "#, + results, + ) + .execute(&mut transaction) + .await + .wrap_err("persisting project flag run results")?; + let consumed_run_ids = consumed_jobs + .iter() + .map(|(run_id, _)| run_id.0) + .collect::>(); + let consumed_project_ids = consumed_jobs + .iter() + .map(|(_, project_id)| project_id.0) + .collect::>(); + sqlx::query!( + r#" + DELETE FROM project_flag_run_pending_jobs AS pending + USING UNNEST($1::UUID[], $2::BIGINT[]) AS consumed(run_id, project_id) + WHERE pending.run_id = consumed.run_id + AND pending.project_id = consumed.project_id + "#, + &consumed_run_ids, + &consumed_project_ids, + ) + .execute(&mut transaction) + .await + .wrap_err("acknowledging consumed project flag jobs")?; + sqlx::query!( + r#" + UPDATE project_flag_runs AS runs + SET status = 'done', completed_at = CURRENT_TIMESTAMP + WHERE runs.id = ANY($1) + AND runs.status = 'running' + AND NOT EXISTS ( + SELECT 1 + FROM project_flag_run_pending_jobs AS pending + WHERE pending.run_id = runs.id + ) + "#, + &run_ids, + ) + .execute(&mut transaction) + .await + .wrap_err("completing project flag runs")?; + transaction + .commit() + .await + .wrap_err("committing project flag run results")?; + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ProjectFlagRunUser { + pub id: ariadne::ids::UserId, + pub username: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct DBProjectFlagRun { + pub id: ProjectFlagRunId, + pub status: ProjectFlagRunStatus, + pub total_projects: i64, + pub num_errors: i64, + pub started_by: Option, + pub started_at: DateTime, + pub completed_at: Option>, + pub applied_at: Option>, + pub applied_by: Option, + pub output: Option, + pub error: Option, +} + +impl DBProjectFlagRun { + pub async fn num_scanning(pool: &ReadOnlyPgPool) -> Result { + sqlx::query_scalar!( + r#" + SELECT COUNT(*) AS "num_scanning!" + FROM project_flag_run_pending_jobs AS pending + JOIN project_flag_runs AS runs ON runs.id = pending.run_id + WHERE runs.status = 'running' + "#, + ) + .fetch_one(&**pool) + .await + .wrap_err("counting unfinished project flag scans") + } + + pub async fn delete(id: ProjectFlagRunId, pool: &PgPool) -> Result { + let mut transaction = pool + .begin() + .await + .wrap_err("starting project flag run deletion")?; + let run = sqlx::query_scalar!( + r#"SELECT id FROM project_flag_runs WHERE id = $1 FOR UPDATE"#, + id.0, + ) + .fetch_optional(&mut transaction) + .await + .wrap_err("locking project flag run for deletion")?; + if run.is_none() { + return Ok(false); + } + sqlx::query!( + r#" + SELECT id FROM mods + WHERE id IN (SELECT project_id FROM project_flags WHERE run_id = $1) + ORDER BY id + FOR UPDATE + "#, + id.0, + ) + .fetch_all(&mut transaction) + .await + .wrap_err("locking projects for run deletion")?; + sqlx::query!( + r#"UPDATE project_flags SET run_id = NULL WHERE run_id = $1 AND active"#, + id.0, + ).execute(&mut transaction).await.wrap_err("detaching active project flags")?; + let result = sqlx::query!( + r#" + DELETE FROM project_flag_runs + WHERE id = $1 + "#, + id.0, + ) + .execute(&mut transaction) + .await + .wrap_err("deleting project flag run")?; + transaction + .commit() + .await + .wrap_err("committing project flag run deletion")?; + Ok(result.rows_affected() > 0) + } + + pub async fn queued_project_ids(pool: &PgPool) -> Result> { + let ids = sqlx::query_scalar!( + r#"SELECT id FROM mods WHERE status = 'processing'"#, + ) + .fetch_all(pool) + .await + .wrap_err("fetching queued project IDs")?; + Ok(ids.into_iter().map(DBProjectId).collect()) + } + + pub async fn create( + user_id: DBUserId, + project_ids: &[DBProjectId], + pool: &PgPool, + ) -> Result { + let mut project_ids = + project_ids.iter().map(|id| id.0).collect::>(); + project_ids.sort_unstable(); + project_ids.dedup(); + let total_projects = i64::try_from(project_ids.len()) + .wrap_err("counting project flag run jobs")?; + let username = sqlx::query_scalar!( + r#"SELECT username FROM users WHERE id = $1"#, + user_id.0, + ) + .fetch_one(pool) + .await + .wrap_err("fetching project flag run starter")?; + let run = Self { + id: ProjectFlagRunId(uuid::Uuid::now_v7()), + status: ProjectFlagRunStatus::Running, + total_projects, + num_errors: 0, + started_by: Some(ProjectFlagRunUser { + id: user_id.into(), + username, + }), + started_at: Utc::now(), + completed_at: None, + applied_at: None, + applied_by: None, + output: None, + error: None, + }; + let mut transaction = pool + .begin() + .await + .wrap_err("starting project flag run creation")?; + sqlx::query!( + r#" + INSERT INTO project_flag_runs (id, status, started_by, started_at, total_projects) + VALUES ($1, 'running', $2, $3, $4) + "#, + run.id.0, + user_id.0, + run.started_at, + total_projects, + ) + .execute(&mut transaction) + .await + .wrap_err("creating project flag run")?; + sqlx::query!( + r#" + INSERT INTO project_flag_run_pending_jobs (run_id, project_id) + SELECT $1, project_id FROM UNNEST($2::BIGINT[]) AS jobs(project_id) + "#, + run.id.0, + &project_ids, + ) + .execute(&mut transaction) + .await + .wrap_err("creating pending project flag jobs")?; + transaction + .commit() + .await + .wrap_err("committing project flag run creation")?; + Ok(run) + } + + pub async fn finish( + &mut self, + error: Option, + pool: &PgPool, + ) -> Result<()> { + let status = ProjectFlagRunStatus::Done; + let completed_at = Utc::now(); + sqlx::query!( + r#" + UPDATE project_flag_runs + SET status = $2, completed_at = $3, error = $4 + WHERE id = $1 + "#, + self.id.0, + status as ProjectFlagRunStatus, + completed_at, + error, + ) + .execute(pool) + .await + .wrap_err("finishing project flag run")?; + self.status = status; + self.completed_at = Some(completed_at); + self.error = error; + Ok(()) + } + + pub async fn list( + limit: i64, + offset: i64, + pool: &ReadOnlyPgPool, + ) -> Result> { + let rows = sqlx::query!( + r#" + SELECT runs.id, runs.status AS "status: ProjectFlagRunStatus", runs.total_projects, + starter.id AS "started_by_id?", starter.username AS "started_by_username?", + applier.id AS "applied_by_id?", applier.username AS "applied_by_username?", + runs.started_at, runs.completed_at, runs.applied_at, + runs.output AS "output: sqlx::types::Json", runs.error, + (SELECT COUNT(*) FROM project_flag_run_results AS results + WHERE results.run_id = runs.id AND results.error IS NOT NULL + ) AS "num_errors!" + FROM project_flag_runs AS runs + LEFT JOIN users AS starter ON starter.id = runs.started_by + LEFT JOIN users AS applier ON applier.id = runs.applied_by + ORDER BY runs.started_at DESC, runs.id DESC + LIMIT $1 OFFSET $2 + "#, + limit, + offset, + ) + .fetch_all(&**pool) + .await + .wrap_err("listing project flag runs")?; + + Ok(rows + .into_iter() + .map(|row| Self { + id: ProjectFlagRunId(row.id), + status: row.status, + total_projects: row.total_projects, + num_errors: row.num_errors, + started_by: row.started_by_id.zip(row.started_by_username).map( + |(id, username)| ProjectFlagRunUser { + id: DBUserId(id).into(), + username, + }, + ), + started_at: row.started_at, + completed_at: row.completed_at, + applied_at: row.applied_at, + applied_by: row.applied_by_id.zip(row.applied_by_username).map( + |(id, username)| ProjectFlagRunUser { + id: DBUserId(id).into(), + username, + }, + ), + output: row.output.map(|output| output.0), + error: row.error, + }) + .collect()) + } +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + Serialize, + Deserialize, + sqlx::Type, + utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +#[sqlx(type_name = "text", rename_all = "snake_case")] +pub enum ProjectFlagRunStatus { + Running, + Done, +} + +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProjectFlagRunOutput {} diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index e3daaedbc9b..f9ad395ad0d 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -1,4 +1,7 @@ -use std::{any::type_name, convert::Infallible, str::FromStr, sync::LazyLock}; +use std::{ + any::type_name, convert::Infallible, num::NonZeroUsize, str::FromStr, + sync::LazyLock, +}; use derive_more::{Deref, DerefMut}; use eyre::{Context, eyre}; @@ -13,6 +16,7 @@ use xredis::{ macro_rules! vars { ( $( + $(#[$meta:meta])* $field:ident: $ty:ty $(= $default:expr)?; )* ) => { @@ -23,6 +27,7 @@ macro_rules! vars { )] pub struct EnvVars { $( + $(#[$meta])* pub $field: $ty, )* } @@ -336,6 +341,24 @@ vars! { PUBLIC_DISCORD_WEBHOOK: String = ""; MODERATION_SLACK_WEBHOOK: String = ""; + /// Maximum execution time for each project flag rule evaluation, in milliseconds. + PROJECT_FLAG_RULE_TIMEOUT: u64 = 5000u64; + /// Maximum retained printed log bytes per rule evaluation, excluding the omission summary. + PROJECT_FLAG_RULE_MAX_LOG_BYTES: usize = 128usize * 1024; + /// Maximum number of project flag jobs collected per consumer batch. + PROJECT_FLAG_SCAN_BATCH_SIZE: NonZeroUsize = NonZeroUsize::new(100).unwrap(); + /// Maximum number of concurrent project flag artifact downloads per process. + PROJECT_FLAG_SCAN_DOWNLOAD_CONCURRENCY: NonZeroUsize = NonZeroUsize::new(8).unwrap(); + /// Maximum buffered artifact size in bytes. + PROJECT_FLAG_SCAN_MAX_ARTIFACT_BYTES: u64 = 512u64 * 1024 * 1024; + /// Maximum scanned text file size in bytes. + PROJECT_FLAG_SCAN_MAX_TEXT_BYTES: u64 = 8u64 * 1024 * 1024; + /// Maximum cumulative downloaded and decompressed bytes per project scan. + PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES: u64 = 2u64 * 1024 * 1024 * 1024; + /// Maximum archive entries per project scan, including directories. + PROJECT_FLAG_SCAN_MAX_ENTRIES: usize = 50_000usize; + /// Maximum archive nesting layers, including the uploaded archive. + PROJECT_FLAG_SCAN_MAX_DEPTH: usize = 4usize; DELPHI_SLACK_WEBHOOK: String = ""; TREMENDOUS_CAMPAIGN_ID: String = "none"; diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index c3e40cbcd77..37d42ca5999 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -19,6 +19,7 @@ use util::gotenberg::GotenbergClient; use crate::database::{PgPool, ReadOnlyPgPool}; use crate::env::ENV; use crate::queue::billing::{index_billing, index_subscriptions}; +use crate::queue::moderation::project_flag::ProjectFlagEngine; use crate::routes::internal::delphi::rescan::enqueue_tech_review_files_for_new_delphi_version; use crate::util::anrok; use crate::util::archon::ArchonClient; @@ -64,6 +65,7 @@ pub struct LabrinthConfig { pub ip_salt: Pepper, pub search_state: web::Data, pub session_queue: web::Data, + pub project_flag_engine: web::Data, pub payouts_queue: web::Data, pub active_sockets: web::Data, pub rate_limiter: web::Data, @@ -241,6 +243,15 @@ pub fn app_setup( .expect("failed to build webauthn"), ); + let project_flag_engine = web::Data::new( + ProjectFlagEngine::new( + ro_pool.clone(), + redis_pool.clone(), + file_host.clone().into_inner(), + ) + .expect("failed to create project flag engine"), + ); + LabrinthConfig { pool, ro_pool, @@ -252,6 +263,7 @@ pub fn app_setup( ip_salt, search_state, session_queue, + project_flag_engine, payouts_queue: web::Data::new(PayoutsQueue::new()), active_sockets, rate_limiter: limiter, @@ -306,6 +318,7 @@ pub fn app_data_config( .app_data(labrinth_config.http_client.clone()) .app_data(labrinth_config.tiltify_client.clone()) .app_data(labrinth_config.session_queue.clone()) + .app_data(labrinth_config.project_flag_engine.clone()) .app_data(labrinth_config.payouts_queue.clone()) .app_data(labrinth_config.email_queue.clone()) .app_data(web::Data::new(labrinth_config.ip_salt.clone())) diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index 44ab19eccbb..b34f4ab782f 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -138,6 +138,7 @@ async fn app() -> std::io::Result<()> { let kafka_client = actix_web::web::Data::new( labrinth::util::kafka::KafkaClientState::new() + .await .expect("Kafka connection failed"), ); diff --git a/apps/labrinth/src/models/v3/organizations.rs b/apps/labrinth/src/models/v3/organizations.rs index 09032cc44c4..27021cc80f6 100644 --- a/apps/labrinth/src/models/v3/organizations.rs +++ b/apps/labrinth/src/models/v3/organizations.rs @@ -4,7 +4,7 @@ use crate::models::ids::{OrganizationId, TeamId}; use serde::{Deserialize, Serialize}; /// An organization of users who control a project -#[derive(Serialize, Deserialize, utoipa::ToSchema)] +#[derive(Clone, Serialize, Deserialize, utoipa::ToSchema)] pub struct Organization { /// The id of the organization pub id: OrganizationId, diff --git a/apps/labrinth/src/models/v3/thread_issues.rs b/apps/labrinth/src/models/v3/thread_issues.rs index e44ee51414a..1c46528c2cf 100644 --- a/apps/labrinth/src/models/v3/thread_issues.rs +++ b/apps/labrinth/src/models/v3/thread_issues.rs @@ -101,6 +101,7 @@ pub enum ThreadIssueTarget { AddGalleryImages { #[serde(default)] original_image_ids: Vec, + #[serde(default)] auto_resolve: bool, }, ModifyGalleryImage { diff --git a/apps/labrinth/src/queue/moderation.rs b/apps/labrinth/src/queue/moderation/mod.rs similarity index 97% rename from apps/labrinth/src/queue/moderation.rs rename to apps/labrinth/src/queue/moderation/mod.rs index f43dc8b2bfa..d7dfa32a327 100644 --- a/apps/labrinth/src/queue/moderation.rs +++ b/apps/labrinth/src/queue/moderation/mod.rs @@ -1,3 +1,8 @@ +pub mod project_flag; +mod project_flag_attribution; +mod project_flag_files; +pub mod project_flag_run; + use serde::{Deserialize, Serialize}; use std::collections::HashMap; diff --git a/apps/labrinth/src/queue/moderation/project_flag.rs b/apps/labrinth/src/queue/moderation/project_flag.rs new file mode 100644 index 00000000000..96684baa2d1 --- /dev/null +++ b/apps/labrinth/src/queue/moderation/project_flag.rs @@ -0,0 +1,572 @@ +use std::{ + cell::RefCell, + collections::{BTreeMap, HashMap}, + sync::{Arc, LazyLock}, + time::Duration, +}; + +use eyre::{Result, eyre}; +use regex::Regex; +use xredis::RedisPool; + +pub use super::project_flag_attribution::FlagAttributionGroup; +use super::project_flag_attribution::load_attributions; +use super::project_flag_files::{ + ProjectScanFile, load_project_files, scan_project_files, +}; + +use crate::{ + database::{ + ReadOnlyPgPool, + models::{ + DBOrganization, DBProject, DBProjectDisclosure, DBProjectId, + DBTeamMember, DBThread, DBUser, DBVersion, DBVersionId, + project_flag_item::{PatternName, ProjectFlag, ProjectRuleParams}, + }, + }, + env::ENV, + file_hosting::FileHost, + models::{ + ids::{FileId, TeamId, ThreadId, VersionId}, + organizations::Organization, + projects::{Project, Version}, + teams::TeamMember, + threads::ThreadMessage, + users::User, + v3::disclosures::ProjectDisclosureData, + }, + util::error::Context, +}; + +static DOWNLOADS: LazyLock> = LazyLock::new(|| { + Arc::new(tokio::sync::Semaphore::new( + ENV.PROJECT_FLAG_SCAN_DOWNLOAD_CONCURRENCY.get(), + )) +}); + +thread_local! { + static EVALUATION_LOG: RefCell> = const { RefCell::new(None) }; +} + +/// Collects string messages into a log, up to a certain `max_bytes` capacity, +/// then will output `(omitted N bytes)` instead. +#[derive(Default)] +struct CappedLog { + messages: Vec, + bytes: usize, + omitted_bytes: usize, +} + +impl CappedLog { + fn push(&mut self, message: &str, max_bytes: usize) { + if self.omitted_bytes > 0 { + self.omitted_bytes = + self.omitted_bytes.saturating_add(message.len()); + return; + } + let mut retained = + message.len().min(max_bytes.saturating_sub(self.bytes)); + while !message.is_char_boundary(retained) { + retained -= 1; + } + if retained > 0 { + self.messages.push(message[..retained].to_owned()); + self.bytes += retained; + } + self.omitted_bytes = message.len() - retained; + } + + fn finish(mut self) -> Vec { + if self.omitted_bytes > 0 { + self.messages + .push(format!("(omitted {} bytes)", self.omitted_bytes)); + } + self.messages + } +} + +#[cfg(test)] +mod evaluation_log_tests { + use super::CappedLog; + + #[test] + fn retains_messages_within_budget() { + let mut log = CappedLog::default(); + log.push("hello", 10); + log.push("world", 10); + assert_eq!(log.finish(), vec!["hello", "world"]); + } + + #[test] + fn counts_all_bytes_after_truncation() { + let mut log = CappedLog::default(); + log.push("hello", 7); + log.push("world", 7); + log.push("é", 7); + assert_eq!(log.finish(), vec!["hello", "wo", "(omitted 5 bytes)"]); + } + + #[test] + fn truncates_at_utf8_boundary_and_does_not_resume() { + let mut log = CappedLog::default(); + log.push("aé", 2); + log.push("b", 2); + assert_eq!(log.finish(), vec!["a", "(omitted 3 bytes)"]); + } + + #[test] + fn zero_budget_only_retains_summary() { + let mut log = CappedLog::default(); + log.push("hello", 0); + log.push("world", 0); + assert_eq!(log.finish(), vec!["(omitted 10 bytes)"]); + } + + #[test] + fn empty_messages_do_not_allocate_entries() { + let mut log = CappedLog::default(); + log.push("", 10); + assert!(log.finish().is_empty()); + } +} + +struct EvaluationLog(Option); + +impl EvaluationLog { + fn start() -> Self { + Self(EVALUATION_LOG.replace(Some(CappedLog::default()))) + } + + fn take(&self) -> Vec { + EVALUATION_LOG + .with(|log| log.borrow_mut().take().unwrap_or_default().finish()) + } +} + +impl Drop for EvaluationLog { + fn drop(&mut self) { + EVALUATION_LOG.replace(self.0.take()); + } +} + +#[derive(serde::Serialize, utoipa::ToSchema)] +pub struct ProjectFlagEvaluation { + pub flags: Vec, + pub log: Vec, +} + +#[derive(Clone, serde::Serialize, utoipa::ToSchema)] +pub struct FlagContext { + pub project: Project, + pub team: FlagTeam, + pub users: Vec, + pub organization: Option, + pub scanned_files: Vec, + pub versions: Vec, + pub disclosures: Vec, + pub attributions: Vec, + pub moderation_thread: FlagModerationThread, +} + +#[derive(Clone, serde::Serialize, utoipa::ToSchema)] +pub struct FlagModerationThread { + pub id: ThreadId, + pub messages: Vec, +} + +#[derive(Clone, serde::Serialize, utoipa::ToSchema)] +pub struct FlagTeam { + pub id: TeamId, + pub members: Vec, +} + +#[derive(Clone, serde::Serialize, utoipa::ToSchema)] +pub struct ScannedFile { + #[schema(example = "AABBCCDD")] + pub version_id: VersionId, + #[schema(example = "EEFFGGHH")] + pub file_id: FileId, + #[schema(example = "META-INF/mods.toml")] + pub file_path: String, + #[schema( + example = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + )] + pub sha256: String, + pub found_patterns: Vec, +} + +#[derive(Clone, Debug)] +pub struct ProjectRuleCall { + pub ast: rhai::AST, + pub scan_files_for: BTreeMap, +} + +#[derive(Clone)] +pub struct ProjectFlagEngine { + rhai: Arc, + ro_pool: ReadOnlyPgPool, + redis: RedisPool, + file_host: Arc, +} + +impl ProjectFlagEngine { + pub fn new( + ro_pool: ReadOnlyPgPool, + redis: RedisPool, + file_host: Arc, + ) -> Result { + let mut rhai = rhai::Engine::new(); + rhai.disable_symbol("eval"); // eval considered evil + rhai.set_max_operations(1_000_000); + rhai.set_max_call_levels(64); + rhai.set_max_string_size(1_048_576); + rhai.set_max_array_size(10_000); + rhai.set_max_map_size(10_000); + rhai.on_print(|message| { + EVALUATION_LOG.with(|log| { + if let Some(log) = log.borrow_mut().as_mut() { + log.push(message, ENV.PROJECT_FLAG_RULE_MAX_LOG_BYTES); + } + }); + }); + Ok(Self { + rhai: Arc::new(rhai), + ro_pool, + redis, + file_host, + }) + } + + pub fn make_call( + &self, + params: &ProjectRuleParams, + ) -> Result { + let ast = self + .rhai + .compile(¶ms.expression) + .wrap_err("compiling script")?; + Ok(ProjectRuleCall { + ast, + scan_files_for: params.scan_files_for.clone(), + }) + } + + pub(super) async fn load_contexts( + &self, + project_ids: &[DBProjectId], + ) -> Result> { + if project_ids.is_empty() { + return Ok(HashMap::new()); + } + let projects = + DBProject::get_many_ids(project_ids, &*self.ro_pool, &self.redis) + .await + .wrap_err("fetching projects for flag evaluation")?; + let ids = project_ids.iter().map(|id| id.0).collect::>(); + let version_ids = sqlx::query_scalar!( + r#"SELECT id FROM versions WHERE mod_id = ANY($1) ORDER BY date_published DESC, id DESC"#, + &ids, + ).fetch_all(&*self.ro_pool).await + .wrap_err("fetching version IDs for flag evaluation")? + .into_iter().map(DBVersionId).collect::>(); + let mut versions = HashMap::<_, Vec>::new(); + for version in + DBVersion::get_many(&version_ids, &*self.ro_pool, &self.redis) + .await + .wrap_err("fetching versions for flag evaluation")? + { + versions + .entry(version.inner.project_id) + .or_default() + .push(Version::from(version)); + } + for versions in versions.values_mut() { + versions.sort_by(|a, b| { + b.date_published + .cmp(&a.date_published) + .then_with(|| b.id.0.cmp(&a.id.0)) + }); + } + let mut disclosures = HashMap::<_, Vec>::new(); + for disclosure in DBProjectDisclosure::get_many_for_projects( + project_ids, + false, + &*self.ro_pool, + ) + .await + .wrap_err("fetching disclosures for flag evaluation")? + { + disclosures + .entry(disclosure.project_id) + .or_default() + .push(ProjectDisclosureData::from_db(disclosure, true, true)); + } + let mut attributions = + load_attributions(project_ids, &self.ro_pool).await?; + let thread_ids = projects + .iter() + .map(|project| project.thread_id) + .collect::>(); + let mut threads = DBThread::get_many(&thread_ids, &*self.ro_pool) + .await + .wrap_err("fetching moderation threads for flag evaluation")? + .into_iter() + .map(|thread| (thread.id, thread)) + .collect::>(); + let mut organization_ids = projects + .iter() + .filter_map(|p| p.inner.organization_id) + .collect::>(); + organization_ids.sort_unstable_by_key(|id| id.0); + organization_ids.dedup(); + let organizations = DBOrganization::get_many_ids( + &organization_ids, + &*self.ro_pool, + &self.redis, + ) + .await + .wrap_err("fetching organizations for flag evaluation")? + .into_iter() + .map(|organization| (organization.id, organization)) + .collect::>(); + let mut team_ids = projects + .iter() + .map(|p| p.inner.team_id) + .chain(organizations.values().map(|o| o.team_id)) + .collect::>(); + team_ids.sort_unstable_by_key(|id| id.0); + team_ids.dedup(); + let members = DBTeamMember::get_from_team_full_many( + &team_ids, + &*self.ro_pool, + &self.redis, + ) + .await + .wrap_err("fetching team members for flag evaluation")?; + let mut user_ids = + members.iter().map(|m| m.user_id).collect::>(); + user_ids.sort_unstable_by_key(|id| id.0); + user_ids.dedup(); + let users = + DBUser::get_many_ids(&user_ids, &*self.ro_pool, &self.redis) + .await + .wrap_err("fetching team users for flag evaluation")? + .into_iter() + .map(|user| (user.id, User::from(user))) + .collect::>(); + let mut teams = HashMap::<_, Vec>::new(); + for member in members { + let user = users + .get(&member.user_id) + .cloned() + .ok_or_else(|| eyre!("team member user not found"))?; + teams + .entry(member.team_id) + .or_default() + .push(TeamMember::from_model(member, user, true)); + } + let mut contexts = HashMap::new(); + for project in projects { + let id = project.inner.id; + let team_id = project.inner.team_id; + let members = teams.get(&team_id).cloned().unwrap_or_default(); + let organization = project + .inner + .organization_id + .map(|id| { + let organization = + organizations.get(&id).cloned().ok_or_else(|| { + eyre!("project organization not found") + })?; + let members = teams + .get(&organization.team_id) + .cloned() + .unwrap_or_default(); + eyre::Ok(Organization::from(organization, members)) + }) + .transpose()?; + let thread = threads + .remove(&project.thread_id) + .ok_or_else(|| eyre!("project moderation thread not found"))?; + let moderation_thread = FlagModerationThread { + id: thread.id.into(), + messages: thread + .messages + .into_iter() + .map(|message| ThreadMessage { + id: message.id.into(), + author_id: message.author_id.map(Into::into), + body: message.body, + created: message.created, + hide_identity: message.hide_identity, + }) + .collect(), + }; + contexts.insert( + id, + FlagContext { + project: Project::from(project), + users: members + .iter() + .map(|member| member.user.clone()) + .collect(), + team: FlagTeam { + id: team_id.into(), + members, + }, + organization, + scanned_files: Vec::new(), + versions: versions.remove(&id).unwrap_or_default(), + disclosures: disclosures.remove(&id).unwrap_or_default(), + attributions: attributions.remove(&id).unwrap_or_default(), + moderation_thread, + }, + ); + } + + Ok(contexts) + } + + pub(super) async fn load_files( + &self, + project_ids: &[DBProjectId], + ) -> Result>> { + load_project_files(project_ids, &self.ro_pool).await + } + + pub(super) async fn prepare_context( + &self, + mut context: FlagContext, + files: &[ProjectScanFile], + patterns: &BTreeMap, + ) -> Result { + context.scanned_files = scan_project_files( + files, + patterns, + self.file_host.as_ref(), + &DOWNLOADS, + ) + .await?; + Ok(context) + } + + pub fn evaluate( + &self, + call: &ProjectRuleCall, + context: &FlagContext, + ) -> Result { + let (flags, log) = self.evaluate_with_log(call, context); + Ok(ProjectFlagEvaluation { flags: flags?, log }) + } + + pub(super) fn evaluate_with_log( + &self, + call: &ProjectRuleCall, + context: &FlagContext, + ) -> (Result>, Vec) { + self.evaluate_with_scan_error(call, context, None) + } + + pub(super) fn evaluate_with_scan_error( + &self, + call: &ProjectRuleCall, + context: &FlagContext, + scan_error: Option<&str>, + ) -> (Result>, Vec) { + let log = EvaluationLog::start(); + let flags = (|| { + if let Some(error) = scan_error + && !call.scan_files_for.is_empty() + { + return Err(eyre!("project file scan failed: {error}")); + } + let context = rhai::serde::to_dynamic(context) + .wrap_err("serializing flag context for script")? + .try_cast::() + .ok_or_else(|| eyre!("flag context must serialize to a map"))?; + let mut scope = rhai::Scope::new(); + for (name, value) in context { + if scan_error.is_some() && name == "scanned_files" { + continue; + } + scope.push_dynamic(name.to_string(), value); + } + let output = self + .rhai + .eval_ast_with_scope::(&mut scope, &call.ast) + .wrap_err("running script"); + let output = match scan_error { + Some(error) => output.wrap_err_with(|| { + format!("file scan unavailable for script: {error}") + }), + None => output, + }?; + output + .into_iter() + .map(|value| { + let value = + rhai::serde::from_dynamic::(&value) + .wrap_err("converting project flag to json")?; + serde_json::from_value::(value) + .wrap_err("decoding project flag") + }) + .collect::>>() + })(); + (flags, log.take()) + } + + pub async fn scan_flags_with_log( + &self, + call: &ProjectRuleCall, + project_id: DBProjectId, + ) -> Result { + tokio::time::timeout( + Duration::from_millis(ENV.PROJECT_FLAG_RULE_TIMEOUT), + async { + let context = self + .load_contexts(&[project_id]) + .await? + .remove(&project_id) + .ok_or_else(|| eyre!("project not found"))?; + let files = self + .load_files(&[project_id]) + .await? + .remove(&project_id) + .unwrap_or_default(); + let (context, scan_error) = match self + .prepare_context( + context.clone(), + &files, + &call.scan_files_for, + ) + .await + { + Ok(context) => (context, None), + Err(error) => (context, Some(format!("{error:#}"))), + }; + let engine = self.clone(); + let call = call.clone(); + tokio::task::spawn_blocking(move || { + let (flags, log) = engine.evaluate_with_scan_error( + &call, + &context, + scan_error.as_deref(), + ); + Ok(ProjectFlagEvaluation { flags: flags?, log }) + }) + .await + .wrap_err("joining project flag evaluation")? + }, + ) + .await + .wrap_err("project flag rule evaluation timed out")? + } + + pub async fn scan_flags( + &self, + call: &ProjectRuleCall, + project_id: DBProjectId, + ) -> Result> { + Ok(self.scan_flags_with_log(call, project_id).await?.flags) + } +} diff --git a/apps/labrinth/src/queue/moderation/project_flag_attribution.rs b/apps/labrinth/src/queue/moderation/project_flag_attribution.rs new file mode 100644 index 00000000000..1fe2aeebda1 --- /dev/null +++ b/apps/labrinth/src/queue/moderation/project_flag_attribution.rs @@ -0,0 +1,303 @@ +use std::collections::{HashMap, HashSet}; + +use ariadne::ids::UserId; +use chrono::{DateTime, Utc}; +use eyre::{Result, WrapErr}; +use serde::Serialize; +use utoipa::ToSchema; + +use super::ApprovalType; +use crate::{ + database::{ReadOnlyPgPool, models::DBProjectId}, + models::{ + ids::{AttributionGroupId, ProjectId, VersionId}, + projects::{AttributionResolution, FlameProject}, + }, +}; + +#[derive(Clone, Serialize, ToSchema)] +pub struct FlagAttributionGroup { + pub id: AttributionGroupId, + pub flame_project: Option, + pub attribution: Option, + pub attributed_at: Option>, + pub attributed_by: Option, + pub files: Vec, + pub versions: Vec, + pub override_files_on_platform: Vec, +} + +#[derive(Clone, Serialize, ToSchema)] +pub struct FlagAttributionFile { + pub name: String, + pub sha1: String, + pub versions: Vec, + pub moderation_external_license_id: Option, + pub moderation_external_license: Option, +} + +#[derive(Clone, Serialize, ToSchema)] +pub struct FlagAttributionVersion { + pub id: VersionId, + pub name: String, + pub version_number: String, + pub date_created: DateTime, +} + +#[derive(Clone, Serialize, ToSchema)] +pub struct FlagAttributionOverrideFile { + pub file_path: String, + pub sha1: String, + pub version_id: VersionId, + pub platform_version_id: VersionId, + pub platform_project_id: ProjectId, +} + +#[derive(Clone, Serialize, ToSchema)] +pub struct FlagAttributionExternalLicense { + pub id: i64, + pub title: Option, + pub status: ApprovalType, + pub link: Option, + pub exceptions: Option, + pub proof: Option, + pub flame_project_id: Option, + pub inserted_at: Option>, + pub inserted_by: Option, + pub updated_at: Option>, + pub updated_by: Option, +} + +/// Loads unredacted moderator-only permission evidence for flag evaluation. +pub(super) async fn load_attributions( + project_ids: &[DBProjectId], + pool: &ReadOnlyPgPool, +) -> Result>> { + let mut result: HashMap<_, Vec<_>> = project_ids + .iter() + .copied() + .map(|id| (id, Vec::new())) + .collect(); + if project_ids.is_empty() { + return Ok(result); + } + let ids: Vec<_> = project_ids.iter().map(|id| id.0).collect(); + let groups = sqlx::query!( + r#" + SELECT id, project_id, flame_project, attribution, attributed_at, attributed_by + FROM project_attribution_groups + WHERE project_id = ANY($1) + ORDER BY project_id, id + "#, + &ids, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching attribution groups for flag evaluation")?; + if groups.is_empty() { + return Ok(result); + } + let group_ids: Vec<_> = groups.iter().map(|group| group.id).collect(); + let files = sqlx::query!( + r#" + SELECT paf.group_id AS "group_id!", paf.name AS "name!", + CONVERT_FROM(paf.sha1, 'UTF8') AS "sha1!", + paf.moderation_external_license_id, + COALESCE(ARRAY_AGG(DISTINCT aev.id) FILTER (WHERE aev.id IS NOT NULL), '{}') AS "version_ids!: Vec" + FROM project_attribution_files paf + INNER JOIN project_attribution_groups pag ON pag.id = paf.group_id + LEFT JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1 + LEFT JOIN files f ON f.id = ofs.file_id + LEFT JOIN versions v ON v.id = f.version_id AND v.mod_id = pag.project_id + LEFT JOIN attribution_enforced_versions aev ON aev.id = v.id + WHERE paf.group_id = ANY($1) + GROUP BY paf.group_id, paf.name, paf.sha1, paf.moderation_external_license_id + ORDER BY paf.group_id, paf.name, paf.sha1 + "#, + &group_ids, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching attribution files for flag evaluation")?; + + let mut license_ids: Vec<_> = files + .iter() + .filter_map(|file| file.moderation_external_license_id) + .collect(); + license_ids.sort_unstable(); + license_ids.dedup(); + let mut licenses = HashMap::new(); + if !license_ids.is_empty() { + let rows = sqlx::query!( + r#" + SELECT id, title, status, link, exceptions, proof, flame_project_id, + inserted_at, inserted_by, updated_at, updated_by + FROM moderation_external_licenses + WHERE id = ANY($1) + "#, + &license_ids, + ) + .fetch_all(&**pool) + .await + .wrap_err( + "fetching attribution external licenses for flag evaluation", + )?; + for row in rows { + licenses.insert( + row.id, + FlagAttributionExternalLicense { + id: row.id, + title: row.title, + status: ApprovalType::from_string(&row.status) + .unwrap_or(ApprovalType::Unidentified), + link: row.link, + exceptions: row.exceptions, + proof: row.proof, + flame_project_id: row.flame_project_id, + inserted_at: row.inserted_at, + inserted_by: row.inserted_by.map(|id| UserId(id as u64)), + updated_at: row.updated_at, + updated_by: row.updated_by.map(|id| UserId(id as u64)), + }, + ); + } + } + + let mut version_ids: Vec<_> = files + .iter() + .flat_map(|file| file.version_ids.iter().copied()) + .collect(); + version_ids.sort_unstable(); + version_ids.dedup(); + let mut versions = Vec::new(); + if !version_ids.is_empty() { + let rows = sqlx::query!( + r#" + SELECT id, name, version_number, date_published + FROM versions + WHERE id = ANY($1) + ORDER BY date_published DESC, id + "#, + &version_ids, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching attribution versions for flag evaluation")?; + versions.extend(rows.into_iter().map(|row| FlagAttributionVersion { + id: VersionId(row.id as u64), + name: row.name, + version_number: row.version_number, + date_created: row.date_published, + })); + } + let version_order: HashMap<_, _> = versions + .iter() + .enumerate() + .map(|(index, version)| (version.id, index)) + .collect(); + let mut files_by_group: HashMap<_, Vec<_>> = HashMap::new(); + for file in files { + let mut file_versions: Vec<_> = file + .version_ids + .into_iter() + .map(|id| VersionId(id as u64)) + .collect(); + file_versions.sort_by_key(|id| { + version_order.get(id).copied().unwrap_or(usize::MAX) + }); + files_by_group.entry(file.group_id).or_default().push( + FlagAttributionFile { + name: file.name, + sha1: file.sha1, + versions: file_versions, + moderation_external_license_id: file + .moderation_external_license_id, + moderation_external_license: file + .moderation_external_license_id + .and_then(|id| licenses.get(&id).cloned()), + }, + ); + } + + let platform_files = sqlx::query!( + r#" + SELECT source_version.mod_id AS project_id, + ofs.file_path AS "file_path!", CONVERT_FROM(ofs.sha1, 'UTF8') AS "sha1!", + source_file.version_id, platform_file.version_id AS platform_version_id, + platform_version.mod_id AS platform_project_id + FROM files source_file + INNER JOIN versions source_version ON source_version.id = source_file.version_id + INNER JOIN override_file_sources ofs ON ofs.file_id = source_file.id + INNER JOIN hashes h ON h.algorithm = 'sha1' AND h.hash = ofs.sha1 + INNER JOIN files platform_file ON platform_file.id = h.file_id + INNER JOIN versions platform_version ON platform_version.id = platform_file.version_id + WHERE source_version.mod_id = ANY($1) + ORDER BY source_version.mod_id, source_file.version_id, ofs.file_path, + platform_file.version_id, platform_version.mod_id + "#, + &ids, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching attribution override files on platform for flag evaluation")?; + let mut platform_files_by_project: HashMap<_, Vec<_>> = HashMap::new(); + for row in platform_files { + platform_files_by_project + .entry(row.project_id) + .or_default() + .push(FlagAttributionOverrideFile { + file_path: row.file_path, + sha1: row.sha1, + version_id: VersionId(row.version_id as u64), + platform_version_id: VersionId(row.platform_version_id as u64), + platform_project_id: ProjectId(row.platform_project_id as u64), + }); + } + + for group in groups { + let files = files_by_group.remove(&group.id).unwrap_or_default(); + let group_version_ids: HashSet<_> = files + .iter() + .flat_map(|file| file.versions.iter().copied()) + .collect(); + let sha1s: HashSet<_> = + files.iter().map(|file| file.sha1.as_str()).collect(); + let override_files_on_platform = platform_files_by_project + .get(&group.project_id) + .into_iter() + .flatten() + .filter(|file| sha1s.contains(file.sha1.as_str())) + .cloned() + .collect(); + let attribution = group + .attribution + .map(serde_json::from_value::) + .transpose() + .wrap_err( + "deserializing attribution resolution for flag evaluation", + )?; + let flame_project = group + .flame_project + .map(serde_json::from_value::) + .transpose() + .wrap_err("deserializing attribution source for flag evaluation")?; + result + .entry(DBProjectId(group.project_id)) + .or_default() + .push(FlagAttributionGroup { + id: AttributionGroupId(group.id as u64), + flame_project, + attribution, + attributed_at: group.attributed_at, + attributed_by: group.attributed_by.map(|id| UserId(id as u64)), + versions: versions + .iter() + .filter(|version| group_version_ids.contains(&version.id)) + .cloned() + .collect(), + files, + override_files_on_platform, + }); + } + Ok(result) +} diff --git a/apps/labrinth/src/queue/moderation/project_flag_files.rs b/apps/labrinth/src/queue/moderation/project_flag_files.rs new file mode 100644 index 00000000000..e342510576b --- /dev/null +++ b/apps/labrinth/src/queue/moderation/project_flag_files.rs @@ -0,0 +1,326 @@ +use std::collections::{BTreeMap, HashMap}; +use std::io::{Cursor, Read}; +use std::sync::Arc; + +use eyre::{Context, Result, ensure, eyre}; +use regex::Regex; +use sha2::{Digest, Sha256}; + +use zip::ZipArchive; + +use crate::database::ReadOnlyPgPool; +use crate::database::models::ids::{DBFileId, DBProjectId, DBVersionId}; +use crate::env::ENV; +use crate::file_hosting::{FileHost, FileHostPublicity}; + +use crate::database::models::project_flag_item::PatternName; + +use super::project_flag::ScannedFile; + +const PREFIX_BYTES: usize = 4096; + +type ScannedLeaf = (String, String, Vec); + +#[derive(Default)] +struct Budget { + bytes: u64, + entries: usize, +} + +impl Budget { + fn consume(&mut self, bytes: u64) -> Result<()> { + (bytes <= ENV.PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES - self.bytes).then_some(()).ok_or_else(|| eyre!("project file scan exceeds total byte budget")).wrap_err_with(|| eyre!("requested {bytes} bytes, remaining {} bytes, maximum {} bytes", ENV.PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES - self.bytes, ENV.PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES))?; + self.bytes += bytes; + Ok(()) + } +} + +pub(super) struct ProjectScanFile { + pub file_id: DBFileId, + pub version_id: DBVersionId, + pub filename: String, + pub url: String, + pub size: i64, +} + +pub(super) async fn load_project_files( + project_ids: &[DBProjectId], + ro_pool: &ReadOnlyPgPool, +) -> Result>> { + let mut files = project_ids + .iter() + .copied() + .map(|id| (id, Vec::new())) + .collect::>(); + if project_ids.is_empty() { + return Ok(files); + } + let ids = project_ids.iter().map(|id| id.0).collect::>(); + let rows = sqlx::query!( + r#" + SELECT v.mod_id AS project_id, f.id, f.version_id, f.filename, f.url, f.size + FROM files f + INNER JOIN versions v ON v.id = f.version_id + WHERE v.mod_id = ANY($1) + ORDER BY v.mod_id, v.date_published DESC, v.id DESC, f.id + "#, + &ids + ) + .fetch_all(&**ro_pool) + .await + .wrap_err("fetching project files for content scanning")?; + + for row in rows { + files.entry(DBProjectId(row.project_id)).or_default().push( + ProjectScanFile { + file_id: DBFileId(row.id), + version_id: DBVersionId(row.version_id), + filename: row.filename, + url: row.url, + size: i64::from(row.size), + }, + ); + } + Ok(files) +} + +pub(super) async fn scan_project_files( + files: &[ProjectScanFile], + patterns: &BTreeMap, + file_host: &dyn FileHost, + downloads: &Arc, +) -> Result> { + let mut scanned_files = Vec::new(); + let mut budget = Budget::default(); + let patterns = Arc::new(patterns.clone()); + for file in files { + (file.size >= 0 + && file.size as u64 <= ENV.PROJECT_FLAG_SCAN_MAX_ARTIFACT_BYTES) + .then_some(()) + .ok_or_else(|| { + eyre!( + "file {} exceeds declared artifact size budget", + file.file_id.0 + ) + }) + .wrap_err_with(|| { + eyre!( + "declared size {} bytes, maximum {} bytes", + file.size, + ENV.PROJECT_FLAG_SCAN_MAX_ARTIFACT_BYTES + ) + })?; + (file.size as u64 + <= ENV.PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES - budget.bytes).then_some(()).ok_or_else(|| eyre!("file {} exceeds remaining declared byte budget", + file.file_id.0)).wrap_err_with(|| eyre!("declared size {} bytes, remaining {} bytes, maximum {} bytes", file.size, ENV.PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES - budget.bytes, ENV.PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES))?; + let cdn_prefix = format!("{}/", ENV.CDN_URL.trim_end_matches('/')); + let key = file.url.strip_prefix(&cdn_prefix).ok_or_else(|| { + eyre::eyre!("project file url is outside the configured cdn") + })?; + let key = urlencoding::decode(key) + .wrap_err("decoding project file storage key")?; + ensure!( + !key.is_empty() + && !key.contains('\\') + && key.split('/').all(|part| !matches!(part, "" | "." | "..")), + "invalid project file storage key" + ); + let permit = Arc::clone(downloads) + .acquire_owned() + .await + .wrap_err("acquiring project file download permit")?; + let data = file_host + .read_file(&key, FileHostPublicity::Public) + .await + .wrap_err_with(|| { + format!("reading project file {} from storage", file.file_id.0) + })?; + + (data.len() as u64 <= ENV.PROJECT_FLAG_SCAN_MAX_ARTIFACT_BYTES) + .then_some(()) + .ok_or_else(|| eyre!("project file exceeds artifact size budget")) + .wrap_err_with(|| { + eyre!( + "actual size {} bytes, maximum {} bytes", + data.len(), + ENV.PROJECT_FLAG_SCAN_MAX_ARTIFACT_BYTES + ) + })?; + budget.consume(data.len() as u64)?; + let path = format!("{}/{}", file.version_id.0, file.filename); + let patterns = Arc::clone(&patterns); + let (artifact_leaves, next_budget) = + tokio::task::spawn_blocking(move || { + let _permit = permit; + let mut leaves = Vec::new(); + scan_contents( + &data, + &path, + 0, + &patterns, + &mut budget, + &mut leaves, + )?; + eyre::Ok((leaves, budget)) + }) + .await + .wrap_err("joining project file content scan")??; + budget = next_budget; + scanned_files.extend(artifact_leaves.into_iter().map( + |(file_path, sha256, found_patterns)| ScannedFile { + version_id: file.version_id.into(), + file_id: file.file_id.into(), + file_path, + sha256, + found_patterns, + }, + )); + } + Ok(scanned_files) +} + +fn is_archive(data: &[u8], path: &str) -> bool { + data.starts_with(b"PK\x03\x04") + || data.starts_with(b"PK\x05\x06") + || data.starts_with(b"PK\x07\x08") + || path.rsplit('.').next().is_some_and(|extension| { + ["zip", "jar", "mrpack"] + .iter() + .any(|candidate| extension.eq_ignore_ascii_case(candidate)) + }) +} + +fn is_text_prefix(data: &[u8]) -> bool { + let text = match std::str::from_utf8(data) { + Ok(text) => text, + Err(error) if error.error_len().is_none() => { + std::str::from_utf8(&data[..error.valid_up_to()]) + .unwrap_or_default() + } + Err(_) => return false, + }; + !text.chars().any(|character| { + character == '\0' + || (character.is_control() && !character.is_whitespace()) + }) +} + +fn scan_contents( + data: &[u8], + path: &str, + depth: usize, + patterns: &BTreeMap, + budget: &mut Budget, + leaves: &mut Vec, +) -> Result<()> { + if is_archive(data, path) { + (depth < ENV.PROJECT_FLAG_SCAN_MAX_DEPTH) + .then_some(()) + .ok_or_else(|| { + eyre!("archive nesting depth budget exceeded at {path}") + }) + .wrap_err_with(|| { + eyre!( + "attempted archive layer {}, maximum {} layers", + depth + 1, + ENV.PROJECT_FLAG_SCAN_MAX_DEPTH + ) + })?; + let mut archive = ZipArchive::new(Cursor::new(data)) + .wrap_err_with(|| format!("opening archive {path}"))?; + (archive.len() <= ENV.PROJECT_FLAG_SCAN_MAX_ENTRIES - budget.entries).then_some(()).ok_or_else(|| eyre!("archive entry budget exceeded at {path}")).wrap_err_with(|| eyre!("archive contains {} entries, remaining {} entries, maximum {} entries", archive.len(), ENV.PROJECT_FLAG_SCAN_MAX_ENTRIES - budget.entries, ENV.PROJECT_FLAG_SCAN_MAX_ENTRIES))?; + budget.entries += archive.len(); + for index in 0..archive.len() { + let mut entry = archive + .by_index(index) + .wrap_err_with(|| format!("opening entry {index} in {path}"))?; + let entry_path = format!( + "{path}!/{}", + entry.name().wrap_err("decoding archive entry name")? + ); + if entry.is_dir() { + continue; + } + let mut contents = Vec::new(); + let mut prefix = [0; PREFIX_BYTES]; + while contents.len() < PREFIX_BYTES { + let remaining = PREFIX_BYTES - contents.len(); + let count = entry + .read(&mut prefix[..remaining]) + .wrap_err("reading archive entry prefix")?; + if count == 0 { + break; + } + budget.consume(count as u64)?; + contents.extend_from_slice(&prefix[..count]); + } + let archive_entry = is_archive(&contents, &entry_path); + if !archive_entry && !is_text_prefix(&contents) { + continue; + } + let limit = if archive_entry { + ENV.PROJECT_FLAG_SCAN_MAX_ARTIFACT_BYTES + } else { + ENV.PROJECT_FLAG_SCAN_MAX_TEXT_BYTES + }; + (entry.size() <= limit) + .then_some(()) + .ok_or_else(|| { + eyre!("archive entry size budget exceeded at {entry_path}") + }) + .wrap_err_with(|| { + eyre!( + "declared size {} bytes, maximum {limit} bytes", + entry.size() + ) + })?; + (entry.size().saturating_sub(contents.len() as u64) + <= ENV.PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES - budget.bytes).then_some(()).ok_or_else(|| eyre!("remaining byte budget exceeded at {entry_path}")).wrap_err_with(|| eyre!("requested {} bytes, remaining {} bytes, maximum {} bytes", entry.size().saturating_sub(contents.len() as u64), ENV.PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES - budget.bytes, ENV.PROJECT_FLAG_SCAN_MAX_TOTAL_BYTES))?; + loop { + let count = entry + .read(&mut prefix) + .wrap_err("reading archive entry contents")?; + if count == 0 { + break; + } + (contents.len() as u64 + count as u64 <= limit).then_some(()).ok_or_else(|| eyre!("archive entry size budget exceeded at {entry_path}")).wrap_err_with(|| eyre!("actual size at least {} bytes, maximum {limit} bytes", contents.len() as u64 + count as u64))?; + budget.consume(count as u64)?; + contents.extend_from_slice(&prefix[..count]); + } + scan_contents( + &contents, + &entry_path, + depth + 1, + patterns, + budget, + leaves, + )?; + } + } else if is_text_prefix(&data[..data.len().min(PREFIX_BYTES)]) { + (data.len() as u64 <= ENV.PROJECT_FLAG_SCAN_MAX_TEXT_BYTES) + .then_some(()) + .ok_or_else(|| eyre!("text size budget exceeded at {path}")) + .wrap_err_with(|| { + eyre!( + "actual size {} bytes, maximum {} bytes", + data.len(), + ENV.PROJECT_FLAG_SCAN_MAX_TEXT_BYTES + ) + })?; + if let Ok(text) = std::str::from_utf8(data) + && is_text_prefix(data) + { + let found_patterns = patterns + .iter() + .filter(|(_, pattern)| pattern.is_match(text)) + .map(|(name, _)| name.clone()) + .collect(); + leaves.push(( + path.to_owned(), + hex::encode(Sha256::digest(data)), + found_patterns, + )); + } + } + Ok(()) +} diff --git a/apps/labrinth/src/queue/moderation/project_flag_run.rs b/apps/labrinth/src/queue/moderation/project_flag_run.rs new file mode 100644 index 00000000000..1b16d67fa03 --- /dev/null +++ b/apps/labrinth/src/queue/moderation/project_flag_run.rs @@ -0,0 +1,516 @@ +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + sync::Arc, + time::Duration, +}; + +use super::project_flag::{ProjectFlagEngine, ProjectRuleCall}; +use eyre::{Result, WrapErr, eyre}; +use futures::{StreamExt, TryStreamExt, stream}; +use rdkafka::{ + Message, Offset, TopicPartitionList, + consumer::{CommitMode, Consumer, StreamConsumer}, + error::KafkaError, + producer::{FutureRecord, Producer}, +}; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use xredis::RedisPool; + +use crate::{ + database::models::{ + DBProjectId, ProjectFlagRuleId, ProjectFlagRunId, + project_flag_item::{ + DBProjectFlagRule, DBProjectFlagRunResult, PatternName, + ProjectFlagRuleResult, + }, + }, + database::{PgPool, ReadOnlyPgPool}, + env::ENV, + file_hosting::FileHost, + models::ids::ProjectId, + util::kafka::{ + KAFKA_DELIVERY_TIMEOUT, KAFKA_OPERATION_INTERVAL, KafkaClientState, + KafkaEvent, consumer_config, create_project_flag_run_producer, + sentry_trace_headers, + }, +}; + +pub const PROJECT_FLAG_RUN_TOPIC: &str = "private.labrinth.project-flag-run.v1"; +pub const PROJECT_FLAG_RUN_CONSUMER_GROUP: &str = "project-flag-runs"; +const SCAN_BATCH_DELAY: Duration = Duration::from_secs(1); + +/// Persists each batch before acknowledging its Kafka offsets. +pub async fn consume_project_flag_runs( + pool: PgPool, + ro_pool: ReadOnlyPgPool, + redis: RedisPool, + file_host: Arc, +) -> Result<()> { + let engine = ProjectFlagEngine::new(ro_pool, redis.clone(), file_host)?; + let batch_size = ENV.PROJECT_FLAG_SCAN_BATCH_SIZE.get(); + let consumer: StreamConsumer = + consumer_config(PROJECT_FLAG_RUN_CONSUMER_GROUP) + .set("isolation.level", "read_committed") + .set("enable.auto.offset.store", "false") + .create() + .wrap_err("creating project flag run Kafka consumer")?; + let consumer = Arc::new(consumer); + consumer + .subscribe(&[PROJECT_FLAG_RUN_TOPIC]) + .wrap_err("subscribing to project flag run Kafka topic")?; + + tracing::warn!( + topic = PROJECT_FLAG_RUN_TOPIC, + consumer_group = PROJECT_FLAG_RUN_CONSUMER_GROUP, + batch_size, + "Started project flag run consumer" + ); + + let mut messages = Vec::with_capacity(batch_size); + loop { + messages.clear(); + let result = async { + messages.push(consumer.recv().await.wrap_err( + "receiving initial project flag run Kafka message", + )?); + let deadline = tokio::time::Instant::now() + SCAN_BATCH_DELAY; + while messages.len() < batch_size { + match tokio::time::timeout_at(deadline, consumer.recv()).await { + Ok(message) => messages.push(message.wrap_err( + "receiving project flag run Kafka batch message", + )?), + Err(_) => break, + } + } + + let events = messages + .iter() + .map(|message| { + let payload = message.payload().ok_or_else(|| { + eyre!("project flag run Kafka message has no payload") + })?; + serde_json::from_slice::(payload) + .wrap_err("decoding project flag run Kafka message") + }) + .collect::>>()?; + + scan_batch(&engine, &redis, &pool, events).await?; + let mut offsets = BTreeMap::new(); + for message in &messages { + offsets + .entry(message.partition()) + .and_modify(|offset: &mut i64| { + *offset = (*offset).max(message.offset() + 1) + }) + .or_insert(message.offset() + 1); + } + let mut positions = TopicPartitionList::new(); + for (partition, offset) in offsets { + positions + .add_partition_offset( + PROJECT_FLAG_RUN_TOPIC, + partition, + Offset::Offset(offset), + ) + .wrap_err("building project flag run consumer offsets")?; + } + let consumer = consumer.clone(); + tokio::task::spawn_blocking(move || { + consumer.commit(&positions, CommitMode::Sync) + }) + .await + .wrap_err("joining project flag run offset commit")? + .wrap_err("committing project flag run consumer offsets")?; + eyre::Ok(()) + } + .await; + if let Err(error) = result { + tracing::error!(?error, "Project flag run consumer failed"); + return Err(error); + } + } +} + +struct PreparedRule { + id: ProjectFlagRuleId, + call: Result, + pattern_names: BTreeMap, +} + +struct PreparedRun { + error: Option, + rules: Vec, + patterns: BTreeMap, +} + +async fn scan_batch( + engine: &ProjectFlagEngine, + redis: &RedisPool, + pool: &PgPool, + events: Vec, +) -> Result<()> { + let mut snapshots = HashMap::new(); + let mut connection = redis + .connect() + .await + .wrap_err("connecting to Redis for project flag rule snapshots")?; + for run_id in events.iter().map(|event| event.run_id) { + if snapshots.contains_key(&run_id) { + continue; + } + let Some(id) = run_id else { + let rules = DBProjectFlagRule::all(pool) + .await + .wrap_err("loading rules for standalone project scans")?; + snapshots.insert(None, Ok(rules)); + continue; + }; + let key = connection.key().entity("project_flag_run_rules:v1", id.0); + let snapshot: Option> = redis::cmd("GET") + .arg(key) + .query_async(&mut connection) + .await + .wrap_err("fetching project flag rule snapshot")?; + let rules = snapshot + .ok_or_else(|| format!("rule snapshot missing for run {}", id.0)) + .and_then(|snapshot| { + serde_json::from_slice::>(&snapshot) + .map_err(|error| { + format!("decoding project flag rule snapshot: {error}") + }) + }); + snapshots.insert(run_id, rules); + } + drop(connection); + + let compile_engine = engine.clone(); + let runs = tokio::task::spawn_blocking(move || { + snapshots + .into_iter() + .map(|(run_id, rules)| { + let mut prepared = PreparedRun { + error: None, + rules: Vec::new(), + patterns: BTreeMap::new(), + }; + let rules = match rules { + Ok(rules) => rules, + Err(error) => { + prepared.error = Some(error); + return eyre::Ok((run_id, prepared)); + } + }; + for rule in rules { + let call = compile_engine + .make_call(&rule.params) + .map_err(|error| format!("{error:#}")); + let mut pattern_names = BTreeMap::new(); + for (name, regex) in &rule.params.scan_files_for { + let internal = + PatternName(format!("{}:{}", rule.id.0, name.0)); + prepared + .patterns + .insert(internal.clone(), regex.clone()); + pattern_names.insert(internal, name.clone()); + } + prepared.rules.push(PreparedRule { + id: rule.id, + call, + pattern_names, + }); + } + eyre::Ok((run_id, prepared)) + }) + .collect::>>() + }) + .await + .wrap_err("joining project flag rule compilation")??; + let runs = Arc::new(runs); + let project_ids = events + .iter() + .map(|event| DBProjectId::from(event.project_id)) + .collect::>() + .into_iter() + .collect::>(); + let (contexts, files) = tokio::try_join!( + engine.load_contexts(&project_ids), + engine.load_files(&project_ids), + )?; + + let consumed_jobs = events + .iter() + .filter_map(|event| { + event.run_id.map(|run_id| (run_id, event.project_id.into())) + }) + .collect::>(); + let events = events.into_iter().filter(|event| { + let exists = contexts.contains_key(&DBProjectId::from(event.project_id)); + if !exists { + tracing::info!(project_id = %event.project_id, "Skipping deleted project flag scan"); + } + exists + }); + let prepared = futures::future::join_all(events.map(|event| { + let contexts = &contexts; + let files = &files; + let runs = &runs; + async move { + let prepared = async { + let project_id = DBProjectId::from(event.project_id); + let context = contexts + .get(&project_id) + .ok_or_else(|| { + eyre!( + "project context missing for {}", + event.project_id + ) + })? + .clone(); + let run = runs + .get(&event.run_id) + .ok_or_else(|| eyre!("project scan rules not found"))?; + if let Some(error) = &run.error { + return Err(eyre!(error.clone())); + } + let files = files + .get(&project_id) + .map(Vec::as_slice) + .unwrap_or_default(); + let scanned_context = engine + .prepare_context(context.clone(), files, &run.patterns) + .await + .wrap_err_with(|| { + format!( + "preparing project {} for flag scanning", + event.project_id + ) + }); + let (context, scan_error) = match scanned_context { + Ok(context) => (context, None), + Err(error) => (context, Some(format!("{error:#}"))), + }; + eyre::Ok((context, scan_error)) + } + .await; + (event, prepared) + } + })) + .await; + + let tasks = prepared + .into_iter() + .map(|(event, context)| { + let engine = engine.clone(); + let runs = runs.clone(); + tokio::task::spawn_blocking(move || { + let (mut context, scan_error) = match context { + Ok(context) => context, + Err(error) => { + return DBProjectFlagRunResult { + run_id: event.run_id, + project_id: event.project_id.into(), + rules: BTreeMap::new(), + error: Some(format!("{error:#}")), + }; + } + }; + let original_patterns = context + .scanned_files + .iter() + .map(|file| file.found_patterns.clone()) + .collect::>(); + let mut rules = BTreeMap::new(); + for rule in &runs[&event.run_id].rules { + for (file, names) in + context.scanned_files.iter_mut().zip(&original_patterns) + { + file.found_patterns = names + .iter() + .filter_map(|name| { + rule.pattern_names.get(name).cloned() + }) + .collect(); + } + let (evaluation, log) = match &rule.call { + Ok(call) => engine.evaluate_with_scan_error( + call, + &context, + scan_error.as_deref(), + ), + Err(error) => (Err(eyre!(error.clone())), Vec::new()), + }; + let (flags, error) = match evaluation { + Ok(flags) => (flags, None), + Err(error) => (Vec::new(), Some(format!("{error:#}"))), + }; + rules.insert( + rule.id.0.to_string(), + ProjectFlagRuleResult { flags, log, error }, + ); + } + DBProjectFlagRunResult { + run_id: event.run_id, + project_id: event.project_id.into(), + rules, + error: None, + } + }) + }) + .collect::>(); + let mut results = Vec::new(); + for result in futures::future::join_all(tasks).await { + results.push(result.wrap_err("joining project flag batch evaluation")?); + } + let result_count = results.len(); + DBProjectFlagRunResult::upsert_many(results, &consumed_jobs, pool).await?; + tracing::info!(result_count, "Persisted project flag run batch results"); + Ok(()) +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ProjectFlagRunEvent { + pub project_id: ProjectId, + pub run_id: Option, +} + +#[derive(Debug, thiserror::Error)] +#[error("project flag run dispatch outcome is unknown")] +pub struct ProjectFlagDispatchUncertain; + +/// Stores a permanent JSON rules snapshot under the run ID and atomically +/// publishes its project jobs. Consumers must use `read_committed` isolation. +/// Snapshots are retained on failure, including indeterminate commit outcomes. +/// `ProjectFlagDispatchUncertain` errors must leave the run active so committed +/// jobs can persist their results. Commit retries reuse the same transaction; +/// retrying an entire run is not guaranteed to be duplicate-free. +pub async fn dispatch_project_flag_run( + run_id: ProjectFlagRunId, + rules: Vec, + project_ids: Vec, + redis: &RedisPool, + kafka: &KafkaClientState, +) -> Result<()> { + let snapshot = serde_json::to_vec(&rules) + .wrap_err("serializing project flag run rules")?; + tokio::time::timeout(KAFKA_DELIVERY_TIMEOUT, async { + let mut connection = redis + .connect() + .await + .wrap_err("connecting to Redis for project flag run")?; + let key = connection + .key() + .entity("project_flag_run_rules:v1", run_id.0); + redis::cmd("SET") + .arg(&key) + .arg(&snapshot) + .arg("NX") + .query_async::>(&mut connection) + .await + .wrap_err("storing project flag run rules")? + .ok_or_else(|| eyre!("project flag run rules already exist"))?; + eyre::Ok(()) + }) + .await + .wrap_err("timed out storing project flag run rules")??; + + if project_ids.is_empty() { + return Ok(()); + } + + let headers = sentry_trace_headers(); + let runtime = tokio::runtime::Handle::current(); + let mut guard = kafka.project_flag_run_producer.clone().lock_owned().await; + tokio::task::spawn_blocking(move || { + let producer = match guard.take() { + Some(producer) => producer, + None => create_project_flag_run_producer()?, + }; + + let mut commit_uncertain = false; + let result = (|| { + producer + .begin_transaction() + .wrap_err("beginning project flag run Kafka transaction")?; + runtime.block_on(async { + tokio::time::timeout(KAFKA_DELIVERY_TIMEOUT, async { + let producer = &producer; + let headers = &headers; + stream::iter(project_ids) + .map(|project_id| async move { + let project_id = ProjectId::from(project_id); + let event = KafkaEvent::new( + PROJECT_FLAG_RUN_TOPIC, + ProjectFlagRunEvent { project_id, run_id: Some(run_id) }, + ); + let payload = serde_json::to_vec(&event) + .wrap_err("serializing project flag run event")?; + let project_key = project_id.to_string(); + producer + .send( + FutureRecord::to(PROJECT_FLAG_RUN_TOPIC) + .key(&project_key) + .payload(&payload) + .headers(headers.clone()), + KAFKA_OPERATION_INTERVAL, + ) + .await + .map_err(|(error, _)| eyre!(error)) + .wrap_err("publishing project flag run event")?; + eyre::Ok(()) + }) + .buffer_unordered(100) + .try_collect::<()>() + .await + }) + .await + .wrap_err("timed out publishing project flag run events")? + })?; + commit_uncertain = true; + for attempt in 0..3 { + match producer.commit_transaction(KAFKA_DELIVERY_TIMEOUT) { + Ok(()) => { + commit_uncertain = false; + break; + } + Err(KafkaError::Transaction(error)) + if error.txn_requires_abort() => + { + commit_uncertain = false; + return Err(eyre!(error)) + .wrap_err("committing project flag run Kafka transaction"); + } + Err(KafkaError::Transaction(error)) + if error.is_retriable() && attempt < 2 => + { + tracing::warn!(%error, "Retrying project flag run Kafka commit"); + } + Err(error) => { + return Err(eyre!(error)) + .wrap_err("committing project flag run Kafka transaction"); + } + } + } + eyre::Ok(()) + })(); + + if let Err(error) = result { + if commit_uncertain { + return Err(error.wrap_err(ProjectFlagDispatchUncertain)); + } + if let Err(abort_error) = + producer.abort_transaction(KAFKA_DELIVERY_TIMEOUT) + { + return Err(error.wrap_err(format!( + "aborting project flag run Kafka transaction failed: {abort_error}" + ))); + } + return Err(error); + } + *guard = Some(producer); + Ok(()) + }) + .await + .wrap_err("joining project flag run dispatch") + .wrap_err(ProjectFlagDispatchUncertain)? +} diff --git a/apps/labrinth/src/routes/internal/mod.rs b/apps/labrinth/src/routes/internal/mod.rs index d31ad83ead6..30cc179db62 100644 --- a/apps/labrinth/src/routes/internal/mod.rs +++ b/apps/labrinth/src/routes/internal/mod.rs @@ -103,10 +103,26 @@ pub fn config(cfg: &mut web::ServiceConfig) { pats::create_pat, pats::edit_pat, pats::delete_pat, + + moderation::project_flag::project::get_project_flags, + moderation::project_flag::rule::list_runs, + moderation::project_flag::rule::get_run_activity, + moderation::project_flag::rule::list_run_results, + moderation::project_flag::rule::start_run, + moderation::project_flag::rule::delete_run, + moderation::project_flag::rule::eval_rule, + moderation::project_flag::rule::list_rules, + moderation::project_flag::rule::get_rule_schema, + moderation::project_flag::rule::get_rule, + moderation::project_flag::rule::create_rule, + moderation::project_flag::rule::update_rule, + moderation::project_flag::rule::delete_rule, + moderation::copyright_archives::list, moderation::copyright_archives::create, moderation::copyright_archives::replace, moderation::copyright_archives::delete, + moderation::get_projects, moderation::get_project_ids, moderation::get_project_meta, diff --git a/apps/labrinth/src/routes/internal/moderation/mod.rs b/apps/labrinth/src/routes/internal/moderation/mod.rs index 33e81c4b51d..2c365a33eb6 100644 --- a/apps/labrinth/src/routes/internal/moderation/mod.rs +++ b/apps/labrinth/src/routes/internal/moderation/mod.rs @@ -29,6 +29,7 @@ use xredis::RedisPool; pub mod copyright_archives; pub mod external_license; mod ownership; +pub mod project_flag; pub mod tech_review; pub mod user_restrictions; @@ -50,6 +51,10 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) { .service(get_users_project_by_status) .service(get_organization_project_by_status) .service(get_organizations_project_by_status) + .service( + utoipa_actix_web::scope::scope("/project-flag") + .configure(project_flag::config), + ) .service(web::scope("/tech-review").configure(tech_review::config)) .service( web::scope("/external-license").configure(external_license::config), diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs new file mode 100644 index 00000000000..cbf5d504433 --- /dev/null +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs @@ -0,0 +1,20 @@ +pub mod project; +pub mod rule; +pub mod run_response; +mod schema_example; + +pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig<'_>) { + cfg.service(rule::list_rules) + .service(rule::get_rule_schema) + .service(rule::list_runs) + .service(rule::get_run_activity) + .service(rule::list_run_results) + .service(rule::start_run) + .service(rule::delete_run) + .service(rule::get_rule) + .service(rule::create_rule) + .service(rule::update_rule) + .service(rule::delete_rule) + .service(rule::eval_rule) + .service(project::get_project_flags); +} diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs new file mode 100644 index 00000000000..d6ccb56f816 --- /dev/null +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs @@ -0,0 +1,49 @@ +use actix_web::{HttpRequest, get, web}; +use xredis::RedisPool; + +use crate::{ + auth::get_user_from_headers, + database::{ + ReadOnlyPgPool, + models::project_flag_item::{DBProjectFlag, ProjectFlag}, + }, + models::{ids::ProjectId, pats::Scopes}, + queue::session::AuthQueue, + routes::ApiError, + util::error::Context as _, +}; + +/// Fetch current and retained flags for a project. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + params(("id" = ProjectId, Path)), + responses((status = OK, body = [ProjectFlag]), (status = NOT_FOUND)) +)] +#[get("/project/{id}")] +pub async fn get_project_flags( + req: HttpRequest, + path: web::Path, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result>, ApiError> { + get_user_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_READ, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + let project_id = path.into_inner().into(); + + let flags = DBProjectFlag::get_project_flags(project_id, &ro_pool) + .await + .wrap_internal_err("fetching project flags")? + .wrap_not_found_err("project not found")?; + Ok(web::Json(flags)) +} diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs new file mode 100644 index 00000000000..dbfe98e264f --- /dev/null +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs @@ -0,0 +1,616 @@ +use actix_web::{HttpRequest, delete, get, post, put, web}; +use eyre::eyre; +use serde::Deserialize; +use validator::Validate; +use xredis::RedisPool; + +use crate::auth::get_user_from_headers; +use crate::database::models::project_flag_item::{ + DBProjectFlagRunResult, ProjectFlagRunResults, ProjectFlagRunStatus, + ProjectRuleParams, +}; +use crate::database::models::{ + DBProject, DBProjectFlagRule, DBProjectFlagRun, ProjectFlagRuleId, + ProjectFlagRunId, +}; +use crate::database::{PgPool, ReadOnlyPgPool}; +use crate::models::ids::ProjectId; +use crate::models::pats::Scopes; + +use crate::queue::moderation::project_flag::{ + FlagContext, ProjectFlagEngine, ProjectFlagEvaluation, +}; +use crate::queue::moderation::project_flag_run::ProjectFlagDispatchUncertain; +use crate::queue::session::AuthQueue; +use crate::routes::ApiError; +use crate::util::error::Context as _; +use crate::util::kafka::KafkaClientState; +use crate::util::validate::{validate_name, validation_errors_to_string}; + +use super::run_response::ProjectFlagRunResponse; +use super::schema_example; + +/// Get an example project flag rule context. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + responses((status = OK, body = FlagContext)) +)] +#[get("/rules/schema")] +pub async fn get_rule_schema( + req: HttpRequest, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + get_user_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_READ, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + Ok(web::Json( + schema_example::generate::() + .wrap_internal_err("generating project flag context example")?, + )) +} + +#[derive(Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListRules { + #[serde(default = "default_limit")] + pub limit: u16, + #[serde(default)] + pub offset: u32, +} + +fn default_limit() -> u16 { + 100 +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(ListRules), + responses((status = OK, body = [DBProjectFlagRule])) +)] +#[get("/rules")] +pub async fn list_rules( + req: HttpRequest, + query: web::Query, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result>, ApiError> { + get_user_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_READ, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if query.limit == 0 || query.limit > 200 { + return Err(ApiError::Request(eyre!( + "`limit` must be between 1 and 200" + ))); + } + let rules = DBProjectFlagRule::list( + i64::from(query.limit), + i64::from(query.offset), + &ro_pool, + ) + .await + .wrap_internal_err("listing project flag rules")?; + Ok(web::Json(rules)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(("id" = ProjectFlagRuleId, Path)), + responses((status = OK, body = DBProjectFlagRule), (status = NOT_FOUND)) +)] +#[get("/rules/{id}")] +pub async fn get_rule( + req: HttpRequest, + path: web::Path, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + get_user_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_READ, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + let rule = DBProjectFlagRule::get(path.into_inner(), &ro_pool) + .await + .wrap_internal_err("fetching project flag rule")? + .wrap_not_found_err("project flag rule not found")?; + Ok(web::Json(rule)) +} + +#[derive(Deserialize, Validate, utoipa::ToSchema)] +pub struct CreateRule { + #[validate( + length(min = 1, max = 1023), + custom(function = "validate_name") + )] + pub name: String, + #[validate(nested)] + pub params: ProjectRuleParams, +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + request_body = CreateRule, + responses((status = OK, body = DBProjectFlagRule)) +)] +#[post("/rules")] +pub async fn create_rule( + req: HttpRequest, + body: web::Json, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_WRITE, + ) + .await + .wrap_auth_err("authenticating moderator")?; + body.validate().map_err(|error| { + ApiError::Request(eyre!(validation_errors_to_string(error, None))) + })?; + + let rule = DBProjectFlagRule::create( + &body.name, + &body.params, + user.user.id.into(), + &pool, + ) + .await + .wrap_internal_err("creating project flag rule")?; + Ok(web::Json(rule)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(("id" = ProjectFlagRuleId, Path)), + request_body = CreateRule, + responses((status = OK, body = DBProjectFlagRule), (status = NOT_FOUND)) +)] +#[put("/rules/{id}")] +pub async fn update_rule( + req: HttpRequest, + path: web::Path, + body: web::Json, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_WRITE, + ) + .await + .wrap_auth_err("authenticating moderator")?; + body.validate().map_err(|error| { + ApiError::Request(eyre!(validation_errors_to_string(error, None))) + })?; + + let rule = DBProjectFlagRule::update( + path.into_inner(), + &body.name, + &body.params, + user.user.id.into(), + &pool, + ) + .await + .wrap_internal_err("updating project flag rule")? + .wrap_not_found_err("project flag rule not found")?; + Ok(web::Json(rule)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(("id" = ProjectFlagRuleId, Path)), + responses((status = NO_CONTENT), (status = NOT_FOUND)) +)] +#[delete("/rules/{id}")] +pub async fn delete_rule( + req: HttpRequest, + path: web::Path, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result<(), ApiError> { + get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_WRITE, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if !DBProjectFlagRule::delete(path.into_inner(), &pool) + .await + .wrap_internal_err("deleting project flag rule")? + { + return Err(ApiError::NotFound(eyre!("project flag rule not found"))); + } + Ok(()) +} + +#[derive(Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListRunResults { + #[serde(default = "default_results_limit")] + #[param(default = 25, minimum = 1, maximum = 200)] + pub limit: u16, + #[serde(default)] + #[param(default = 0)] + pub offset: u32, +} + +fn default_results_limit() -> u16 { + 25 +} + +/// List completed project evaluations for a project flag run. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + params(("id" = ProjectFlagRunId, Path), ListRunResults), + responses( + (status = OK, body = ProjectFlagRunResults), + (status = BAD_REQUEST), + (status = UNAUTHORIZED), + (status = FORBIDDEN), + (status = NOT_FOUND) + ) +)] +#[get("/rules/run/{id}/results")] +pub async fn list_run_results( + req: HttpRequest, + path: web::Path, + query: web::Query, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + get_user_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_READ, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if query.limit == 0 || query.limit > 200 { + return Err(ApiError::Request(eyre!( + "`limit` must be between 1 and 200" + ))); + } + let results = DBProjectFlagRunResult::list( + path.into_inner(), + i64::from(query.limit), + i64::from(query.offset), + &ro_pool, + ) + .await + .wrap_internal_err("listing project flag run results")? + .wrap_not_found_err("project flag run not found")?; + Ok(web::Json(results)) +} + +#[derive(serde::Serialize, utoipa::ToSchema)] +pub struct ProjectFlagRunActivity { + pub num_scanning: i64, + pub num_consumers: usize, +} + +/// Count unfinished projects and consumers assigned to the project scan topic. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + responses((status = OK, body = ProjectFlagRunActivity)) +)] +#[get("/rules/run/activity")] +pub async fn get_run_activity( + req: HttpRequest, + kafka: web::Data, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + get_user_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_READ, + ) + .await + .wrap_auth_err("authenticating moderator")?; + let (num_scanning, num_consumers) = tokio::try_join!( + DBProjectFlagRun::num_scanning(&ro_pool), + kafka.project_flag_scan_consumers(), + ) + .wrap_internal_err("fetching project flag scan activity")?; + Ok(web::Json(ProjectFlagRunActivity { + num_scanning, + num_consumers, + })) +} + +#[derive(Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListRuns { + #[serde(default = "default_limit")] + pub limit: u16, + #[serde(default)] + pub offset: u32, +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(ListRuns), + responses((status = OK, body = [ProjectFlagRunResponse])) +)] +#[get("/rules/run")] +pub async fn list_runs( + req: HttpRequest, + query: web::Query, + kafka: web::Data, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result>, ApiError> { + get_user_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_READ, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if query.limit == 0 || query.limit > 200 { + return Err(ApiError::Request(eyre!( + "`limit` must be between 1 and 200" + ))); + } + let runs = DBProjectFlagRun::list( + i64::from(query.limit), + i64::from(query.offset), + &ro_pool, + ) + .await + .wrap_internal_err("listing project flag runs")?; + let num_queued = if runs + .iter() + .any(|run| run.status == ProjectFlagRunStatus::Running) + { + kafka + .project_flag_scan_lag() + .await + .wrap_internal_err("fetching project flag scan lag")? + } else { + 0 + }; + Ok(web::Json( + runs.into_iter() + .map(|run| ProjectFlagRunResponse::from_run(run, num_queued)) + .collect(), + )) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + responses((status = OK, body = ProjectFlagRunResponse)) +)] +#[post("/rules/run")] +pub async fn start_run( + req: HttpRequest, + kafka: web::Data, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_WRITE, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + let rules = DBProjectFlagRule::all(&pool) + .await + .wrap_internal_err("snapshotting project flag rules")?; + let project_ids = DBProjectFlagRun::queued_project_ids(&pool) + .await + .wrap_internal_err("fetching queued project IDs")?; + let mut run = + DBProjectFlagRun::create(user.user.id.into(), &project_ids, &pool) + .await + .wrap_internal_err("creating project flag run")?; + + let response_kafka = kafka.clone(); + // Keep dispatch alive if the caller disconnects while Kafka is publishing. + let run = tokio::spawn(async move { + let empty = project_ids.is_empty(); + let result = crate::queue::moderation::project_flag_run::dispatch_project_flag_run( + run.id, rules, project_ids, &redis, &kafka, + ).await; + if let Err(error) = result { + if error.downcast_ref::().is_some() { + tracing::error!(run_id = %run.id.0, error = %error, "Project flag run dispatch outcome unknown; leaving run active"); + return Err(ApiError::Internal(error)); + } + run.finish(Some(format!("{error:#}")), &pool) + .await + .wrap_internal_err("recording project flag run dispatch failure")?; + return Err(ApiError::Internal(error)); + } + if empty { + run.finish(None, &pool) + .await + .wrap_internal_err("completing empty project flag run")?; + } + Ok(run) + }) + .await + .wrap_internal_err("joining project flag run dispatch")??; + let num_queued = if run.status == ProjectFlagRunStatus::Running { + response_kafka + .project_flag_scan_lag() + .await + .wrap_internal_err("fetching project flag scan lag")? + } else { + 0 + }; + Ok(web::Json(ProjectFlagRunResponse::from_run(run, num_queued))) +} + +/// Delete a run, its results, and its inactive flags, preserving active flags. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + params(("id" = ProjectFlagRunId, Path)), + responses((status = OK), (status = NOT_FOUND)) +)] +#[delete("/rules/run/{id}")] +pub async fn delete_run( + req: HttpRequest, + path: web::Path, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result<(), ApiError> { + get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_WRITE, + ) + .await + .wrap_auth_err("authenticating moderator")?; + let id = path.into_inner(); + if !DBProjectFlagRun::delete(id, &pool) + .await + .wrap_internal_err("deleting project flag run")? + { + return Err(ApiError::NotFound(eyre!("project flag run not found"))); + } + let cleanup = async { + let mut connection = redis.connect().await?; + let key = connection.key().entity("project_flag_run_rules:v1", id.0); + redis::cmd("DEL") + .arg(key) + .query_async::(&mut connection) + .await?; + eyre::Ok(()) + } + .await; + if let Err(error) = cleanup { + tracing::warn!(run_id = %id.0, ?error, "Failed to remove deleted run rule snapshot"); + } + Ok(()) +} + +#[derive(Deserialize, Validate, utoipa::ToSchema)] +pub struct EvalRule { + pub project_id: ProjectId, + #[validate(nested)] + pub params: ProjectRuleParams, +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + request_body = EvalRule, + responses( + (status = OK, body = ProjectFlagEvaluation), + (status = BAD_REQUEST), + (status = NOT_FOUND) + ) +)] +#[post("/rule/eval")] +pub async fn eval_rule( + req: HttpRequest, + body: web::Json, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, + engine: web::Data, +) -> Result, ApiError> { + get_user_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS | Scopes::MODERATION_READ, + ) + .await + .wrap_auth_err("authenticating moderator")?; + body.validate().map_err(|error| { + ApiError::Request(eyre!(validation_errors_to_string(error, None))) + })?; + let body = body.into_inner(); + let project_id = body.project_id.into(); + DBProject::get_id(project_id, &***ro_pool, &redis) + .await + .wrap_internal_err("fetching project for flag evaluation")? + .wrap_not_found_err("project not found")?; + let compile_engine = engine.clone(); + let call = tokio::task::spawn_blocking(move || { + compile_engine.make_call(&body.params) + }) + .await + .wrap_internal_err("joining project flag compilation")? + .wrap_request_err("compiling project flag expression")?; + let evaluation = engine + .scan_flags_with_log(&call, project_id) + .await + .wrap_request_err("evaluating project flag expression")?; + Ok(web::Json(evaluation)) +} diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/run_response.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/run_response.rs new file mode 100644 index 00000000000..71c647f40a5 --- /dev/null +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/run_response.rs @@ -0,0 +1,53 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::database::models::ProjectFlagRunId; +use crate::database::models::project_flag_item::{ + DBProjectFlagRun, ProjectFlagRunOutput, ProjectFlagRunStatus, + ProjectFlagRunUser, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum RunResponseStatus { + Done, + Running { num_queued: i64 }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProjectFlagRunResponse { + pub id: ProjectFlagRunId, + pub status: RunResponseStatus, + pub total_projects: i64, + pub num_errors: i64, + pub started_by: Option, + pub started_at: DateTime, + pub completed_at: Option>, + pub applied_at: Option>, + pub applied_by: Option, + pub output: Option, + pub error: Option, +} + +impl ProjectFlagRunResponse { + pub fn from_run(run: DBProjectFlagRun, num_queued: i64) -> Self { + Self { + id: run.id, + status: match run.status { + ProjectFlagRunStatus::Done => RunResponseStatus::Done, + ProjectFlagRunStatus::Running => { + RunResponseStatus::Running { num_queued } + } + }, + total_projects: run.total_projects, + num_errors: run.num_errors, + started_by: run.started_by, + started_at: run.started_at, + completed_at: run.completed_at, + applied_at: run.applied_at, + applied_by: run.applied_by, + output: run.output, + error: run.error, + } + } +} diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/schema_example.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/schema_example.rs new file mode 100644 index 00000000000..f7ed6a4bdca --- /dev/null +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/schema_example.rs @@ -0,0 +1,145 @@ +use eyre::{Context, Result}; +use serde_json::{Map, Value, json}; + +pub(super) fn generate() -> Result { + let schema = serde_json::to_value(T::schema()) + .wrap_err("serializing input schema")?; + let mut schemas = Vec::new(); + T::schemas(&mut schemas); + let components = schemas + .into_iter() + .map(|(name, schema)| { + serde_json::to_value(schema) + .map(|schema| (name, schema)) + .wrap_err("serializing component schema") + }) + .collect::>>()?; + Ok(example(&schema, &components, &mut Vec::new(), 0)) +} + +fn example( + schema: &Value, + components: &Map, + visited: &mut Vec, + depth: usize, +) -> Value { + if depth >= 16 { + return Value::Null; + } + if let Some(value) = [ + schema.get("example"), + schema + .get("examples") + .and_then(Value::as_array) + .and_then(|v| v.first()), + schema.get("default"), + schema.get("const"), + ] + .into_iter() + .flatten() + .next() + { + return value.clone(); + } + if let Some(reference) = schema.get("$ref").and_then(Value::as_str) { + let Some(name) = reference.strip_prefix("#/components/schemas/") else { + return Value::Null; + }; + let name = name.replace("~1", "/").replace("~0", "~"); + if visited.contains(&name) { + return Value::Null; + } + let Some(target) = components.get(&name) else { + return Value::Null; + }; + visited.push(name); + let value = example(target, components, visited, depth + 1); + visited.pop(); + return value; + } + if let Some(values) = schema.get("enum").and_then(Value::as_array) { + return values + .iter() + .find(|v| !v.is_null()) + .cloned() + .unwrap_or(Value::Null); + } + for key in ["oneOf", "anyOf"] { + if let Some(parts) = schema.get(key).and_then(Value::as_array) { + return parts + .iter() + .find(|part| part["type"] != "null") + .or_else(|| parts.first()) + .map(|part| example(part, components, visited, depth + 1)) + .unwrap_or(Value::Null); + } + } + if let Some(parts) = schema.get("allOf").and_then(Value::as_array) { + let mut result = Value::Null; + for part in parts { + let value = example(part, components, visited, depth + 1); + if let (Some(target), Some(source)) = + (result.as_object_mut(), value.as_object()) + { + target.extend(source.clone()); + } else if !value.is_null() { + result = value; + } + } + return result; + } + let kind = match schema.get("type") { + Some(Value::Array(types)) => { + types.iter().find(|t| *t != "null").and_then(Value::as_str) + } + value => value.and_then(Value::as_str), + }; + if kind == Some("object") + || schema.get("properties").is_some() + || schema.get("additionalProperties").is_some() + { + let mut object = Map::new(); + if let Some(properties) = + schema.get("properties").and_then(Value::as_object) + { + for (name, property) in properties { + object.insert( + name.clone(), + example(property, components, visited, depth + 1), + ); + } + } + if let Some(additional) = + schema.get("additionalProperties").filter(|v| v.is_object()) + { + object.entry("key").or_insert_with(|| { + example(additional, components, visited, depth + 1) + }); + } + return Value::Object(object); + } + match kind { + Some("array") => schema + .get("items") + .filter(|v| v.is_object()) + .map(|items| { + json!([example(items, components, visited, depth + 1)]) + }) + .unwrap_or_else(|| json!([])), + Some("integer" | "number") => schema + .get("minimum") + .filter(|v| v.is_number()) + .cloned() + .unwrap_or(json!(0)), + Some("boolean") => json!(false), + Some("string") => { + json!(match schema.get("format").and_then(Value::as_str) { + Some("date-time") => "2026-01-01T00:00:00Z", + Some("date") => "2026-01-01", + Some("uri" | "url") => "https://example.com/", + _ => "string", + }) + } + _ => Value::Null, + } +} diff --git a/apps/labrinth/src/test/mod.rs b/apps/labrinth/src/test/mod.rs index 3fab97260b4..8934b99eb8a 100644 --- a/apps/labrinth/src/test/mod.rs +++ b/apps/labrinth/src/test/mod.rs @@ -52,6 +52,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig { .expect("Failed to create Gotenberg client"); let kafka_client = actix_web::web::Data::new( crate::util::kafka::KafkaClientState::new() + .await .expect("Kafka connection failed"), ); diff --git a/apps/labrinth/src/util/kafka.rs b/apps/labrinth/src/util/kafka.rs index 05879e8727e..14d5a82154e 100644 --- a/apps/labrinth/src/util/kafka.rs +++ b/apps/labrinth/src/util/kafka.rs @@ -1,15 +1,19 @@ -use std::time::Duration; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; use crate::env::ENV; use chrono::{DateTime, Utc}; -use eyre::WrapErr; +use eyre::{WrapErr, ensure, eyre}; use rdkafka::{ - ClientConfig, - consumer::StreamConsumer, + ClientConfig, Offset, TopicPartitionList, + consumer::{Consumer, StreamConsumer}, message::{Header, OwnedHeaders}, - producer::FutureProducer, + producer::{FutureProducer, Producer}, }; use serde::Serialize; +use tokio::sync::Mutex; use uuid::Uuid; pub const KAFKA_OPERATION_INTERVAL: Duration = Duration::from_secs(5); @@ -18,20 +22,14 @@ pub const INCREMENTAL_INDEX_SEARCH_TASK: &str = "incremental-index-search"; pub struct KafkaClientState { pub client: FutureProducer, + pub project_flag_run_producer: Arc>>, pub incremental_index_search_consumer: StreamConsumer, + project_flag_lag_consumer: Arc, } impl KafkaClientState { - pub fn new() -> eyre::Result { - let client = ClientConfig::new() - .set("bootstrap.servers", ENV.KAFKA_BOOTSTRAP_SERVERS.0.join(",")) - .set("client.id", &ENV.KAFKA_CLIENT_ID) - .set("acks", "all") - .set( - "delivery.timeout.ms", - KAFKA_DELIVERY_TIMEOUT.as_millis().to_string(), - ) - .set("broker.address.family", "v4") + pub async fn new() -> eyre::Result { + let client = producer_config() .create() .wrap_err("failed to create Kafka producer")?; let incremental_index_search_consumer = create_consumer( @@ -45,24 +43,208 @@ impl KafkaClientState { "Connected to Kafka" ); + let project_flag_run_producer = + tokio::task::spawn_blocking(create_project_flag_run_producer) + .await + .wrap_err( + "joining project flag run producer initialization", + )??; + + let project_flag_lag_consumer = create_consumer( + crate::queue::moderation::project_flag_run::PROJECT_FLAG_RUN_CONSUMER_GROUP, + ).wrap_err("creating project flag lag client")?; + Ok(Self { + project_flag_lag_consumer: Arc::new(project_flag_lag_consumer), + project_flag_run_producer: Arc::new(Mutex::new(Some( + project_flag_run_producer, + ))), client, incremental_index_search_consumer, }) } + + /// Counts group members assigned partitions of the project scan topic. + pub async fn project_flag_scan_consumers(&self) -> eyre::Result { + let consumer = self.project_flag_lag_consumer.clone(); + tokio::task::spawn_blocking(move || { + use crate::queue::moderation::project_flag_run::{ + PROJECT_FLAG_RUN_CONSUMER_GROUP, PROJECT_FLAG_RUN_TOPIC, + }; + let groups = consumer + .fetch_group_list( + Some(PROJECT_FLAG_RUN_CONSUMER_GROUP), + KAFKA_OPERATION_INTERVAL, + ) + .wrap_err("fetching project flag consumer group members")?; + let mut count = 0; + for group in groups + .groups() + .iter() + .filter(|group| group.name() == PROJECT_FLAG_RUN_CONSUMER_GROUP) + { + if group.members().is_empty() { + continue; + } + ensure!( + group.protocol_type() == "consumer", + "unexpected project flag consumer group protocol" + ); + for member in group.members() { + if let Some(assignment) = member.assignment() + && assignment_has_topic( + assignment, + PROJECT_FLAG_RUN_TOPIC, + )? + { + count += 1; + } + } + } + Ok(count) + }) + .await + .wrap_err("joining project flag consumer group lookup")? + } + + /// Reads topic-wide group lag without subscribing, consuming, or committing offsets. + pub async fn project_flag_scan_lag(&self) -> eyre::Result { + let consumer = self.project_flag_lag_consumer.clone(); + tokio::task::spawn_blocking(move || { + let topic = crate::queue::moderation::project_flag_run::PROJECT_FLAG_RUN_TOPIC; + let deadline = Instant::now() + KAFKA_OPERATION_INTERVAL; + let remaining = || deadline.checked_duration_since(Instant::now()) + .ok_or_else(|| eyre!("timed out fetching project flag scan lag")); + let metadata = consumer.fetch_metadata(Some(topic), remaining()?) + .wrap_err("fetching project flag topic metadata")?; + let metadata = metadata.topics().iter().find(|metadata| metadata.name() == topic) + .ok_or_else(|| eyre!("project flag topic metadata missing"))?; + if let Some(error) = metadata.error() { + return Err(eyre!("project flag topic metadata error: {error:?}")); + } + let mut partitions = TopicPartitionList::new(); + for partition in metadata.partitions() { + partitions.add_partition(topic, partition.id()); + } + let committed = consumer.committed_offsets(partitions, remaining()?) + .wrap_err("fetching project flag consumer group offsets")?; + let mut lag = 0i64; + for partition in committed.elements() { + partition.error().wrap_err("fetching project flag partition offset")?; + let (low, high) = consumer.fetch_watermarks(topic, partition.partition(), remaining()?) + .wrap_err("fetching project flag partition watermarks")?; + let offset = match partition.offset() { + Offset::Offset(offset) => offset.max(low), + Offset::Invalid | Offset::Beginning => low, + offset => return Err(eyre!("unexpected project flag consumer offset: {offset:?}")), + }; + lag = lag.saturating_add(high.saturating_sub(offset).max(0)); + } + Ok(lag) + }).await.wrap_err("joining project flag scan lag lookup")? + } +} + +/// Kafka's classic ConsumerProtocolAssignment contains a version and topic/partition arrays. +fn assignment_has_topic( + mut assignment: &[u8], + topic: &str, +) -> eyre::Result { + if assignment.is_empty() { + return Ok(false); + } + let version = i16::from_be_bytes(take_assignment_bytes(&mut assignment)?); + ensure!( + version == 0, + "unsupported Kafka consumer assignment version: {version}" + ); + let topics = usize::try_from(i32::from_be_bytes(take_assignment_bytes( + &mut assignment, + )?)) + .wrap_err("decoding Kafka consumer assignment topic count")?; + let mut assigned = false; + for _ in 0..topics { + let name_length = usize::try_from(i16::from_be_bytes( + take_assignment_bytes(&mut assignment)?, + )) + .wrap_err("decoding Kafka assignment topic length")?; + let (name, rest) = assignment + .split_at_checked(name_length) + .ok_or_else(|| eyre!("truncated Kafka assignment topic"))?; + assignment = rest; + let partitions = usize::try_from(i32::from_be_bytes( + take_assignment_bytes(&mut assignment)?, + )) + .wrap_err("decoding Kafka assignment partition count")?; + let length = partitions.checked_mul(4).ok_or_else(|| { + eyre!("Kafka assignment partition count overflow") + })?; + let (_, rest) = assignment + .split_at_checked(length) + .ok_or_else(|| eyre!("truncated Kafka assignment partitions"))?; + assignment = rest; + assigned |= name == topic.as_bytes() && partitions > 0; + } + Ok(assigned) +} + +fn take_assignment_bytes( + input: &mut &[u8], +) -> eyre::Result<[u8; N]> { + let (bytes, rest) = input + .split_first_chunk::() + .ok_or_else(|| eyre!("truncated Kafka consumer assignment"))?; + *input = rest; + Ok(*bytes) +} + +pub fn create_project_flag_run_producer() -> eyre::Result { + let producer: FutureProducer = producer_config() + .set("enable.idempotence", "true") + .set( + "transactional.id", + format!("project-flag-run-{}", Uuid::now_v7()), + ) + .set("transaction.timeout.ms", "60000") + .create() + .wrap_err("creating project flag run Kafka producer")?; + producer + .init_transactions(KAFKA_DELIVERY_TIMEOUT) + .wrap_err("initializing project flag run Kafka transaction")?; + Ok(producer) +} + +pub fn producer_config() -> ClientConfig { + let mut config = ClientConfig::new(); + config + .set("bootstrap.servers", ENV.KAFKA_BOOTSTRAP_SERVERS.0.join(",")) + .set("client.id", &ENV.KAFKA_CLIENT_ID) + .set("acks", "all") + .set("broker.address.family", "v4") + .set( + "delivery.timeout.ms", + KAFKA_DELIVERY_TIMEOUT.as_millis().to_string(), + ); + config } pub fn create_consumer(group_id: &str) -> eyre::Result { - ClientConfig::new() + consumer_config(group_id) + .create() + .wrap_err("failed to create Kafka consumer") +} + +pub fn consumer_config(group_id: &str) -> ClientConfig { + let mut config = ClientConfig::new(); + config .set("bootstrap.servers", ENV.KAFKA_BOOTSTRAP_SERVERS.0.join(",")) .set("client.id", &ENV.KAFKA_CLIENT_ID) .set("group.id", group_id) .set("enable.auto.commit", "false") .set("auto.offset.reset", "earliest") .set("max.poll.interval.ms", "1800000") - .set("broker.address.family", "v4") - .create() - .wrap_err("failed to create Kafka consumer") + .set("broker.address.family", "v4"); + config } /// Sentry distributed tracing headers for the active span, so consumers can diff --git a/packages/api-client/src/modules/index.ts b/packages/api-client/src/modules/index.ts index 6c977760ec1..f2eaa87cb53 100644 --- a/packages/api-client/src/modules/index.ts +++ b/packages/api-client/src/modules/index.ts @@ -48,6 +48,7 @@ import { LabrinthOrganizationsV3Module } from './labrinth/organizations/v3' import { LabrinthPatsV2Module } from './labrinth/pats/v2' import { LabrinthPayoutV3Module } from './labrinth/payout/v3' import { LabrinthPayoutsV3Module } from './labrinth/payouts/v3' +import { LabrinthProjectFlagsInternalModule } from './labrinth/project-flags/internal' import { LabrinthProjectsV2Module } from './labrinth/projects/v2' import { LabrinthProjectsV3Module } from './labrinth/projects/v3' import { LabrinthReportsV3Module } from './labrinth/reports/v3' @@ -133,6 +134,7 @@ export const MODULE_REGISTRY = { labrinth_limits_v3: LabrinthLimitsV3Module, labrinth_payout_v3: LabrinthPayoutV3Module, labrinth_payouts_v3: LabrinthPayoutsV3Module, + labrinth_project_flags_internal: LabrinthProjectFlagsInternalModule, labrinth_projects_v2: LabrinthProjectsV2Module, labrinth_projects_v3: LabrinthProjectsV3Module, labrinth_reports_v3: LabrinthReportsV3Module, diff --git a/packages/api-client/src/modules/labrinth/project-flags/internal.ts b/packages/api-client/src/modules/labrinth/project-flags/internal.ts new file mode 100644 index 00000000000..7f1a9bff5c7 --- /dev/null +++ b/packages/api-client/src/modules/labrinth/project-flags/internal.ts @@ -0,0 +1,134 @@ +import { AbstractModule } from '../../../core/abstract-module' +import type { + CreateProjectFlagRuleRequest, + EvalProjectFlagRuleRequest, + ListProjectFlagRulesParams, + ListProjectFlagRunResultsParams, + ListProjectFlagRunsParams, + ProjectFlagEvaluation, + ProjectFlagRule, + ProjectFlagRuleSchemaResponse, + ProjectFlagRun, + ProjectFlagRunActivity, + ProjectFlagRunResults, +} from './types' + +export class LabrinthProjectFlagsInternalModule extends AbstractModule { + public getModuleID(): string { + return 'labrinth_project_flags_internal' + } + + public async getRuleSchema(): Promise { + return this.client.request( + '/moderation/project-flag/rules/schema', + { + api: 'labrinth', + version: 'internal', + method: 'GET', + }, + ) + } + + public async list(params: ListProjectFlagRulesParams = {}): Promise { + return this.client.request('/moderation/project-flag/rules', { + api: 'labrinth', + version: 'internal', + method: 'GET', + params, + }) + } + + public async listRuns(params: ListProjectFlagRunsParams = {}): Promise { + return this.client.request('/moderation/project-flag/rules/run', { + api: 'labrinth', + version: 'internal', + method: 'GET', + params, + }) + } + + public async getRunActivity(): Promise { + return this.client.request( + '/moderation/project-flag/rules/run/activity', + { + api: 'labrinth', + version: 'internal', + method: 'GET', + }, + ) + } + + public async startRun(): Promise { + return this.client.request('/moderation/project-flag/rules/run', { + api: 'labrinth', + version: 'internal', + method: 'POST', + retry: false, + }) + } + + public async deleteRun(id: string): Promise { + return this.client.request( + `/moderation/project-flag/rules/run/${encodeURIComponent(id)}`, + { api: 'labrinth', version: 'internal', method: 'DELETE', retry: false }, + ) + } + + public async getRunResults( + id: string, + params: ListProjectFlagRunResultsParams = {}, + ): Promise { + return this.client.request( + `/moderation/project-flag/rules/run/${encodeURIComponent(id)}/results`, + { + api: 'labrinth', + version: 'internal', + method: 'GET', + params, + }, + ) + } + + public async get(id: number): Promise { + return this.client.request(`/moderation/project-flag/rules/${id}`, { + api: 'labrinth', + version: 'internal', + method: 'GET', + }) + } + + public async create(data: CreateProjectFlagRuleRequest): Promise { + return this.client.request('/moderation/project-flag/rules', { + api: 'labrinth', + version: 'internal', + method: 'POST', + body: data, + }) + } + + public async update(id: number, data: CreateProjectFlagRuleRequest): Promise { + return this.client.request(`/moderation/project-flag/rules/${id}`, { + api: 'labrinth', + version: 'internal', + method: 'PUT', + body: data, + }) + } + + public async evalRule(data: EvalProjectFlagRuleRequest): Promise { + return this.client.request('/moderation/project-flag/rule/eval', { + api: 'labrinth', + version: 'internal', + method: 'POST', + body: data, + }) + } + + public async delete(id: number): Promise { + return this.client.request(`/moderation/project-flag/rules/${id}`, { + api: 'labrinth', + version: 'internal', + method: 'DELETE', + }) + } +} diff --git a/packages/api-client/src/modules/labrinth/project-flags/types.ts b/packages/api-client/src/modules/labrinth/project-flags/types.ts new file mode 100644 index 00000000000..60aab7f3174 --- /dev/null +++ b/packages/api-client/src/modules/labrinth/project-flags/types.ts @@ -0,0 +1,102 @@ +export interface ProjectFlagRuleParams { + expression: string + scan_files_for: Record +} + +export interface ProjectFlagRule { + id: number + name: string + created_by: number | null + created_at: string + updated_by: number | null + updated_at: string + params: ProjectFlagRuleParams +} + +export type ListProjectFlagRulesParams = { + limit?: number + offset?: number +} + +export interface CreateProjectFlagRuleRequest { + name: string + params: Omit & { + scan_files_for?: Record + } +} + +export interface EvalProjectFlagRuleRequest { + project_id: string + params: CreateProjectFlagRuleRequest['params'] +} + +export type ProjectFlagRuleSchemaResponse = Record + +export interface ProjectFlagEvaluation { + flags: ProjectFlag[] + log: string[] +} + +export interface ProjectFlag { + key: string + retain_if_inactive: boolean + [field: string]: ProjectFlagValue +} + +export interface ProjectFlagRunActivity { + num_scanning: number + num_consumers: number +} + +export interface ProjectFlagRunUser { + id: string + username: string +} + +export interface ProjectFlagRun { + id: string + status: 'done' | { running: { num_queued: number } } + total_projects: number + num_errors: number + started_by: ProjectFlagRunUser | null + started_at: string + completed_at: string | null + applied_at: string | null + applied_by: ProjectFlagRunUser | null + output: Record | null + error: string | null +} + +export interface ProjectFlagRuleResult { + flags: ProjectFlag[] + log: string[] + error: string | null +} + +export interface ProjectFlagRunResult { + project: { + id: string + title: string + slug: string | null + icon_url: string | null + } + rules: Record + error: string | null + evaluated_at: string +} + +export interface ProjectFlagRunResults { + results: ProjectFlagRunResult[] + total: number +} + +export type ListProjectFlagRunsParams = ListProjectFlagRulesParams +export type ListProjectFlagRunResultsParams = ListProjectFlagRulesParams + +export type ProjectFlagValue = + | null + | boolean + | number + | string + | ProjectFlagValue[] + | { [key: string]: ProjectFlagValue } diff --git a/packages/api-client/src/modules/types.ts b/packages/api-client/src/modules/types.ts index bd25e5f707c..5a9aeba0b57 100644 --- a/packages/api-client/src/modules/types.ts +++ b/packages/api-client/src/modules/types.ts @@ -1,6 +1,7 @@ export * from './archon/types' export * from './iso3166/types' export * from './kyros/types' +export * from './labrinth/project-flags/types' export * from './labrinth/types' export * from './launcher-meta/types' export * from './mclogs/types'