From 393aca35b73fe93adac9f9ba1290d967f862e5e9 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:23:22 +0000 Subject: [PATCH 01/26] flag rules backend model --- .editorconfig | 2 +- .../20261028145153_flag_rules_backend.sql | 25 ++++++++++ apps/labrinth/src/database/models/ids.rs | 3 ++ apps/labrinth/src/database/models/mod.rs | 5 ++ .../database/models/moderation_flag_item.rs | 48 +++++++++++++++++++ 5 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 apps/labrinth/migrations/20261028145153_flag_rules_backend.sql create mode 100644 apps/labrinth/src/database/models/moderation_flag_item.rs diff --git a/.editorconfig b/.editorconfig index 0216e875f8b..8d56d24b8f9 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,7 +17,7 @@ max_line_length = off indent_size = 2 # YAML requires space indentation by spec -[*.{yml,yaml}] +[*.{yml,yaml,sql}] indent_size = 2 indent_style = space diff --git a/apps/labrinth/migrations/20261028145153_flag_rules_backend.sql b/apps/labrinth/migrations/20261028145153_flag_rules_backend.sql new file mode 100644 index 00000000000..82d55b5ec74 --- /dev/null +++ b/apps/labrinth/migrations/20261028145153_flag_rules_backend.sql @@ -0,0 +1,25 @@ +CREATE TABLE moderation_flag_rules ( + id BIGINT NOT NULL, + name VARCHAR(1023) NOT NULL, + created_by BIGINT REFERENCES users(id) ON DELETE UPDATE, + created_at TIMESTAMPTZ NOT NULL, + updated_by BIGINT REFERENCES users(id) ON DELETE UPDATE, + updated_at TIMESTAMPTZ NOT NULL, + expression VARCHAR(65535) NOT NULL +); + +CREATE TABLE moderation_flags ( + id BIGINT NOT NULL, + rule_id BIGINT REFERENCES moderation_flag_rules(id) ON DELETE UPDATE, + what JSONB NOT NULL +); + +CREATE TABLE moderation_flag_runs ( + id BIGINT NOT NULL, + status TEXT NOT NULL, + started_by BIGINT REFERENCES users(id) ON DELETE UPDATE, + started_at TIMESTAMPTZ NOT NULL, + completed_at TIMESTAMPTZ, + output JSONB, + error TEXT +); diff --git a/apps/labrinth/src/database/models/ids.rs b/apps/labrinth/src/database/models/ids.rs index 99e6a16466f..49b174a0a86 100644 --- a/apps/labrinth/src/database/models/ids.rs +++ b/apps/labrinth/src/database/models/ids.rs @@ -304,3 +304,6 @@ id_type!(DelphiReportIssueId as i64); id_type!(DelphiReportIssueDetailsId as i64); id_type!(DelphiRuleId as i64); id_type!(SharedInstanceId as i64); +id_type!(ModerationFlagRuleId as i64); +id_type!(ModerationFlagId as i64); +id_type!(ModerationFlagRunId as i64); diff --git a/apps/labrinth/src/database/models/mod.rs b/apps/labrinth/src/database/models/mod.rs index 850ab584939..c0b5a974280 100644 --- a/apps/labrinth/src/database/models/mod.rs +++ b/apps/labrinth/src/database/models/mod.rs @@ -11,6 +11,7 @@ pub mod image_item; pub mod legacy_loader_fields; pub mod loader_fields; pub mod moderation_external_item; +pub mod moderation_flag_item; pub mod moderation_lock_item; pub mod moderation_note_item; pub mod notification_item; @@ -62,5 +63,9 @@ pub use thread_item::{DBThread, DBThreadMessage}; pub use user_item::DBUser; pub use version_item::DBVersion; +pub use moderation_flag_item::{ + DBModerationFlag, DBModerationFlagRule, DBModerationFlagRun, FlagRunOutput, + FlagRunStatus, +}; pub use moderation_lock_item::{DBModerationLock, ModerationLockWithUser}; pub use moderation_note_item::DBModerationNote; diff --git a/apps/labrinth/src/database/models/moderation_flag_item.rs b/apps/labrinth/src/database/models/moderation_flag_item.rs new file mode 100644 index 00000000000..3233fbbb1e3 --- /dev/null +++ b/apps/labrinth/src/database/models/moderation_flag_item.rs @@ -0,0 +1,48 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use super::{ + DBUserId, ModerationFlagId, ModerationFlagRuleId, ModerationFlagRunId, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DBModerationFlagRule { + pub id: ModerationFlagRuleId, + pub name: String, + pub created_by: Option, + pub created_at: DateTime, + pub updated_by: Option, + pub updated_at: DateTime, + pub expression: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DBModerationFlag { + pub id: ModerationFlagId, + pub rule_id: Option, + pub what: serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DBModerationFlagRun { + pub id: ModerationFlagRunId, + pub status: FlagRunStatus, + pub started_by: Option, + pub started_at: DateTime, + pub completed_at: Option>, + pub output: Option, +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, sqlx::Type, +)] +#[serde(rename_all = "snake_case")] +#[sqlx(type_name = "text", rename_all = "snake_case")] +pub enum FlagRunStatus { + Running, + Ok, + Err, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FlagRunOutput {} From 4bed0f520eb6aefc1390fb8edf0fb93fcaf0bf2a Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:30:16 +0000 Subject: [PATCH 02/26] project flag routes --- Cargo.lock | 12 + Cargo.toml | 1 + apps/labrinth/Cargo.toml | 1 + .../20261028145153_flag_rules_backend.sql | 14 +- .../database/models/moderation_flag_item.rs | 158 +++++++++++- apps/labrinth/src/routes/internal/mod.rs | 5 + .../src/routes/internal/moderation/mod.rs | 5 + .../internal/moderation/project_flag/mod.rs | 231 ++++++++++++++++++ 8 files changed, 419 insertions(+), 8 deletions(-) create mode 100644 apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs diff --git a/Cargo.lock b/Cargo.lock index e5f309744cf..3244ebeed4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6808,6 +6808,7 @@ dependencies = [ "url", "urlencoding", "utoipa", + "utoipa-actix-web", "uuid 1.28.0", "validator", "webauthn-rs", @@ -15429,6 +15430,17 @@ dependencies = [ "utoipa-gen", ] +[[package]] +name = "utoipa-actix-web" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3af2eb471d6791cfead324a893342eb73530d48bd37c485dd6296bc9fd8101ed" +dependencies = [ + "actix-service", + "actix-web", + "utoipa", +] + [[package]] name = "utoipa-gen" version = "6.0.1" diff --git a/Cargo.toml b/Cargo.toml index 8f86595c711..a07d72f57b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -255,6 +255,7 @@ unicode-segmentation = "1.13.3" url = "2.5.8" urlencoding = "2.1.3" utoipa = { version = "6.0.0", features = ["actix_extras", "chrono", "decimal"] } +utoipa-actix-web = "0.2.0" uuid = "1.28.0" validator = "0.21.0" webauthn-rs = "0.5.5" diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 7169b8c6007..a630e497c20 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -132,6 +132,7 @@ unicode-segmentation = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } utoipa = { workspace = true, features = ["url"] } +utoipa-actix-web = { workspace = true } uuid = { workspace = true, features = ["fast-rng", "serde", "v4", "v7"] } validator = { workspace = true, features = ["derive"] } webauthn-rs = { workspace = true, features = [ diff --git a/apps/labrinth/migrations/20261028145153_flag_rules_backend.sql b/apps/labrinth/migrations/20261028145153_flag_rules_backend.sql index 82d55b5ec74..2326c9d5de6 100644 --- a/apps/labrinth/migrations/20261028145153_flag_rules_backend.sql +++ b/apps/labrinth/migrations/20261028145153_flag_rules_backend.sql @@ -1,23 +1,23 @@ CREATE TABLE moderation_flag_rules ( - id BIGINT NOT NULL, + id BIGINT PRIMARY KEY, name VARCHAR(1023) NOT NULL, - created_by BIGINT REFERENCES users(id) ON DELETE UPDATE, + created_by BIGINT REFERENCES users(id) ON DELETE SET NULL, created_at TIMESTAMPTZ NOT NULL, - updated_by BIGINT REFERENCES users(id) ON DELETE UPDATE, + updated_by BIGINT REFERENCES users(id) ON DELETE SET NULL, updated_at TIMESTAMPTZ NOT NULL, expression VARCHAR(65535) NOT NULL ); CREATE TABLE moderation_flags ( - id BIGINT NOT NULL, - rule_id BIGINT REFERENCES moderation_flag_rules(id) ON DELETE UPDATE, + id BIGINT PRIMARY KEY, + rule_id BIGINT REFERENCES moderation_flag_rules(id) ON DELETE SET NULL, what JSONB NOT NULL ); CREATE TABLE moderation_flag_runs ( - id BIGINT NOT NULL, + id BIGINT PRIMARY KEY, status TEXT NOT NULL, - started_by BIGINT REFERENCES users(id) ON DELETE UPDATE, + started_by BIGINT REFERENCES users(id) ON DELETE SET NULL, started_at TIMESTAMPTZ NOT NULL, completed_at TIMESTAMPTZ, output JSONB, diff --git a/apps/labrinth/src/database/models/moderation_flag_item.rs b/apps/labrinth/src/database/models/moderation_flag_item.rs index 3233fbbb1e3..7b76fedf324 100644 --- a/apps/labrinth/src/database/models/moderation_flag_item.rs +++ b/apps/labrinth/src/database/models/moderation_flag_item.rs @@ -1,11 +1,15 @@ +use ariadne::ids::random_base62; use chrono::{DateTime, Utc}; +use eyre::{Result, WrapErr}; + +use crate::database::{PgPool, ReadOnlyPgPool}; use serde::{Deserialize, Serialize}; use super::{ DBUserId, ModerationFlagId, ModerationFlagRuleId, ModerationFlagRunId, }; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] pub struct DBModerationFlagRule { pub id: ModerationFlagRuleId, pub name: String, @@ -16,6 +20,158 @@ pub struct DBModerationFlagRule { pub expression: String, } +impl DBModerationFlagRule { + pub async fn list( + limit: i64, + offset: i64, + pool: &ReadOnlyPgPool, + ) -> Result> { + let rows = sqlx::query!( + r#" + SELECT id, name, expression, created_by, created_at, updated_by, updated_at + FROM moderation_flag_rules + ORDER BY created_at DESC, id DESC + LIMIT $1 OFFSET $2 + "#, + limit, + offset, + ) + .fetch_all(&**pool) + .await + .wrap_err("listing moderation flag rules")?; + + Ok(rows + .into_iter() + .map(|row| Self { + id: ModerationFlagRuleId(row.id), + name: row.name, + expression: row.expression, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + }) + .collect()) + } + + pub async fn get( + id: ModerationFlagRuleId, + pool: &ReadOnlyPgPool, + ) -> Result> { + let row = sqlx::query!( + r#" + SELECT id, name, expression, created_by, created_at, updated_by, updated_at + FROM moderation_flag_rules + WHERE id = $1 + "#, + id.0, + ) + .fetch_optional(&**pool) + .await + .wrap_err("fetching moderation flag rule")?; + + Ok(row.map(|row| Self { + id: ModerationFlagRuleId(row.id), + name: row.name, + expression: row.expression, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + })) + } + + pub async fn create( + name: &str, + expression: &str, + user_id: DBUserId, + pool: &PgPool, + ) -> Result { + let id = random_base62(8) as i64; + let now = Utc::now(); + let row = sqlx::query!( + r#" + INSERT INTO moderation_flag_rules + (id, name, expression, created_by, created_at, updated_by, updated_at) + VALUES ($1, $2, $3, $4, $5, $4, $5) + RETURNING id, name, expression, created_by, created_at, updated_by, updated_at + "#, + id, + name, + expression, + user_id.0, + now, + ) + .fetch_one(pool) + .await + .wrap_err("creating moderation flag rule")?; + + Ok(Self { + id: ModerationFlagRuleId(row.id), + name: row.name, + expression: row.expression, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + }) + } + + pub async fn update( + id: ModerationFlagRuleId, + name: &str, + expression: &str, + user_id: DBUserId, + pool: &PgPool, + ) -> Result> { + let now = Utc::now(); + let row = sqlx::query!( + r#" + UPDATE moderation_flag_rules + SET name = $2, expression = $3, updated_by = $4, updated_at = $5 + WHERE id = $1 + RETURNING id, name, expression, created_by, created_at, updated_by, updated_at + "#, + id.0, + name, + expression, + user_id.0, + now, + ) + .fetch_optional(pool) + .await + .wrap_err("updating moderation flag rule")?; + + Ok(row.map(|row| Self { + id: ModerationFlagRuleId(row.id), + name: row.name, + expression: row.expression, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + })) + } + + pub async fn delete( + id: ModerationFlagRuleId, + pool: &PgPool, + ) -> Result { + let result = sqlx::query!( + r#" + DELETE FROM moderation_flag_rules + WHERE id = $1 + "#, + id.0, + ) + .execute(pool) + .await + .wrap_err("deleting moderation flag rule")?; + + Ok(result.rows_affected() > 0) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DBModerationFlag { pub id: ModerationFlagId, diff --git a/apps/labrinth/src/routes/internal/mod.rs b/apps/labrinth/src/routes/internal/mod.rs index 7be92f76e78..3a8ff289dc2 100644 --- a/apps/labrinth/src/routes/internal/mod.rs +++ b/apps/labrinth/src/routes/internal/mod.rs @@ -103,6 +103,11 @@ pub fn config(cfg: &mut web::ServiceConfig) { pats::create_pat, pats::edit_pat, pats::delete_pat, + moderation::project_flag::list_rules, + moderation::project_flag::get_rule, + moderation::project_flag::create_rule, + moderation::project_flag::update_rule, + moderation::project_flag::delete_rule, moderation::get_projects, moderation::get_project_ids, moderation::get_project_meta, diff --git a/apps/labrinth/src/routes/internal/moderation/mod.rs b/apps/labrinth/src/routes/internal/moderation/mod.rs index 535f3796879..1f016e5799a 100644 --- a/apps/labrinth/src/routes/internal/moderation/mod.rs +++ b/apps/labrinth/src/routes/internal/moderation/mod.rs @@ -29,6 +29,7 @@ use xredis::RedisPool; pub mod external_license; mod ownership; +pub mod project_flag; pub mod tech_review; pub mod user_restrictions; @@ -48,6 +49,10 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) { .service(get_users_project_by_status) .service(get_organization_project_by_status) .service(get_organizations_project_by_status) + .service( + utoipa_actix_web::scope::scope("/project-flag") + .configure(project_flag::config), + ) .service(web::scope("/tech-review").configure(tech_review::config)) .service( web::scope("/external-license").configure(external_license::config), diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs new file mode 100644 index 00000000000..8d0048402f9 --- /dev/null +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs @@ -0,0 +1,231 @@ +use actix_web::{HttpRequest, delete, get, post, put, web}; +use eyre::eyre; +use serde::Deserialize; +use validator::Validate; +use xredis::RedisPool; + +use crate::auth::check_is_moderator_from_headers; +use crate::database::models::{DBModerationFlagRule, ModerationFlagRuleId}; +use crate::database::{PgPool, ReadOnlyPgPool}; +use crate::models::pats::Scopes; +use crate::queue::session::AuthQueue; +use crate::routes::ApiError; +use crate::util::error::Context as _; +use crate::util::validate::{validate_name, validation_errors_to_string}; + +pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig<'_>) { + cfg.service(list_rules) + .service(get_rule) + .service(create_rule) + .service(update_rule) + .service(delete_rule); +} + +#[derive(Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListRules { + #[serde(default = "default_limit")] + pub limit: u16, + #[serde(default)] + pub offset: u32, +} + +fn default_limit() -> u16 { + 100 +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(ListRules), + responses((status = OK, body = [DBModerationFlagRule])) +)] +#[get("/rules")] +pub async fn list_rules( + req: HttpRequest, + query: web::Query, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result>, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if query.limit == 0 || query.limit > 200 { + return Err(ApiError::Request(eyre!( + "`limit` must be between 1 and 200" + ))); + } + let rules = DBModerationFlagRule::list( + i64::from(query.limit), + i64::from(query.offset), + &ro_pool, + ) + .await + .wrap_internal_err("listing moderation flag rules")?; + Ok(web::Json(rules)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(("id" = ModerationFlagRuleId, Path)), + responses((status = OK, body = DBModerationFlagRule), (status = NOT_FOUND)) +)] +#[get("/rules/{id}")] +pub async fn get_rule( + req: HttpRequest, + path: web::Path, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + let rule = DBModerationFlagRule::get(path.into_inner(), &ro_pool) + .await + .wrap_internal_err("fetching moderation flag rule")? + .wrap_not_found_err("moderation flag rule not found")?; + Ok(web::Json(rule)) +} + +#[derive(Deserialize, Validate, utoipa::ToSchema)] +pub struct CreateRule { + #[validate( + length(min = 1, max = 1023), + custom(function = "validate_name") + )] + pub name: String, + #[validate(length(min = 1, max = 65535))] + pub expression: String, +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + request_body = CreateRule, + responses((status = OK, body = DBModerationFlagRule)) +)] +#[post("/rules")] +pub async fn create_rule( + req: HttpRequest, + body: web::Json, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + body.validate().map_err(|error| { + ApiError::Request(eyre!(validation_errors_to_string(error, None))) + })?; + + let rule = DBModerationFlagRule::create( + &body.name, + &body.expression, + user.id.into(), + &pool, + ) + .await + .wrap_internal_err("creating moderation flag rule")?; + Ok(web::Json(rule)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(("id" = ModerationFlagRuleId, Path)), + request_body = CreateRule, + responses((status = OK, body = DBModerationFlagRule), (status = NOT_FOUND)) +)] +#[put("/rules/{id}")] +pub async fn update_rule( + req: HttpRequest, + path: web::Path, + body: web::Json, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + body.validate().map_err(|error| { + ApiError::Request(eyre!(validation_errors_to_string(error, None))) + })?; + + let rule = DBModerationFlagRule::update( + path.into_inner(), + &body.name, + &body.expression, + user.id.into(), + &pool, + ) + .await + .wrap_internal_err("updating moderation flag rule")? + .wrap_not_found_err("moderation flag rule not found")?; + Ok(web::Json(rule)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(("id" = ModerationFlagRuleId, Path)), + responses((status = NO_CONTENT), (status = NOT_FOUND)) +)] +#[delete("/rules/{id}")] +pub async fn delete_rule( + req: HttpRequest, + path: web::Path, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result<(), ApiError> { + check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if !DBModerationFlagRule::delete(path.into_inner(), &pool) + .await + .wrap_internal_err("deleting moderation flag rule")? + { + return Err(ApiError::NotFound(eyre!( + "moderation flag rule not found" + ))); + } + Ok(()) +} From 687352006b1d956782a30039b2c4cb7470ed3443 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:12:10 +0000 Subject: [PATCH 03/26] add rule editor frontend and rhai eval --- Cargo.lock | 81 +++- Cargo.toml | 1 + .../project-flag/ProjectFlagRuleTester.vue | 110 ++++++ .../project-flag/RhaiExpressionEditor.vue | 75 ++++ .../ui/moderation/project-flag/ace-rhai.ts | 154 ++++++++ apps/frontend/src/pages/moderation.vue | 6 + .../src/pages/moderation/project-flag.vue | 346 ++++++++++++++++++ apps/labrinth/Cargo.toml | 1 + ... 20261028145153_project_flags_backend.sql} | 8 +- apps/labrinth/src/database/models/ids.rs | 6 +- apps/labrinth/src/database/models/mod.rs | 10 +- ...tion_flag_item.rs => project_flag_item.rs} | 117 ++++-- apps/labrinth/src/env.rs | 4 + apps/labrinth/src/lib.rs | 7 + .../{moderation.rs => moderation/mod.rs} | 2 + .../src/queue/moderation/project_flag.rs | 109 ++++++ apps/labrinth/src/routes/internal/mod.rs | 2 + .../internal/moderation/project_flag/mod.rs | 177 +++++++-- packages/api-client/src/modules/index.ts | 2 + .../labrinth/project-flags/internal.ts | 66 ++++ .../modules/labrinth/project-flags/types.ts | 37 ++ packages/ui/src/utils/ace-theme.ts | 2 +- 22 files changed, 1240 insertions(+), 83 deletions(-) create mode 100644 apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue create mode 100644 apps/frontend/src/components/ui/moderation/project-flag/RhaiExpressionEditor.vue create mode 100644 apps/frontend/src/components/ui/moderation/project-flag/ace-rhai.ts create mode 100644 apps/frontend/src/pages/moderation/project-flag.vue rename apps/labrinth/migrations/{20261028145153_flag_rules_backend.sql => 20261028145153_project_flags_backend.sql} (78%) rename apps/labrinth/src/database/models/{moderation_flag_item.rs => project_flag_item.rs} (62%) rename apps/labrinth/src/queue/{moderation.rs => moderation/mod.rs} (99%) create mode 100644 apps/labrinth/src/queue/moderation/project_flag.rs create mode 100644 packages/api-client/src/modules/labrinth/project-flags/internal.ts create mode 100644 packages/api-client/src/modules/labrinth/project-flags/types.ts diff --git a/Cargo.lock b/Cargo.lock index 3244ebeed4e..e9a5d022447 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -396,6 +396,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -2878,6 +2879,20 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "serde", + "static_assertions", + "zmij", +] + [[package]] name = "component-derive" version = "0.0.0" @@ -6777,6 +6792,7 @@ dependencies = [ "redis", "regex", "reqwest 0.13.5", + "rhai", "rust_decimal", "rust_iso3166", "rustls 0.23.46", @@ -8492,6 +8508,15 @@ dependencies = [ "libc", ] +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +dependencies = [ + "spin 0.5.2", +] + [[package]] name = "no_std_io2" version = "0.9.4" @@ -11111,6 +11136,37 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "rhai" +version = "1.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3925b784c0bee6542637e425ed7511a63257bf11b7ecf51ffd845e2d24d9163" +dependencies = [ + "ahash", + "bitflags 2.13.2", + "compact_str 0.10.0", + "no-std-compat", + "num-traits", + "once_cell", + "rhai_codegen", + "serde", + "smallvec", + "smartstring", + "thin-vec", + "web-time", +] + +[[package]] +name = "rhai_codegen" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d9661edb4a4b8145b017582d634c5eb88ff1a5284a5aa701f83a28f0657a826" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "ring" version = "0.17.14" @@ -11534,7 +11590,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826e80413b9a35e9d33217b3dcac04cf95f6559d15944b93887a08be5496c4a4" dependencies = [ - "compact_str", + "compact_str 0.7.1", ] [[package]] @@ -12428,6 +12484,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "smol_str" version = "0.1.24" @@ -12520,6 +12587,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.9" @@ -13902,6 +13975,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" diff --git a/Cargo.toml b/Cargo.toml index a07d72f57b5..a19e66cd568 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -175,6 +175,7 @@ redis = "1.7.1" regex = "1.13.1" reqwest = { version = "0.13.5", default-features = false, features = ["form", "query"] } rgb = "0.8.53" +rhai = "1.27.0" rust_decimal = { version = "1.43.0", features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = "0.2.0" rust-s3 = { version = "0.38.0", default-features = false, features = [ diff --git a/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue new file mode 100644 index 00000000000..195a6661c93 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue @@ -0,0 +1,110 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/project-flag/RhaiExpressionEditor.vue b/apps/frontend/src/components/ui/moderation/project-flag/RhaiExpressionEditor.vue new file mode 100644 index 00000000000..867f6490b6d --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/RhaiExpressionEditor.vue @@ -0,0 +1,75 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/project-flag/ace-rhai.ts b/apps/frontend/src/components/ui/moderation/project-flag/ace-rhai.ts new file mode 100644 index 00000000000..692e05d5519 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/ace-rhai.ts @@ -0,0 +1,154 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import 'ace-builds/src-noconflict/mode-rust' + +import ace from 'ace-builds' +import type { Ace } from 'ace-builds' + +const RHAI_KEYWORDS = + 'let const fn private if else switch for in while loop do until break continue return throw try catch import export as is this' + +const RHAI_SNIPPETS = [ + { caption: 'let', snippet: 'let ${1:name} = ${2:value};' }, + { caption: 'const', snippet: 'const ${1:NAME} = ${2:value};' }, + { caption: 'fn', snippet: 'fn ${1:name}(${2:args}) {\n\t${0}\n}' }, + { caption: 'if', snippet: 'if ${1:condition} {\n\t${0}\n}' }, + { caption: 'for', snippet: 'for ${1:item} in ${2:items} {\n\t${0}\n}' }, + { caption: 'while', snippet: 'while ${1:condition} {\n\t${0}\n}' }, + { caption: 'try', snippet: 'try {\n\t${1}\n} catch (${2:error}) {\n\t${0}\n}' }, +] + +const RHAI_COMPLETER: Ace.Completer = { + getCompletions(_editor, _session, _position, _prefix, callback) { + callback(null, [ + ...`${RHAI_KEYWORDS} true false`.split(' ').map((value) => ({ + caption: value, + value, + meta: 'Rhai keyword', + score: 100, + })), + ...RHAI_SNIPPETS.map((snippet) => ({ ...snippet, meta: 'Rhai snippet', score: 90 })), + ]) + }, +} + +export const RHAI_EDITOR_OPTIONS: Partial = { + useWorker: false, + tabSize: 4, + useSoftTabs: false, + enableBasicAutocompletion: [RHAI_COMPLETER], + enableLiveAutocompletion: [RHAI_COMPLETER], + enableSnippets: true, +} + +ace['define']( + 'ace/mode/rhai_highlight_rules', + ['require', 'exports', 'ace/lib/oop', 'ace/mode/text_highlight_rules'], + function (require, exports) { + const oop = require('ace/lib/oop') + const TextHighlightRules = require('ace/mode/text_highlight_rules').TextHighlightRules + + const RhaiHighlightRules = function (this: any) { + const keywords = this.createKeywordMapper( + { + keyword: RHAI_KEYWORDS.replaceAll(' ', '|'), + 'constant.language': 'true|false', + }, + 'identifier', + false, + ) + + this.$rules = { + start: [ + { token: 'comment', regex: /\/\/.*$/.source }, + { token: 'comment', regex: /\/\*/.source, push: 'comment' }, + { token: 'string', regex: /"/.source, push: 'string' }, + { token: 'string', regex: /`/.source, push: 'template' }, + { + token: 'string', + regex: /'(?:\\(?:u\{[\da-fA-F]+\}|u[\da-fA-F]{4}|x[\da-fA-F]{2}|.)|[^'\\])'/.source, + }, + { token: 'constant.language', regex: /\(\s*\)/.source }, + { + token: 'constant.numeric', + regex: + /\b(?:0x[\da-fA-F][\da-fA-F_]*|0o[0-7][0-7_]*|0b[01][01_]*|\d[\d_]*(?:\.(?!\.)\d[\d_]*)?(?:[eE][+-]?\d[\d_]*)?)\b/ + .source, + }, + { + token: (value: string) => { + const token = keywords(value) + return token === 'identifier' ? 'entity.name.function' : token + }, + regex: /[a-zA-Z_][\w]*(?=\s*\()/.source, + }, + { token: keywords, regex: /[a-zA-Z_][\w]*/.source }, + { token: 'paren.lparen', regex: /#\{|[\[({]/.source }, + { token: 'paren.rparen', regex: /[\])}]/.source }, + { + token: 'keyword.operator', + regex: + /\*\*=|<<=|>>=|\.\.=|==|!=|<=|>=|&&|\|\||\?\?|\?\.|=>|::|\*\*|<<|>>|\.\.|[+\-*/%&|^]=?|[!~<>=?:.]/ + .source, + }, + { token: 'punctuation.operator', regex: /[,;]/.source }, + { token: 'text', regex: /\s+/.source }, + ], + comment: [ + { token: 'comment', regex: /\/\*/.source, push: 'comment' }, + { token: 'comment', regex: /\*\//.source, next: 'pop' }, + { defaultToken: 'comment' }, + ], + string: [ + { token: 'constant.language.escape', regex: /\\./.source }, + { token: 'string', regex: /"/.source, next: 'pop' }, + { defaultToken: 'string' }, + ], + template: [ + { token: 'constant.language.escape', regex: /\\./.source }, + { token: 'paren.lparen', regex: /\$\{/.source, push: 'interpolation' }, + { token: 'string', regex: /`/.source, next: 'pop' }, + { defaultToken: 'string' }, + ], + interpolation: [ + { token: 'paren.lparen', regex: /#?\{/.source, push: 'interpolation' }, + { token: 'paren.rparen', regex: /\}/.source, next: 'pop' }, + { include: 'start' }, + ], + } + this.normalizeRules() + } + + oop.inherits(RhaiHighlightRules, TextHighlightRules) + exports.RhaiHighlightRules = RhaiHighlightRules + }, +) + +ace['define']( + 'ace/mode/rhai', + [ + 'require', + 'exports', + 'ace/lib/oop', + 'ace/mode/rust', + 'ace/mode/rhai_highlight_rules', + 'ace/mode/folding/cstyle', + ], + function (require, exports) { + const oop = require('ace/lib/oop') + const RustMode = require('ace/mode/rust').Mode + const RhaiHighlightRules = require('ace/mode/rhai_highlight_rules').RhaiHighlightRules + const FoldMode = require('ace/mode/folding/cstyle').FoldMode + + const Mode = function (this: any) { + RustMode.call(this) + this.HighlightRules = RhaiHighlightRules + this.foldingRules = new FoldMode() + this.lineCommentStart = '//' + this.blockComment = { start: '/*', end: '*/', nestable: true } + this.$id = 'ace/mode/rhai' + } + + oop.inherits(Mode, RustMode) + exports.Mode = Mode + }, +) diff --git a/apps/frontend/src/pages/moderation.vue b/apps/frontend/src/pages/moderation.vue index 0c92f2f7eed..81258c8783b 100644 --- a/apps/frontend/src/pages/moderation.vue +++ b/apps/frontend/src/pages/moderation.vue @@ -91,6 +91,7 @@ const moderationLinks = [ href: '/moderation/technical-review/rules', icon: SettingsIcon, }, + { label: 'Project flag rules', href: '/moderation/project-flag', icon: SettingsIcon }, ] const mobileNavOptions = [ @@ -100,6 +101,7 @@ const mobileNavOptions = [ formatMessage(messages.externalFilesTitle), formatMessage(messages.globalDetailTracesTitle), formatMessage(messages.delphiRulesTitle), + 'Project flag rules', ] const selectedChip = computed({ @@ -107,6 +109,8 @@ const selectedChip = computed({ const path = route.path if (path.startsWith('/moderation/technical-review/rules')) { return formatMessage(messages.delphiRulesTitle) + } else if (path.startsWith('/moderation/project-flag')) { + return 'Project flag rules' } else if (path.startsWith('/moderation/technical-review')) { return formatMessage(messages.technicalReviewTitle) } else if (path.startsWith('/moderation/reports')) { @@ -135,6 +139,8 @@ function navigateToPage(selectedOption: string) { router.push('/moderation/global-traces') } else if (selectedOption === formatMessage(messages.delphiRulesTitle)) { router.push('/moderation/technical-review/rules') + } else if (selectedOption === 'Project flag rules') { + router.push('/moderation/project-flag') } else { router.push('/moderation') } diff --git a/apps/frontend/src/pages/moderation/project-flag.vue b/apps/frontend/src/pages/moderation/project-flag.vue new file mode 100644 index 00000000000..31a0ecd27cb --- /dev/null +++ b/apps/frontend/src/pages/moderation/project-flag.vue @@ -0,0 +1,346 @@ + + + diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index a630e497c20..a8e39bfadb4 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -93,6 +93,7 @@ rdkafka = { workspace = true } redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "r2d2", "tokio-comp"] } regex = { workspace = true } reqwest = { workspace = true, features = ["http2", "json", "multipart", "rustls"] } +rhai = { workspace = true, features = ["serde", "sync"] } rust_decimal = { workspace = true, features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = { workspace = true } rustls.workspace = true diff --git a/apps/labrinth/migrations/20261028145153_flag_rules_backend.sql b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql similarity index 78% rename from apps/labrinth/migrations/20261028145153_flag_rules_backend.sql rename to apps/labrinth/migrations/20261028145153_project_flags_backend.sql index 2326c9d5de6..0ae45685d04 100644 --- a/apps/labrinth/migrations/20261028145153_flag_rules_backend.sql +++ b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql @@ -1,4 +1,4 @@ -CREATE TABLE moderation_flag_rules ( +CREATE TABLE project_flag_rules ( id BIGINT PRIMARY KEY, name VARCHAR(1023) NOT NULL, created_by BIGINT REFERENCES users(id) ON DELETE SET NULL, @@ -8,13 +8,13 @@ CREATE TABLE moderation_flag_rules ( expression VARCHAR(65535) NOT NULL ); -CREATE TABLE moderation_flags ( +CREATE TABLE project_flags ( id BIGINT PRIMARY KEY, - rule_id BIGINT REFERENCES moderation_flag_rules(id) ON DELETE SET NULL, + rule_id BIGINT REFERENCES project_flag_rules(id) ON DELETE SET NULL, what JSONB NOT NULL ); -CREATE TABLE moderation_flag_runs ( +CREATE TABLE project_flag_runs ( id BIGINT PRIMARY KEY, status TEXT NOT NULL, started_by BIGINT REFERENCES users(id) ON DELETE SET NULL, diff --git a/apps/labrinth/src/database/models/ids.rs b/apps/labrinth/src/database/models/ids.rs index 49b174a0a86..2ada750519e 100644 --- a/apps/labrinth/src/database/models/ids.rs +++ b/apps/labrinth/src/database/models/ids.rs @@ -304,6 +304,6 @@ id_type!(DelphiReportIssueId as i64); id_type!(DelphiReportIssueDetailsId as i64); id_type!(DelphiRuleId as i64); id_type!(SharedInstanceId as i64); -id_type!(ModerationFlagRuleId as i64); -id_type!(ModerationFlagId as i64); -id_type!(ModerationFlagRunId as i64); +id_type!(ProjectFlagRuleId as i64); +id_type!(ProjectFlagId as i64); +id_type!(ProjectFlagRunId as i64); diff --git a/apps/labrinth/src/database/models/mod.rs b/apps/labrinth/src/database/models/mod.rs index c0b5a974280..3b877785249 100644 --- a/apps/labrinth/src/database/models/mod.rs +++ b/apps/labrinth/src/database/models/mod.rs @@ -11,7 +11,6 @@ pub mod image_item; pub mod legacy_loader_fields; pub mod loader_fields; pub mod moderation_external_item; -pub mod moderation_flag_item; pub mod moderation_lock_item; pub mod moderation_note_item; pub mod notification_item; @@ -29,6 +28,7 @@ pub mod payouts_values_notifications; pub mod product_item; pub mod products_tax_identifier_item; pub mod project_disclosure_item; +pub mod project_flag_item; pub mod project_item; pub mod report_item; pub mod session_item; @@ -63,9 +63,9 @@ pub use thread_item::{DBThread, DBThreadMessage}; pub use user_item::DBUser; pub use version_item::DBVersion; -pub use moderation_flag_item::{ - DBModerationFlag, DBModerationFlagRule, DBModerationFlagRun, FlagRunOutput, - FlagRunStatus, -}; pub use moderation_lock_item::{DBModerationLock, ModerationLockWithUser}; pub use moderation_note_item::DBModerationNote; +pub use project_flag_item::{ + DBProjectFlag, DBProjectFlagRule, DBProjectFlagRun, ProjectFlagRunOutput, + ProjectFlagRunStatus, +}; diff --git a/apps/labrinth/src/database/models/moderation_flag_item.rs b/apps/labrinth/src/database/models/project_flag_item.rs similarity index 62% rename from apps/labrinth/src/database/models/moderation_flag_item.rs rename to apps/labrinth/src/database/models/project_flag_item.rs index 7b76fedf324..d8fe8ef6a30 100644 --- a/apps/labrinth/src/database/models/moderation_flag_item.rs +++ b/apps/labrinth/src/database/models/project_flag_item.rs @@ -5,13 +5,11 @@ use eyre::{Result, WrapErr}; use crate::database::{PgPool, ReadOnlyPgPool}; use serde::{Deserialize, Serialize}; -use super::{ - DBUserId, ModerationFlagId, ModerationFlagRuleId, ModerationFlagRunId, -}; +use super::{DBUserId, ProjectFlagId, ProjectFlagRuleId, ProjectFlagRunId}; #[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] -pub struct DBModerationFlagRule { - pub id: ModerationFlagRuleId, +pub struct DBProjectFlagRule { + pub id: ProjectFlagRuleId, pub name: String, pub created_by: Option, pub created_at: DateTime, @@ -20,7 +18,7 @@ pub struct DBModerationFlagRule { pub expression: String, } -impl DBModerationFlagRule { +impl DBProjectFlagRule { pub async fn list( limit: i64, offset: i64, @@ -29,7 +27,7 @@ impl DBModerationFlagRule { let rows = sqlx::query!( r#" SELECT id, name, expression, created_by, created_at, updated_by, updated_at - FROM moderation_flag_rules + FROM project_flag_rules ORDER BY created_at DESC, id DESC LIMIT $1 OFFSET $2 "#, @@ -38,12 +36,12 @@ impl DBModerationFlagRule { ) .fetch_all(&**pool) .await - .wrap_err("listing moderation flag rules")?; + .wrap_err("listing project flag rules")?; Ok(rows .into_iter() .map(|row| Self { - id: ModerationFlagRuleId(row.id), + id: ProjectFlagRuleId(row.id), name: row.name, expression: row.expression, created_by: row.created_by.map(DBUserId), @@ -55,23 +53,23 @@ impl DBModerationFlagRule { } pub async fn get( - id: ModerationFlagRuleId, + id: ProjectFlagRuleId, pool: &ReadOnlyPgPool, ) -> Result> { let row = sqlx::query!( r#" SELECT id, name, expression, created_by, created_at, updated_by, updated_at - FROM moderation_flag_rules + FROM project_flag_rules WHERE id = $1 "#, id.0, ) .fetch_optional(&**pool) .await - .wrap_err("fetching moderation flag rule")?; + .wrap_err("fetching project flag rule")?; Ok(row.map(|row| Self { - id: ModerationFlagRuleId(row.id), + id: ProjectFlagRuleId(row.id), name: row.name, expression: row.expression, created_by: row.created_by.map(DBUserId), @@ -91,7 +89,7 @@ impl DBModerationFlagRule { let now = Utc::now(); let row = sqlx::query!( r#" - INSERT INTO moderation_flag_rules + INSERT INTO project_flag_rules (id, name, expression, created_by, created_at, updated_by, updated_at) VALUES ($1, $2, $3, $4, $5, $4, $5) RETURNING id, name, expression, created_by, created_at, updated_by, updated_at @@ -104,10 +102,10 @@ impl DBModerationFlagRule { ) .fetch_one(pool) .await - .wrap_err("creating moderation flag rule")?; + .wrap_err("creating project flag rule")?; Ok(Self { - id: ModerationFlagRuleId(row.id), + id: ProjectFlagRuleId(row.id), name: row.name, expression: row.expression, created_by: row.created_by.map(DBUserId), @@ -118,7 +116,7 @@ impl DBModerationFlagRule { } pub async fn update( - id: ModerationFlagRuleId, + id: ProjectFlagRuleId, name: &str, expression: &str, user_id: DBUserId, @@ -127,7 +125,7 @@ impl DBModerationFlagRule { let now = Utc::now(); let row = sqlx::query!( r#" - UPDATE moderation_flag_rules + UPDATE project_flag_rules SET name = $2, expression = $3, updated_by = $4, updated_at = $5 WHERE id = $1 RETURNING id, name, expression, created_by, created_at, updated_by, updated_at @@ -140,10 +138,10 @@ impl DBModerationFlagRule { ) .fetch_optional(pool) .await - .wrap_err("updating moderation flag rule")?; + .wrap_err("updating project flag rule")?; Ok(row.map(|row| Self { - id: ModerationFlagRuleId(row.id), + id: ProjectFlagRuleId(row.id), name: row.name, expression: row.expression, created_by: row.created_by.map(DBUserId), @@ -153,52 +151,95 @@ impl DBModerationFlagRule { })) } - pub async fn delete( - id: ModerationFlagRuleId, - pool: &PgPool, - ) -> Result { + pub async fn delete(id: ProjectFlagRuleId, pool: &PgPool) -> Result { let result = sqlx::query!( r#" - DELETE FROM moderation_flag_rules + DELETE FROM project_flag_rules WHERE id = $1 "#, id.0, ) .execute(pool) .await - .wrap_err("deleting moderation flag rule")?; + .wrap_err("deleting project flag rule")?; Ok(result.rows_affected() > 0) } } #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DBModerationFlag { - pub id: ModerationFlagId, - pub rule_id: Option, +pub struct DBProjectFlag { + pub id: ProjectFlagId, + pub rule_id: Option, pub what: serde_json::Value, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DBModerationFlagRun { - pub id: ModerationFlagRunId, - pub status: FlagRunStatus, +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct DBProjectFlagRun { + pub id: ProjectFlagRunId, + pub status: ProjectFlagRunStatus, pub started_by: Option, pub started_at: DateTime, pub completed_at: Option>, - pub output: Option, + pub output: Option, + pub error: Option, +} + +impl DBProjectFlagRun { + pub async fn list( + limit: i64, + offset: i64, + pool: &ReadOnlyPgPool, + ) -> Result> { + let rows = sqlx::query!( + r#" + SELECT id, status AS "status: ProjectFlagRunStatus", started_by, + started_at, completed_at, + output AS "output: sqlx::types::Json", error + FROM project_flag_runs + ORDER BY started_at DESC, id DESC + LIMIT $1 OFFSET $2 + "#, + limit, + offset, + ) + .fetch_all(&**pool) + .await + .wrap_err("listing project flag runs")?; + + Ok(rows + .into_iter() + .map(|row| Self { + id: ProjectFlagRunId(row.id), + status: row.status, + started_by: row.started_by.map(DBUserId), + started_at: row.started_at, + completed_at: row.completed_at, + output: row.output.map(|output| output.0), + error: row.error, + }) + .collect()) + } } #[derive( - Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, sqlx::Type, + Debug, + Clone, + Copy, + PartialEq, + Eq, + Serialize, + Deserialize, + sqlx::Type, + utoipa::ToSchema, )] #[serde(rename_all = "snake_case")] #[sqlx(type_name = "text", rename_all = "snake_case")] -pub enum FlagRunStatus { +pub enum ProjectFlagRunStatus { Running, Ok, Err, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct FlagRunOutput {} +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProjectFlagRunOutput {} diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index 3edd0f8a87d..b9756e34841 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -12,6 +12,7 @@ use xredis::{ macro_rules! vars { ( $( + $(#[$meta:meta])* $field:ident: $ty:ty $(= $default:expr)?; )* ) => { @@ -22,6 +23,7 @@ macro_rules! vars { )] pub struct EnvVars { $( + $(#[$meta])* pub $field: $ty, )* } @@ -335,6 +337,8 @@ vars! { PUBLIC_DISCORD_WEBHOOK: String = ""; MODERATION_SLACK_WEBHOOK: String = ""; + /// Maximum execution time for each project flag rule evaluation, in milliseconds. + PROJECT_FLAG_RULE_TIMEOUT: u64 = 5000u64; DELPHI_SLACK_WEBHOOK: String = ""; TREMENDOUS_CAMPAIGN_ID: String = "none"; diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index 73c1ec057b3..8d62daffd24 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -20,6 +20,7 @@ use crate::background_task::update_versions; use crate::database::{PgPool, ReadOnlyPgPool}; use crate::env::ENV; use crate::queue::billing::{index_billing, index_subscriptions}; +use crate::queue::moderation::project_flag::ProjectFlagEngine; use crate::routes::internal::delphi::rescan::enqueue_tech_review_files_for_new_delphi_version; use crate::util::anrok; use crate::util::archon::ArchonClient; @@ -64,6 +65,7 @@ pub struct LabrinthConfig { pub ip_salt: Pepper, pub search_state: web::Data, pub session_queue: web::Data, + pub project_flag_engine: web::Data, pub payouts_queue: web::Data, pub active_sockets: web::Data, pub rate_limiter: web::Data, @@ -256,6 +258,10 @@ pub fn app_setup( ip_salt, search_state, session_queue, + project_flag_engine: web::Data::new( + ProjectFlagEngine::new() + .expect("failed to create project flag engine"), + ), payouts_queue: web::Data::new(PayoutsQueue::new()), active_sockets, rate_limiter: limiter, @@ -309,6 +315,7 @@ pub fn app_data_config( .app_data(labrinth_config.http_client.clone()) .app_data(labrinth_config.tiltify_client.clone()) .app_data(labrinth_config.session_queue.clone()) + .app_data(labrinth_config.project_flag_engine.clone()) .app_data(labrinth_config.payouts_queue.clone()) .app_data(labrinth_config.email_queue.clone()) .app_data(web::Data::new(labrinth_config.ip_salt.clone())) diff --git a/apps/labrinth/src/queue/moderation.rs b/apps/labrinth/src/queue/moderation/mod.rs similarity index 99% rename from apps/labrinth/src/queue/moderation.rs rename to apps/labrinth/src/queue/moderation/mod.rs index f43dc8b2bfa..521517ce2b0 100644 --- a/apps/labrinth/src/queue/moderation.rs +++ b/apps/labrinth/src/queue/moderation/mod.rs @@ -1,3 +1,5 @@ +pub mod project_flag; + use serde::{Deserialize, Serialize}; use std::collections::HashMap; diff --git a/apps/labrinth/src/queue/moderation/project_flag.rs b/apps/labrinth/src/queue/moderation/project_flag.rs new file mode 100644 index 00000000000..1f7aaff15b7 --- /dev/null +++ b/apps/labrinth/src/queue/moderation/project_flag.rs @@ -0,0 +1,109 @@ +use std::{cell::RefCell, sync::Arc, time::Duration}; + +use eyre::Result; + +use crate::{env::ENV, models::projects::Project, util::error::Context}; + +thread_local! { + static EVALUATION_LOG: RefCell>> = const { RefCell::new(None) }; +} + +struct EvaluationLog(Option>); + +impl EvaluationLog { + fn start() -> Self { + Self(EVALUATION_LOG.replace(Some(Vec::new()))) + } + + fn take(&self) -> Vec { + EVALUATION_LOG.with(|log| log.borrow_mut().take().unwrap_or_default()) + } +} + +impl Drop for EvaluationLog { + fn drop(&mut self) { + EVALUATION_LOG.replace(self.0.take()); + } +} + +#[derive(serde::Serialize, utoipa::ToSchema)] +pub struct ProjectFlagEvaluation { + pub flags: Vec, + pub log: Vec, +} + +#[derive(Debug)] +pub struct ProjectFlagEngine { + rhai: Arc, +} + +impl ProjectFlagEngine { + pub fn new() -> Result { + let mut rhai = rhai::Engine::new(); + rhai.disable_symbol("eval"); // eval considered evil + rhai.set_max_operations(1_000_000); + rhai.set_max_call_levels(64); + rhai.set_max_string_size(1_048_576); + rhai.set_max_array_size(10_000); + rhai.set_max_map_size(10_000); + rhai.on_print(|message| { + EVALUATION_LOG.with(|log| { + if let Some(log) = log.borrow_mut().as_mut() { + log.push(message.to_owned()); + } + }); + }); + Ok(Self { + rhai: Arc::new(rhai), + }) + } + + pub fn compile(&self, expression: &str) -> Result { + self.rhai.compile(expression).wrap_err("compiling script") + } + + pub async fn scan_flags_with_log( + &self, + ast: &rhai::AST, + project: &Project, + ) -> Result { + let rhai = Arc::clone(&self.rhai); + let ast = ast.clone(); + let project = project.clone(); + tokio::time::timeout( + Duration::from_millis(ENV.PROJECT_FLAG_RULE_TIMEOUT), + tokio::task::spawn_blocking(move || { + let log = EvaluationLog::start(); + let project = rhai::serde::to_dynamic(&project) + .wrap_err("serializing project for script")?; + let mut scope = rhai::Scope::new(); + scope.push("project", project); + let output = rhai + .eval_ast_with_scope::(&mut scope, &ast) + .wrap_err("running script")?; + let flags = output + .into_iter() + .map(|value| { + rhai::serde::from_dynamic::(&value) + .wrap_err("converting project flag to json") + }) + .collect::>>()?; + Ok(ProjectFlagEvaluation { + flags, + log: log.take(), + }) + }), + ) + .await + .wrap_err("project flag rule evaluation timed out")? + .wrap_err("joining project flag evaluation")? + } + + pub async fn scan_flags( + &self, + ast: &rhai::AST, + project: &Project, + ) -> Result> { + Ok(self.scan_flags_with_log(ast, project).await?.flags) + } +} diff --git a/apps/labrinth/src/routes/internal/mod.rs b/apps/labrinth/src/routes/internal/mod.rs index 3a8ff289dc2..de7cfcd1a80 100644 --- a/apps/labrinth/src/routes/internal/mod.rs +++ b/apps/labrinth/src/routes/internal/mod.rs @@ -103,6 +103,8 @@ pub fn config(cfg: &mut web::ServiceConfig) { pats::create_pat, pats::edit_pat, pats::delete_pat, + moderation::project_flag::list_runs, + moderation::project_flag::eval_rule, moderation::project_flag::list_rules, moderation::project_flag::get_rule, moderation::project_flag::create_rule, diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs index 8d0048402f9..2064d150f19 100644 --- a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs @@ -5,9 +5,16 @@ use validator::Validate; use xredis::RedisPool; use crate::auth::check_is_moderator_from_headers; -use crate::database::models::{DBModerationFlagRule, ModerationFlagRuleId}; +use crate::database::models::{ + DBProject, DBProjectFlagRule, DBProjectFlagRun, ProjectFlagRuleId, +}; use crate::database::{PgPool, ReadOnlyPgPool}; +use crate::models::ids::ProjectId; use crate::models::pats::Scopes; +use crate::models::projects::Project; +use crate::queue::moderation::project_flag::{ + ProjectFlagEngine, ProjectFlagEvaluation, +}; use crate::queue::session::AuthQueue; use crate::routes::ApiError; use crate::util::error::Context as _; @@ -18,7 +25,9 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig<'_>) { .service(get_rule) .service(create_rule) .service(update_rule) - .service(delete_rule); + .service(delete_rule) + .service(list_runs) + .service(eval_rule); } #[derive(Deserialize, utoipa::IntoParams)] @@ -38,7 +47,7 @@ fn default_limit() -> u16 { context_path = "/moderation/project-flag", tag = "moderation", params(ListRules), - responses((status = OK, body = [DBModerationFlagRule])) + responses((status = OK, body = [DBProjectFlagRule])) )] #[get("/rules")] pub async fn list_rules( @@ -47,7 +56,7 @@ pub async fn list_rules( ro_pool: web::Data, redis: web::Data, session_queue: web::Data, -) -> Result>, ApiError> { +) -> Result>, ApiError> { check_is_moderator_from_headers( &req, &***ro_pool, @@ -63,30 +72,30 @@ pub async fn list_rules( "`limit` must be between 1 and 200" ))); } - let rules = DBModerationFlagRule::list( + let rules = DBProjectFlagRule::list( i64::from(query.limit), i64::from(query.offset), &ro_pool, ) .await - .wrap_internal_err("listing moderation flag rules")?; + .wrap_internal_err("listing project flag rules")?; Ok(web::Json(rules)) } #[utoipa::path( context_path = "/moderation/project-flag", tag = "moderation", - params(("id" = ModerationFlagRuleId, Path)), - responses((status = OK, body = DBModerationFlagRule), (status = NOT_FOUND)) + params(("id" = ProjectFlagRuleId, Path)), + responses((status = OK, body = DBProjectFlagRule), (status = NOT_FOUND)) )] #[get("/rules/{id}")] pub async fn get_rule( req: HttpRequest, - path: web::Path, + path: web::Path, ro_pool: web::Data, redis: web::Data, session_queue: web::Data, -) -> Result, ApiError> { +) -> Result, ApiError> { check_is_moderator_from_headers( &req, &***ro_pool, @@ -97,10 +106,10 @@ pub async fn get_rule( .await .wrap_auth_err("authenticating moderator")?; - let rule = DBModerationFlagRule::get(path.into_inner(), &ro_pool) + let rule = DBProjectFlagRule::get(path.into_inner(), &ro_pool) .await - .wrap_internal_err("fetching moderation flag rule")? - .wrap_not_found_err("moderation flag rule not found")?; + .wrap_internal_err("fetching project flag rule")? + .wrap_not_found_err("project flag rule not found")?; Ok(web::Json(rule)) } @@ -119,7 +128,7 @@ pub struct CreateRule { context_path = "/moderation/project-flag", tag = "moderation", request_body = CreateRule, - responses((status = OK, body = DBModerationFlagRule)) + responses((status = OK, body = DBProjectFlagRule)) )] #[post("/rules")] pub async fn create_rule( @@ -128,7 +137,7 @@ pub async fn create_rule( pool: web::Data, redis: web::Data, session_queue: web::Data, -) -> Result, ApiError> { +) -> Result, ApiError> { let user = check_is_moderator_from_headers( &req, &**pool, @@ -142,33 +151,33 @@ pub async fn create_rule( ApiError::Request(eyre!(validation_errors_to_string(error, None))) })?; - let rule = DBModerationFlagRule::create( + let rule = DBProjectFlagRule::create( &body.name, &body.expression, user.id.into(), &pool, ) .await - .wrap_internal_err("creating moderation flag rule")?; + .wrap_internal_err("creating project flag rule")?; Ok(web::Json(rule)) } #[utoipa::path( context_path = "/moderation/project-flag", tag = "moderation", - params(("id" = ModerationFlagRuleId, Path)), + params(("id" = ProjectFlagRuleId, Path)), request_body = CreateRule, - responses((status = OK, body = DBModerationFlagRule), (status = NOT_FOUND)) + responses((status = OK, body = DBProjectFlagRule), (status = NOT_FOUND)) )] #[put("/rules/{id}")] pub async fn update_rule( req: HttpRequest, - path: web::Path, + path: web::Path, body: web::Json, pool: web::Data, redis: web::Data, session_queue: web::Data, -) -> Result, ApiError> { +) -> Result, ApiError> { let user = check_is_moderator_from_headers( &req, &**pool, @@ -182,7 +191,7 @@ pub async fn update_rule( ApiError::Request(eyre!(validation_errors_to_string(error, None))) })?; - let rule = DBModerationFlagRule::update( + let rule = DBProjectFlagRule::update( path.into_inner(), &body.name, &body.expression, @@ -190,21 +199,21 @@ pub async fn update_rule( &pool, ) .await - .wrap_internal_err("updating moderation flag rule")? - .wrap_not_found_err("moderation flag rule not found")?; + .wrap_internal_err("updating project flag rule")? + .wrap_not_found_err("project flag rule not found")?; Ok(web::Json(rule)) } #[utoipa::path( context_path = "/moderation/project-flag", tag = "moderation", - params(("id" = ModerationFlagRuleId, Path)), + params(("id" = ProjectFlagRuleId, Path)), responses((status = NO_CONTENT), (status = NOT_FOUND)) )] #[delete("/rules/{id}")] pub async fn delete_rule( req: HttpRequest, - path: web::Path, + path: web::Path, pool: web::Data, redis: web::Data, session_queue: web::Data, @@ -219,13 +228,119 @@ pub async fn delete_rule( .await .wrap_auth_err("authenticating moderator")?; - if !DBModerationFlagRule::delete(path.into_inner(), &pool) + if !DBProjectFlagRule::delete(path.into_inner(), &pool) .await - .wrap_internal_err("deleting moderation flag rule")? + .wrap_internal_err("deleting project flag rule")? { - return Err(ApiError::NotFound(eyre!( - "moderation flag rule not found" - ))); + return Err(ApiError::NotFound(eyre!("project flag rule not found"))); } Ok(()) } + +#[derive(Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListRuns { + #[serde(default = "default_limit")] + pub limit: u16, + #[serde(default)] + pub offset: u32, +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(ListRuns), + responses((status = OK, body = [DBProjectFlagRun])) +)] +#[get("/runs")] +pub async fn list_runs( + req: HttpRequest, + query: web::Query, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result>, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if query.limit == 0 || query.limit > 200 { + return Err(ApiError::Request(eyre!( + "`limit` must be between 1 and 200" + ))); + } + let runs = DBProjectFlagRun::list( + i64::from(query.limit), + i64::from(query.offset), + &ro_pool, + ) + .await + .wrap_internal_err("listing project flag runs")?; + Ok(web::Json(runs)) +} + +#[derive(Deserialize, Validate, utoipa::ToSchema)] +pub struct EvalRule { + pub project_id: ProjectId, + #[validate(length(min = 1, max = 65535))] + pub expression: String, +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + request_body = EvalRule, + responses( + (status = OK, body = ProjectFlagEvaluation), + (status = BAD_REQUEST), + (status = NOT_FOUND) + ) +)] +#[post("/rule/eval")] +pub async fn eval_rule( + req: HttpRequest, + body: web::Json, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, + engine: web::Data, +) -> Result, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + body.validate().map_err(|error| { + ApiError::Request(eyre!(validation_errors_to_string(error, None))) + })?; + let body = body.into_inner(); + let project = + DBProject::get_id(body.project_id.into(), &***ro_pool, &redis) + .await + .wrap_internal_err("fetching project for flag evaluation")? + .wrap_not_found_err("project not found")?; + let project = Project::from(project); + + let compile_engine = engine.clone(); + let ast = tokio::task::spawn_blocking(move || { + compile_engine.compile(&body.expression) + }) + .await + .wrap_internal_err("joining project flag compilation")? + .wrap_request_err("compiling project flag expression")?; + let evaluation = engine + .scan_flags_with_log(&ast, &project) + .await + .wrap_request_err("evaluating project flag expression")?; + Ok(web::Json(evaluation)) +} diff --git a/packages/api-client/src/modules/index.ts b/packages/api-client/src/modules/index.ts index b0699c3280f..8e3857cdb49 100644 --- a/packages/api-client/src/modules/index.ts +++ b/packages/api-client/src/modules/index.ts @@ -46,6 +46,7 @@ import { LabrinthOrganizationsV3Module } from './labrinth/organizations/v3' import { LabrinthPatsV2Module } from './labrinth/pats/v2' import { LabrinthPayoutV3Module } from './labrinth/payout/v3' import { LabrinthPayoutsV3Module } from './labrinth/payouts/v3' +import { LabrinthProjectFlagsInternalModule } from './labrinth/project-flags/internal' import { LabrinthProjectsV2Module } from './labrinth/projects/v2' import { LabrinthProjectsV3Module } from './labrinth/projects/v3' import { LabrinthReportsV3Module } from './labrinth/reports/v3' @@ -129,6 +130,7 @@ export const MODULE_REGISTRY = { labrinth_limits_v3: LabrinthLimitsV3Module, labrinth_payout_v3: LabrinthPayoutV3Module, labrinth_payouts_v3: LabrinthPayoutsV3Module, + labrinth_project_flags_internal: LabrinthProjectFlagsInternalModule, labrinth_projects_v2: LabrinthProjectsV2Module, labrinth_projects_v3: LabrinthProjectsV3Module, labrinth_reports_v3: LabrinthReportsV3Module, diff --git a/packages/api-client/src/modules/labrinth/project-flags/internal.ts b/packages/api-client/src/modules/labrinth/project-flags/internal.ts new file mode 100644 index 00000000000..9bf4acdef16 --- /dev/null +++ b/packages/api-client/src/modules/labrinth/project-flags/internal.ts @@ -0,0 +1,66 @@ +import { AbstractModule } from '../../../core/abstract-module' +import type { + CreateProjectFlagRuleRequest, + EvalProjectFlagRuleRequest, + ListProjectFlagRulesParams, + ProjectFlagEvaluation, + ProjectFlagRule, +} from './types' + +export class LabrinthProjectFlagsInternalModule extends AbstractModule { + public getModuleID(): string { + return 'labrinth_project_flags_internal' + } + + public async list(params: ListProjectFlagRulesParams = {}): Promise { + return this.client.request('/moderation/project-flag/rules', { + api: 'labrinth', + version: 'internal', + method: 'GET', + params, + }) + } + + public async get(id: number): Promise { + return this.client.request(`/moderation/project-flag/rules/${id}`, { + api: 'labrinth', + version: 'internal', + method: 'GET', + }) + } + + public async create(data: CreateProjectFlagRuleRequest): Promise { + return this.client.request('/moderation/project-flag/rules', { + api: 'labrinth', + version: 'internal', + method: 'POST', + body: data, + }) + } + + public async update(id: number, data: CreateProjectFlagRuleRequest): Promise { + return this.client.request(`/moderation/project-flag/rules/${id}`, { + api: 'labrinth', + version: 'internal', + method: 'PUT', + body: data, + }) + } + + public async evalRule(data: EvalProjectFlagRuleRequest): Promise { + return this.client.request('/moderation/project-flag/rule/eval', { + api: 'labrinth', + version: 'internal', + method: 'POST', + body: data, + }) + } + + public async delete(id: number): Promise { + return this.client.request(`/moderation/project-flag/rules/${id}`, { + api: 'labrinth', + version: 'internal', + method: 'DELETE', + }) + } +} diff --git a/packages/api-client/src/modules/labrinth/project-flags/types.ts b/packages/api-client/src/modules/labrinth/project-flags/types.ts new file mode 100644 index 00000000000..2cfbf7526bd --- /dev/null +++ b/packages/api-client/src/modules/labrinth/project-flags/types.ts @@ -0,0 +1,37 @@ +export interface ProjectFlagRule { + id: number + name: string + created_by: number | null + created_at: string + updated_by: number | null + updated_at: string + expression: string +} + +export type ListProjectFlagRulesParams = { + limit?: number + offset?: number +} + +export interface CreateProjectFlagRuleRequest { + name: string + expression: string +} + +export interface EvalProjectFlagRuleRequest { + project_id: string + expression: string +} + +export interface ProjectFlagEvaluation { + flags: ProjectFlagValue[] + log: string[] +} + +export type ProjectFlagValue = + | null + | boolean + | number + | string + | ProjectFlagValue[] + | { [key: string]: ProjectFlagValue } diff --git a/packages/ui/src/utils/ace-theme.ts b/packages/ui/src/utils/ace-theme.ts index e78e42d4835..d8b5c90fe05 100644 --- a/packages/ui/src/utils/ace-theme.ts +++ b/packages/ui/src/utils/ace-theme.ts @@ -1,7 +1,7 @@ +import ace from 'ace-builds' import 'ace-builds/esm-resolver' import cssText from '@modrinth/assets/styles/ace.css?raw' -import ace from 'ace-builds' ace['define']( 'ace/theme/modrinth', From 2f29c7eefec81671bfa591890d1d807eab843fe2 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:44:40 +0000 Subject: [PATCH 04/26] wip: project flags can scan files for patterns --- .../project-flag/FileScanPatternsEditor.vue | 78 ++++++ .../project-flag/ProjectFlagInputSchema.vue | 52 ++++ .../project-flag/ProjectFlagRuleTester.vue | 48 +++- .../src/pages/moderation/project-flag.vue | 21 +- apps/labrinth/src/lib.rs | 14 +- apps/labrinth/src/queue/moderation/mod.rs | 1 + .../src/queue/moderation/project_flag.rs | 179 +++++++++--- .../queue/moderation/project_flag_files.rs | 264 ++++++++++++++++++ apps/labrinth/src/routes/internal/mod.rs | 1 + .../internal/moderation/project_flag/mod.rs | 59 +++- .../moderation/project_flag/schema_example.rs | 145 ++++++++++ .../labrinth/project-flags/internal.ts | 12 + .../modules/labrinth/project-flags/types.ts | 3 + 13 files changed, 810 insertions(+), 67 deletions(-) create mode 100644 apps/frontend/src/components/ui/moderation/project-flag/FileScanPatternsEditor.vue create mode 100644 apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagInputSchema.vue create mode 100644 apps/labrinth/src/queue/moderation/project_flag_files.rs create mode 100644 apps/labrinth/src/routes/internal/moderation/project_flag/schema_example.rs diff --git a/apps/frontend/src/components/ui/moderation/project-flag/FileScanPatternsEditor.vue b/apps/frontend/src/components/ui/moderation/project-flag/FileScanPatternsEditor.vue new file mode 100644 index 00000000000..48402bc3d91 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/FileScanPatternsEditor.vue @@ -0,0 +1,78 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagInputSchema.vue b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagInputSchema.vue new file mode 100644 index 00000000000..bf0266403e0 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagInputSchema.vue @@ -0,0 +1,52 @@ + + + diff --git a/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue index 195a6661c93..24cf728de09 100644 --- a/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue +++ b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuleTester.vue @@ -1,5 +1,6 @@ - + 

{{ rule.expression }}
+ >{{ rule.params.expression }}
>('ruleModal') const isEditing = ref(false) const editingId = ref(null) const form = reactive({ name: '', expression: '' }) +const patterns = ref([]) +const patternErrors = computed(() => { + const names = patterns.value.map((entry) => entry.name.trim()) + return patterns.value.map((entry) => { + const name = entry.name.trim() + if (!name) return 'Enter a pattern name.' + if (names.indexOf(name) !== names.lastIndexOf(name)) { + return 'Each pattern must have a unique name.' + } + if (!entry.pattern.trim()) return 'Enter a regex pattern.' + return '' + }) +}) const ruleToDelete = ref(null) const isValid = computed( () => form.name.trim().length > 0 && Array.from(form.name).length <= 1023 && form.expression.length > 0 && - Array.from(form.expression).length <= 65535, + Array.from(form.expression).length <= 65535 && + patternErrors.value.every((error) => !error), ) const { @@ -265,12 +285,12 @@ const { mutationFn: ({ id, name, - expression, + params, }: { id: number | null name: string - expression: string - }) => (id === null ? api.create({ name, expression }) : api.update(id, { name, expression })), + params: Rule['params'] + }) => (id === null ? api.create({ name, params }) : api.update(id, { name, params })), onSuccess: async (_, { id }) => { isEditing.value = false await nextTick() @@ -303,6 +323,7 @@ function openCreate() { editingId.value = null form.name = '' form.expression = DEFAULT_RULE + patterns.value = [] isEditing.value = true ruleModal.value?.show() } @@ -312,7 +333,11 @@ function openEdit(rule: Rule) { ruleToDelete.value = null editingId.value = rule.id form.name = rule.name - form.expression = rule.expression + form.expression = rule.params.expression + patterns.value = Object.entries(rule.params.scan_files_for).map(([name, pattern]) => ({ + name, + pattern, + })) isEditing.value = true ruleModal.value?.show() } @@ -328,7 +353,16 @@ function handleEditorHide() { function saveRule() { if (!isValid.value || isBusy.value) return - save({ id: editingId.value, name: form.name, expression: form.expression }) + save({ + id: editingId.value, + name: form.name, + params: { + expression: form.expression, + scan_files_for: Object.fromEntries( + patterns.value.map((entry) => [entry.name.trim(), entry.pattern]), + ), + }, + }) } function confirmDelete(rule: Rule) { diff --git a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql index 0ae45685d04..4cc7776809a 100644 --- a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql +++ b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql @@ -5,7 +5,7 @@ CREATE TABLE project_flag_rules ( created_at TIMESTAMPTZ NOT NULL, updated_by BIGINT REFERENCES users(id) ON DELETE SET NULL, updated_at TIMESTAMPTZ NOT NULL, - expression VARCHAR(65535) NOT NULL + params JSONB NOT NULL ); CREATE TABLE project_flags ( diff --git a/apps/labrinth/src/database/models/project_flag_item.rs b/apps/labrinth/src/database/models/project_flag_item.rs index d8fe8ef6a30..740ee4d31c8 100644 --- a/apps/labrinth/src/database/models/project_flag_item.rs +++ b/apps/labrinth/src/database/models/project_flag_item.rs @@ -1,12 +1,45 @@ +use std::collections::BTreeMap; + use ariadne::ids::random_base62; use chrono::{DateTime, Utc}; use eyre::{Result, WrapErr}; use crate::database::{PgPool, ReadOnlyPgPool}; +use regex::Regex; use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, serde_as}; +use utoipa::ToSchema; +use validator::Validate; use super::{DBUserId, ProjectFlagId, ProjectFlagRuleId, ProjectFlagRunId}; +#[derive( + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + ToSchema, +)] +#[serde(transparent)] +#[schema(value_type = String, example = "suspicious_url")] +pub struct PatternName(pub String); + +#[serde_as] +#[derive(Clone, Debug, Serialize, Deserialize, Validate, ToSchema)] +pub struct ProjectRuleParams { + #[validate(length(min = 1, max = 65535))] + pub expression: String, + #[serde(default)] + #[serde_as(as = "BTreeMap<_, DisplayFromStr>")] + #[schema(value_type = BTreeMap)] + pub scan_files_for: BTreeMap, +} + #[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] pub struct DBProjectFlagRule { pub id: ProjectFlagRuleId, @@ -15,7 +48,7 @@ pub struct DBProjectFlagRule { pub created_at: DateTime, pub updated_by: Option, pub updated_at: DateTime, - pub expression: String, + pub params: ProjectRuleParams, } impl DBProjectFlagRule { @@ -26,7 +59,8 @@ impl DBProjectFlagRule { ) -> Result> { let rows = sqlx::query!( r#" - SELECT id, name, expression, created_by, created_at, updated_by, updated_at + SELECT id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at FROM project_flag_rules ORDER BY created_at DESC, id DESC LIMIT $1 OFFSET $2 @@ -43,7 +77,7 @@ impl DBProjectFlagRule { .map(|row| Self { id: ProjectFlagRuleId(row.id), name: row.name, - expression: row.expression, + params: row.params.0, created_by: row.created_by.map(DBUserId), created_at: row.created_at, updated_by: row.updated_by.map(DBUserId), @@ -58,7 +92,8 @@ impl DBProjectFlagRule { ) -> Result> { let row = sqlx::query!( r#" - SELECT id, name, expression, created_by, created_at, updated_by, updated_at + SELECT id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at FROM project_flag_rules WHERE id = $1 "#, @@ -71,7 +106,7 @@ impl DBProjectFlagRule { Ok(row.map(|row| Self { id: ProjectFlagRuleId(row.id), name: row.name, - expression: row.expression, + params: row.params.0, created_by: row.created_by.map(DBUserId), created_at: row.created_at, updated_by: row.updated_by.map(DBUserId), @@ -81,7 +116,7 @@ impl DBProjectFlagRule { pub async fn create( name: &str, - expression: &str, + params: &ProjectRuleParams, user_id: DBUserId, pool: &PgPool, ) -> Result { @@ -90,13 +125,14 @@ impl DBProjectFlagRule { let row = sqlx::query!( r#" INSERT INTO project_flag_rules - (id, name, expression, created_by, created_at, updated_by, updated_at) + (id, name, params, created_by, created_at, updated_by, updated_at) VALUES ($1, $2, $3, $4, $5, $4, $5) - RETURNING id, name, expression, created_by, created_at, updated_by, updated_at + RETURNING id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at "#, id, name, - expression, + sqlx::types::Json(params) as _, user_id.0, now, ) @@ -107,7 +143,7 @@ impl DBProjectFlagRule { Ok(Self { id: ProjectFlagRuleId(row.id), name: row.name, - expression: row.expression, + params: row.params.0, created_by: row.created_by.map(DBUserId), created_at: row.created_at, updated_by: row.updated_by.map(DBUserId), @@ -118,7 +154,7 @@ impl DBProjectFlagRule { pub async fn update( id: ProjectFlagRuleId, name: &str, - expression: &str, + params: &ProjectRuleParams, user_id: DBUserId, pool: &PgPool, ) -> Result> { @@ -126,13 +162,14 @@ impl DBProjectFlagRule { let row = sqlx::query!( r#" UPDATE project_flag_rules - SET name = $2, expression = $3, updated_by = $4, updated_at = $5 + SET name = $2, params = $3, updated_by = $4, updated_at = $5 WHERE id = $1 - RETURNING id, name, expression, created_by, created_at, updated_by, updated_at + RETURNING id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at "#, id.0, name, - expression, + sqlx::types::Json(params) as _, user_id.0, now, ) @@ -143,7 +180,7 @@ impl DBProjectFlagRule { Ok(row.map(|row| Self { id: ProjectFlagRuleId(row.id), name: row.name, - expression: row.expression, + params: row.params.0, created_by: row.created_by.map(DBUserId), created_at: row.created_at, updated_by: row.updated_by.map(DBUserId), diff --git a/apps/labrinth/src/models/mod.rs b/apps/labrinth/src/models/mod.rs index c38b17aeb5c..35c1adf2684 100644 --- a/apps/labrinth/src/models/mod.rs +++ b/apps/labrinth/src/models/mod.rs @@ -1,6 +1,7 @@ pub mod error; pub mod exp; pub mod link_platform; + pub mod v2; pub mod v3; diff --git a/apps/labrinth/src/queue/moderation/project_flag.rs b/apps/labrinth/src/queue/moderation/project_flag.rs index cb5d589484c..6566a8aede9 100644 --- a/apps/labrinth/src/queue/moderation/project_flag.rs +++ b/apps/labrinth/src/queue/moderation/project_flag.rs @@ -8,7 +8,10 @@ use super::project_flag_files::scan_project_files; use crate::{ database::{ ReadOnlyPgPool, - models::{DBProject, DBProjectId}, + models::{ + DBProject, DBProjectId, + project_flag_item::{PatternName, ProjectRuleParams}, + }, }, env::ENV, file_hosting::FileHost, @@ -68,22 +71,6 @@ pub struct ScannedFile { pub found_patterns: Vec, } -#[derive( - Clone, - Debug, - PartialEq, - Eq, - PartialOrd, - Ord, - Hash, - serde::Serialize, - serde::Deserialize, - utoipa::ToSchema, -)] -#[serde(transparent)] -#[schema(value_type = String, example = "suspicious_url")] -pub struct PatternName(pub String); - #[derive(Clone, Debug)] pub struct ProjectRuleCall { pub ast: rhai::AST, @@ -127,22 +114,15 @@ impl ProjectFlagEngine { pub fn make_call( &self, - expression: &str, - scan_files_for: BTreeMap, + params: &ProjectRuleParams, ) -> Result { - let ast = self.rhai.compile(expression).wrap_err("compiling script")?; - let scan_files_for = scan_files_for - .into_iter() - .map(|(name, pattern)| { - let pattern = Regex::new(&pattern).wrap_err_with(|| { - format!("compiling file scan pattern `{name}`") - })?; - Ok((PatternName(name), pattern)) - }) - .collect::>>()?; + let ast = self + .rhai + .compile(¶ms.expression) + .wrap_err("compiling script")?; Ok(ProjectRuleCall { ast, - scan_files_for, + scan_files_for: params.scan_files_for.clone(), }) } diff --git a/apps/labrinth/src/queue/moderation/project_flag_files.rs b/apps/labrinth/src/queue/moderation/project_flag_files.rs index ef08bb9e3e8..e38a14036e8 100644 --- a/apps/labrinth/src/queue/moderation/project_flag_files.rs +++ b/apps/labrinth/src/queue/moderation/project_flag_files.rs @@ -12,7 +12,9 @@ use crate::database::models::ids::{DBFileId, DBProjectId, DBVersionId}; use crate::env::ENV; use crate::file_hosting::{FileHost, FileHostPublicity}; -use super::project_flag::{PatternName, ScannedFile}; +use crate::database::models::project_flag_item::PatternName; + +use super::project_flag::ScannedFile; // Bound each buffered artifact and text, and cumulative network/decompression work. const MAX_ARTIFACT_BYTES: u64 = 64 * 1024 * 1024; diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs index b9124e41819..8a203231748 100644 --- a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use actix_web::{HttpRequest, delete, get, post, put, web}; use eyre::eyre; use serde::Deserialize; @@ -7,6 +5,7 @@ use validator::Validate; use xredis::RedisPool; use crate::auth::check_is_moderator_from_headers; +use crate::database::models::project_flag_item::ProjectRuleParams; use crate::database::models::{ DBProject, DBProjectFlagRule, DBProjectFlagRun, ProjectFlagRuleId, }; @@ -155,8 +154,8 @@ pub struct CreateRule { custom(function = "validate_name") )] pub name: String, - #[validate(length(min = 1, max = 65535))] - pub expression: String, + #[validate(nested)] + pub params: ProjectRuleParams, } #[utoipa::path( @@ -188,7 +187,7 @@ pub async fn create_rule( let rule = DBProjectFlagRule::create( &body.name, - &body.expression, + &body.params, user.id.into(), &pool, ) @@ -229,7 +228,7 @@ pub async fn update_rule( let rule = DBProjectFlagRule::update( path.into_inner(), &body.name, - &body.expression, + &body.params, user.id.into(), &pool, ) @@ -323,10 +322,8 @@ pub async fn list_runs( #[derive(Deserialize, Validate, utoipa::ToSchema)] pub struct EvalRule { pub project_id: ProjectId, - #[validate(length(min = 1, max = 65535))] - pub expression: String, - #[serde(default)] - pub scan_files_for: BTreeMap, + #[validate(nested)] + pub params: ProjectRuleParams, } #[utoipa::path( @@ -368,7 +365,7 @@ pub async fn eval_rule( .wrap_not_found_err("project not found")?; let compile_engine = engine.clone(); let call = tokio::task::spawn_blocking(move || { - compile_engine.make_call(&body.expression, body.scan_files_for) + compile_engine.make_call(&body.params) }) .await .wrap_internal_err("joining project flag compilation")? diff --git a/packages/api-client/src/modules/labrinth/project-flags/types.ts b/packages/api-client/src/modules/labrinth/project-flags/types.ts index a86d6cd0ce5..02254771ca2 100644 --- a/packages/api-client/src/modules/labrinth/project-flags/types.ts +++ b/packages/api-client/src/modules/labrinth/project-flags/types.ts @@ -1,3 +1,8 @@ +export interface ProjectFlagRuleParams { + expression: string + scan_files_for: Record +} + export interface ProjectFlagRule { id: number name: string @@ -5,7 +10,7 @@ export interface ProjectFlagRule { created_at: string updated_by: number | null updated_at: string - expression: string + params: ProjectFlagRuleParams } export type ListProjectFlagRulesParams = { @@ -15,13 +20,14 @@ export type ListProjectFlagRulesParams = { export interface CreateProjectFlagRuleRequest { name: string - expression: string + params: Omit & { + scan_files_for?: Record + } } export interface EvalProjectFlagRuleRequest { project_id: string - expression: string - scan_files_for?: Record + params: CreateProjectFlagRuleRequest['params'] } export type ProjectFlagRuleSchemaResponse = Record From 600a114d04a68ba184a29c034368e0814591de06 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:01:05 +0000 Subject: [PATCH 06/26] more flag context --- .../src/queue/moderation/project_flag.rs | 78 ++++++++++++++++++- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/apps/labrinth/src/queue/moderation/project_flag.rs b/apps/labrinth/src/queue/moderation/project_flag.rs index 6566a8aede9..9106053ccff 100644 --- a/apps/labrinth/src/queue/moderation/project_flag.rs +++ b/apps/labrinth/src/queue/moderation/project_flag.rs @@ -1,4 +1,9 @@ -use std::{cell::RefCell, collections::BTreeMap, sync::Arc, time::Duration}; +use std::{ + cell::RefCell, + collections::{BTreeMap, HashMap}, + sync::Arc, + time::Duration, +}; use eyre::{Result, eyre}; use regex::Regex; @@ -9,15 +14,18 @@ use crate::{ database::{ ReadOnlyPgPool, models::{ - DBProject, DBProjectId, + DBOrganization, DBProject, DBProjectId, DBTeamMember, DBUser, project_flag_item::{PatternName, ProjectRuleParams}, }, }, env::ENV, file_hosting::FileHost, models::{ - ids::{FileId, VersionId}, + ids::{FileId, TeamId, VersionId}, + organizations::Organization, projects::Project, + teams::TeamMember, + users::User, }, util::error::Context, }; @@ -53,9 +61,18 @@ pub struct ProjectFlagEvaluation { #[derive(serde::Serialize, utoipa::ToSchema)] pub struct FlagContext { pub project: Project, + pub team: FlagTeam, + pub users: Vec, + pub organization: Option, pub scanned_files: Vec, } +#[derive(serde::Serialize, utoipa::ToSchema)] +pub struct FlagTeam { + pub id: TeamId, + pub members: Vec, +} + #[derive(serde::Serialize, utoipa::ToSchema)] pub struct ScannedFile { #[schema(example = "AABBCCDD")] @@ -139,6 +156,58 @@ impl ProjectFlagEngine { .await .wrap_err("fetching project for flag evaluation")? .ok_or_else(|| eyre!("project not found"))?; + let team_id = project.inner.team_id; + let organization = match project.inner.organization_id { + Some(id) => Some( + DBOrganization::get_id(id, &*self.ro_pool, &self.redis) + .await + .wrap_err("fetching project organization for flag evaluation")? + .ok_or_else(|| eyre!("project organization not found"))?, + ), + None => None, + }; + let mut team_ids = vec![team_id]; + if let Some(organization) = &organization { + team_ids.push(organization.team_id); + } + let members = DBTeamMember::get_from_team_full_many( + &team_ids, + &*self.ro_pool, + &self.redis, + ) + .await + .wrap_err("fetching team members for flag evaluation")?; + let mut user_ids = members.iter().map(|member| member.user_id).collect::>(); + user_ids.sort_unstable_by_key(|id| id.0); + user_ids.dedup(); + let users = DBUser::get_many_ids(&user_ids, &*self.ro_pool, &self.redis) + .await + .wrap_err("fetching team users for flag evaluation")? + .into_iter() + .map(|user| (user.id, User::from(user))) + .collect::>(); + let mut project_members = Vec::new(); + let mut organization_members = Vec::new(); + for member in members { + let member_team_id = member.team_id; + let user = users.get(&member.user_id) + .cloned() + .ok_or_else(|| eyre!("team member user not found"))?; + let member = TeamMember::from_model(member, user, true); + if member_team_id == team_id { + project_members.push(member); + } else { + organization_members.push(member); + } + } + let users = project_members.iter().map(|member| member.user.clone()).collect(); + let team = FlagTeam { + id: team_id.into(), + members: project_members, + }; + let organization = organization.map(|organization| { + Organization::from(organization, organization_members) + }); let project = Project::from(project); let scanned_files = scan_project_files( project_id, @@ -149,6 +218,9 @@ impl ProjectFlagEngine { .await?; let context = FlagContext { project, + team, + users, + organization, scanned_files, }; let rhai = Arc::clone(&self.rhai); From 8b68be6e8829884e72b0f9de20e2286693cb9ae2 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Mon, 2 Nov 2026 14:01:58 +0000 Subject: [PATCH 07/26] listing and starting runs --- .../20261028145153_project_flags_backend.sql | 2 +- apps/labrinth/src/database/models/ids.rs | 2 +- .../src/database/models/project_flag_item.rs | 95 +++++++++++++ apps/labrinth/src/main.rs | 1 + apps/labrinth/src/queue/moderation/mod.rs | 1 + .../src/queue/moderation/project_flag_run.rs | 132 ++++++++++++++++++ apps/labrinth/src/routes/internal/mod.rs | 1 + .../internal/moderation/project_flag/mod.rs | 63 ++++++++- apps/labrinth/src/test/mod.rs | 1 + apps/labrinth/src/util/kafka.rs | 58 ++++++-- 10 files changed, 340 insertions(+), 16 deletions(-) create mode 100644 apps/labrinth/src/queue/moderation/project_flag_run.rs diff --git a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql index 4cc7776809a..35e370b8987 100644 --- a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql +++ b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql @@ -15,7 +15,7 @@ CREATE TABLE project_flags ( ); CREATE TABLE project_flag_runs ( - id BIGINT PRIMARY KEY, + id UUID PRIMARY KEY, status TEXT NOT NULL, started_by BIGINT REFERENCES users(id) ON DELETE SET NULL, started_at TIMESTAMPTZ NOT NULL, diff --git a/apps/labrinth/src/database/models/ids.rs b/apps/labrinth/src/database/models/ids.rs index 2ada750519e..d980d2d08df 100644 --- a/apps/labrinth/src/database/models/ids.rs +++ b/apps/labrinth/src/database/models/ids.rs @@ -306,4 +306,4 @@ id_type!(DelphiRuleId as i64); id_type!(SharedInstanceId as i64); id_type!(ProjectFlagRuleId as i64); id_type!(ProjectFlagId as i64); -id_type!(ProjectFlagRunId as i64); +id_type!(ProjectFlagRunId as uuid::Uuid); diff --git a/apps/labrinth/src/database/models/project_flag_item.rs b/apps/labrinth/src/database/models/project_flag_item.rs index 740ee4d31c8..80cebbef9fb 100644 --- a/apps/labrinth/src/database/models/project_flag_item.rs +++ b/apps/labrinth/src/database/models/project_flag_item.rs @@ -52,6 +52,33 @@ pub struct DBProjectFlagRule { } impl DBProjectFlagRule { + pub async fn all(pool: &PgPool) -> Result> { + let rows = sqlx::query!( + r#" + SELECT id, name, params AS "params: sqlx::types::Json", + created_by, created_at, updated_by, updated_at + FROM project_flag_rules + ORDER BY id + "#, + ) + .fetch_all(pool) + .await + .wrap_err("snapshotting project flag rules")?; + + Ok(rows + .into_iter() + .map(|row| Self { + id: ProjectFlagRuleId(row.id), + name: row.name, + params: row.params.0, + created_by: row.created_by.map(DBUserId), + created_at: row.created_at, + updated_by: row.updated_by.map(DBUserId), + updated_at: row.updated_at, + }) + .collect()) + } + pub async fn list( limit: i64, offset: i64, @@ -223,6 +250,74 @@ pub struct DBProjectFlagRun { } impl DBProjectFlagRun { + pub async fn queued_project_ids( + pool: &PgPool, + ) -> Result> { + let ids = sqlx::query_scalar!( + r#"SELECT id FROM mods WHERE status = 'processing'"#, + ) + .fetch_all(pool) + .await + .wrap_err("fetching queued project IDs")?; + Ok(ids.into_iter().map(super::DBProjectId).collect()) + } + + pub async fn create(user_id: DBUserId, pool: &PgPool) -> Result { + let run = Self { + id: ProjectFlagRunId(uuid::Uuid::now_v7()), + status: ProjectFlagRunStatus::Running, + started_by: Some(user_id), + started_at: Utc::now(), + completed_at: None, + output: None, + error: None, + }; + sqlx::query!( + r#" + INSERT INTO project_flag_runs (id, status, started_by, started_at) + VALUES ($1, 'running', $2, $3) + "#, + run.id.0, + user_id.0, + run.started_at, + ) + .execute(pool) + .await + .wrap_err("creating project flag run")?; + Ok(run) + } + + pub async fn finish( + &mut self, + error: Option, + pool: &PgPool, + ) -> Result<()> { + let status = if error.is_some() { + ProjectFlagRunStatus::Err + } else { + ProjectFlagRunStatus::Ok + }; + let completed_at = Utc::now(); + sqlx::query!( + r#" + UPDATE project_flag_runs + SET status = $2, completed_at = $3, error = $4 + WHERE id = $1 + "#, + self.id.0, + status as ProjectFlagRunStatus, + completed_at, + error, + ) + .execute(pool) + .await + .wrap_err("finishing project flag run")?; + self.status = status; + self.completed_at = Some(completed_at); + self.error = error; + Ok(()) + } + pub async fn list( limit: i64, offset: i64, diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index c137529afe8..e482f1fa1fa 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -130,6 +130,7 @@ async fn app() -> std::io::Result<()> { let kafka_client = actix_web::web::Data::new( labrinth::util::kafka::KafkaClientState::new() + .await .expect("Kafka connection failed"), ); diff --git a/apps/labrinth/src/queue/moderation/mod.rs b/apps/labrinth/src/queue/moderation/mod.rs index a640a47b3dd..303f5048cc9 100644 --- a/apps/labrinth/src/queue/moderation/mod.rs +++ b/apps/labrinth/src/queue/moderation/mod.rs @@ -1,5 +1,6 @@ pub mod project_flag; mod project_flag_files; +pub mod project_flag_run; use serde::{Deserialize, Serialize}; use std::collections::HashMap; diff --git a/apps/labrinth/src/queue/moderation/project_flag_run.rs b/apps/labrinth/src/queue/moderation/project_flag_run.rs new file mode 100644 index 00000000000..405e96debdc --- /dev/null +++ b/apps/labrinth/src/queue/moderation/project_flag_run.rs @@ -0,0 +1,132 @@ +use eyre::{Result, WrapErr, eyre}; +use futures::{StreamExt, TryStreamExt, stream}; +use rdkafka::producer::{FutureRecord, Producer}; +use serde::{Deserialize, Serialize}; +use xredis::RedisPool; + +use crate::{ + database::models::{ + DBProjectId, ProjectFlagRunId, project_flag_item::DBProjectFlagRule, + }, + models::ids::ProjectId, + util::kafka::{ + KAFKA_DELIVERY_TIMEOUT, KAFKA_OPERATION_INTERVAL, KafkaClientState, + KafkaEvent, create_project_flag_run_producer, sentry_trace_headers, + }, +}; + +pub const PROJECT_FLAG_RUN_TOPIC: &str = "private.labrinth.project-flag-run.v1"; + +#[derive(Debug, Serialize, Deserialize)] +pub struct ProjectFlagRunEvent { + pub project_id: ProjectId, + pub run_id: ProjectFlagRunId, +} + +/// Stores a permanent JSON rules snapshot under the run ID and atomically +/// publishes its project jobs. Consumers must use `read_committed` isolation. +/// Snapshots are retained on failure, including indeterminate commit outcomes. +/// Retrying a run is not guaranteed to be duplicate-free. +pub async fn dispatch_project_flag_run( + run_id: ProjectFlagRunId, + rules: Vec, + project_ids: Vec, + redis: &RedisPool, + kafka: &KafkaClientState, +) -> Result<()> { + let snapshot = serde_json::to_vec(&rules) + .wrap_err("serializing project flag run rules")?; + tokio::time::timeout(KAFKA_DELIVERY_TIMEOUT, async { + let mut connection = redis + .connect() + .await + .wrap_err("connecting to Redis for project flag run")?; + let key = connection + .key() + .entity("project_flag_run_rules:v1", run_id.0); + redis::cmd("SET") + .arg(&key) + .arg(&snapshot) + .arg("NX") + .query_async::>(&mut connection) + .await + .wrap_err("storing project flag run rules")? + .ok_or_else(|| eyre!("project flag run rules already exist"))?; + eyre::Ok(()) + }) + .await + .wrap_err("timed out storing project flag run rules")??; + + if project_ids.is_empty() { + return Ok(()); + } + + let headers = sentry_trace_headers(); + let runtime = tokio::runtime::Handle::current(); + let mut guard = kafka.project_flag_run_producer.clone().lock_owned().await; + tokio::task::spawn_blocking(move || { + let producer = match guard.take() { + Some(producer) => producer, + None => create_project_flag_run_producer()?, + }; + + let result = (|| { + producer + .begin_transaction() + .wrap_err("beginning project flag run Kafka transaction")?; + runtime.block_on(async { + tokio::time::timeout(KAFKA_DELIVERY_TIMEOUT, async { + let producer = &producer; + let headers = &headers; + stream::iter(project_ids) + .map(|project_id| async move { + let project_id = ProjectId::from(project_id); + let event = KafkaEvent::new( + PROJECT_FLAG_RUN_TOPIC, + ProjectFlagRunEvent { project_id, run_id }, + ); + let payload = serde_json::to_vec(&event) + .wrap_err("serializing project flag run event")?; + let project_key = project_id.to_string(); + producer + .send( + FutureRecord::to(PROJECT_FLAG_RUN_TOPIC) + .key(&project_key) + .payload(&payload) + .headers(headers.clone()), + KAFKA_OPERATION_INTERVAL, + ) + .await + .map_err(|(error, _)| eyre!(error)) + .wrap_err("publishing project flag run event")?; + eyre::Ok(()) + }) + .buffer_unordered(100) + .try_collect::<()>() + .await + }) + .await + .wrap_err("timed out publishing project flag run events")? + })?; + producer + .commit_transaction(KAFKA_DELIVERY_TIMEOUT) + .wrap_err("committing project flag run Kafka transaction")?; + eyre::Ok(()) + })(); + + if let Err(error) = result { + if let Err(abort_error) = + producer.abort_transaction(KAFKA_DELIVERY_TIMEOUT) + { + return Err(error.wrap_err(format!( + "aborting project flag run Kafka transaction failed: {abort_error}" + ))); + } + return Err(error); + } + *guard = Some(producer); + Ok(()) + }) + .await + .wrap_err("joining project flag run dispatch")? +} diff --git a/apps/labrinth/src/routes/internal/mod.rs b/apps/labrinth/src/routes/internal/mod.rs index 0b76b5969f8..266270a40a3 100644 --- a/apps/labrinth/src/routes/internal/mod.rs +++ b/apps/labrinth/src/routes/internal/mod.rs @@ -104,6 +104,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { pats::edit_pat, pats::delete_pat, moderation::project_flag::list_runs, + moderation::project_flag::start_run, moderation::project_flag::eval_rule, moderation::project_flag::list_rules, moderation::project_flag::get_rule_schema, diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs index 8a203231748..457edffe78f 100644 --- a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs @@ -26,11 +26,12 @@ mod schema_example; pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig<'_>) { cfg.service(list_rules) .service(get_rule_schema) + .service(list_runs) + .service(start_run) .service(get_rule) .service(create_rule) .service(update_rule) .service(delete_rule) - .service(list_runs) .service(eval_rule); } @@ -286,7 +287,7 @@ pub struct ListRuns { params(ListRuns), responses((status = OK, body = [DBProjectFlagRun])) )] -#[get("/runs")] +#[get("/rules/run")] pub async fn list_runs( req: HttpRequest, query: web::Query, @@ -319,6 +320,64 @@ pub async fn list_runs( Ok(web::Json(runs)) } +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + responses((status = OK, body = DBProjectFlagRun)) +)] +#[post("/rules/run")] +pub async fn start_run( + req: HttpRequest, + kafka: web::Data, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + let rules = DBProjectFlagRule::all(&pool) + .await + .wrap_internal_err("snapshotting project flag rules")?; + let project_ids = DBProjectFlagRun::queued_project_ids(&pool) + .await + .wrap_internal_err("fetching queued project IDs")?; + let mut run = DBProjectFlagRun::create(user.id.into(), &pool) + .await + .wrap_internal_err("creating project flag run")?; + + // Keep dispatch alive if the caller disconnects while Kafka is publishing. + let run = tokio::spawn(async move { + let empty = project_ids.is_empty(); + let result = crate::queue::moderation::project_flag_run::dispatch_project_flag_run( + run.id, rules, project_ids, &redis, &kafka, + ).await; + if let Err(error) = result { + run.finish(Some(format!("{error:#}")), &pool) + .await + .wrap_internal_err("recording project flag run dispatch failure")?; + return Err(ApiError::Internal(error)); + } + if empty { + run.finish(None, &pool) + .await + .wrap_internal_err("completing empty project flag run")?; + } + Ok(web::Json(run)) + }) + .await + .wrap_internal_err("joining project flag run dispatch")??; + Ok(run) +} + #[derive(Deserialize, Validate, utoipa::ToSchema)] pub struct EvalRule { pub project_id: ProjectId, diff --git a/apps/labrinth/src/test/mod.rs b/apps/labrinth/src/test/mod.rs index 97fb991b87d..9c8ddd5d616 100644 --- a/apps/labrinth/src/test/mod.rs +++ b/apps/labrinth/src/test/mod.rs @@ -52,6 +52,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig { .expect("Failed to create Gotenberg client"); let kafka_client = actix_web::web::Data::new( crate::util::kafka::KafkaClientState::new() + .await .expect("Kafka connection failed"), ); diff --git a/apps/labrinth/src/util/kafka.rs b/apps/labrinth/src/util/kafka.rs index 05879e8727e..a2785db002e 100644 --- a/apps/labrinth/src/util/kafka.rs +++ b/apps/labrinth/src/util/kafka.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use crate::env::ENV; use chrono::{DateTime, Utc}; @@ -7,9 +7,10 @@ use rdkafka::{ ClientConfig, consumer::StreamConsumer, message::{Header, OwnedHeaders}, - producer::FutureProducer, + producer::{FutureProducer, Producer}, }; use serde::Serialize; +use tokio::sync::Mutex; use uuid::Uuid; pub const KAFKA_OPERATION_INTERVAL: Duration = Duration::from_secs(5); @@ -18,20 +19,13 @@ pub const INCREMENTAL_INDEX_SEARCH_TASK: &str = "incremental-index-search"; pub struct KafkaClientState { pub client: FutureProducer, + pub project_flag_run_producer: Arc>>, pub incremental_index_search_consumer: StreamConsumer, } impl KafkaClientState { - pub fn new() -> eyre::Result { - let client = ClientConfig::new() - .set("bootstrap.servers", ENV.KAFKA_BOOTSTRAP_SERVERS.0.join(",")) - .set("client.id", &ENV.KAFKA_CLIENT_ID) - .set("acks", "all") - .set( - "delivery.timeout.ms", - KAFKA_DELIVERY_TIMEOUT.as_millis().to_string(), - ) - .set("broker.address.family", "v4") + pub async fn new() -> eyre::Result { + let client = producer_config() .create() .wrap_err("failed to create Kafka producer")?; let incremental_index_search_consumer = create_consumer( @@ -45,13 +39,53 @@ impl KafkaClientState { "Connected to Kafka" ); + let project_flag_run_producer = + tokio::task::spawn_blocking(create_project_flag_run_producer) + .await + .wrap_err( + "joining project flag run producer initialization", + )??; + Ok(Self { + project_flag_run_producer: Arc::new(Mutex::new(Some( + project_flag_run_producer, + ))), client, incremental_index_search_consumer, }) } } +pub fn create_project_flag_run_producer() -> eyre::Result { + let producer: FutureProducer = producer_config() + .set("enable.idempotence", "true") + .set( + "transactional.id", + format!("project-flag-run-{}", Uuid::now_v7()), + ) + .set("transaction.timeout.ms", "60000") + .create() + .wrap_err("creating project flag run Kafka producer")?; + producer + .init_transactions(KAFKA_DELIVERY_TIMEOUT) + .wrap_err("initializing project flag run Kafka transaction")?; + Ok(producer) +} + +pub fn producer_config() -> ClientConfig { + let mut config = ClientConfig::new(); + config + .set("bootstrap.servers", ENV.KAFKA_BOOTSTRAP_SERVERS.0.join(",")) + .set("client.id", &ENV.KAFKA_CLIENT_ID) + .set("acks", "all") + .set("broker.address.family", "v4") + .set( + "delivery.timeout.ms", + KAFKA_DELIVERY_TIMEOUT.as_millis().to_string(), + ); + config +} + pub fn create_consumer(group_id: &str) -> eyre::Result { ClientConfig::new() .set("bootstrap.servers", ENV.KAFKA_BOOTSTRAP_SERVERS.0.join(",")) From 89077186406d27aaaba4217e262570334dc0c883 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Mon, 2 Nov 2026 15:26:09 +0000 Subject: [PATCH 08/26] wip: consumer task for reading scan reqs --- apps/labrinth/src/background_task.rs | 7 + apps/labrinth/src/env.rs | 9 +- apps/labrinth/src/models/v3/organizations.rs | 2 +- .../src/queue/moderation/project_flag.rs | 300 +++++++++++------- .../queue/moderation/project_flag_files.rs | 78 +++-- .../src/queue/moderation/project_flag_run.rs | 248 ++++++++++++++- apps/labrinth/src/util/kafka.rs | 14 +- 7 files changed, 524 insertions(+), 134 deletions(-) diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs index a005cec8d36..ff0de237eb6 100644 --- a/apps/labrinth/src/background_task.rs +++ b/apps/labrinth/src/background_task.rs @@ -25,6 +25,8 @@ pub enum BackgroundTask { IndexBilling, IndexSubscriptions, IncrementalIndexSearch, + /// Consumes project flag run jobs from Kafka. + ProjectFlagRuns, Migrations, Mail, /// Queries server project analytics (e.g. number of verified plays in last @@ -89,6 +91,11 @@ impl BackgroundTask { ) .await } + ProjectFlagRuns => { + crate::queue::moderation::project_flag_run::consume_project_flag_runs( + ro_pool.into(), redis_pool, file_host.into_inner(), + ).await + } Mail => run_email(email_queue).await, CacheAnalytics => { cache_analytics(&pool, &redis_pool, &clickhouse).await diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index b9756e34841..444b4a7fe9a 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -1,4 +1,7 @@ -use std::{any::type_name, convert::Infallible, str::FromStr, sync::LazyLock}; +use std::{ + any::type_name, convert::Infallible, num::NonZeroUsize, str::FromStr, + sync::LazyLock, +}; use derive_more::{Deref, DerefMut}; use eyre::{Context, eyre}; @@ -339,6 +342,10 @@ vars! { MODERATION_SLACK_WEBHOOK: String = ""; /// Maximum execution time for each project flag rule evaluation, in milliseconds. PROJECT_FLAG_RULE_TIMEOUT: u64 = 5000u64; + /// Maximum number of project flag jobs collected per consumer batch. + PROJECT_FLAG_SCAN_BATCH_SIZE: NonZeroUsize = NonZeroUsize::new(100).unwrap(); + /// Maximum number of concurrent project flag artifact downloads per process. + PROJECT_FLAG_SCAN_DOWNLOAD_CONCURRENCY: NonZeroUsize = NonZeroUsize::new(8).unwrap(); DELPHI_SLACK_WEBHOOK: String = ""; TREMENDOUS_CAMPAIGN_ID: String = "none"; diff --git a/apps/labrinth/src/models/v3/organizations.rs b/apps/labrinth/src/models/v3/organizations.rs index 09032cc44c4..27021cc80f6 100644 --- a/apps/labrinth/src/models/v3/organizations.rs +++ b/apps/labrinth/src/models/v3/organizations.rs @@ -4,7 +4,7 @@ use crate::models::ids::{OrganizationId, TeamId}; use serde::{Deserialize, Serialize}; /// An organization of users who control a project -#[derive(Serialize, Deserialize, utoipa::ToSchema)] +#[derive(Clone, Serialize, Deserialize, utoipa::ToSchema)] pub struct Organization { /// The id of the organization pub id: OrganizationId, diff --git a/apps/labrinth/src/queue/moderation/project_flag.rs b/apps/labrinth/src/queue/moderation/project_flag.rs index 9106053ccff..d764dde7934 100644 --- a/apps/labrinth/src/queue/moderation/project_flag.rs +++ b/apps/labrinth/src/queue/moderation/project_flag.rs @@ -1,15 +1,18 @@ use std::{ cell::RefCell, collections::{BTreeMap, HashMap}, - sync::Arc, + sync::{Arc, LazyLock}, time::Duration, }; -use eyre::{Result, eyre}; +use eyre::{Result, ensure, eyre}; use regex::Regex; use xredis::RedisPool; -use super::project_flag_files::scan_project_files; +use super::project_flag_files::{ + ProjectScanFile, load_project_files, scan_project_files, +}; + use crate::{ database::{ ReadOnlyPgPool, @@ -30,6 +33,12 @@ use crate::{ util::error::Context, }; +static DOWNLOADS: LazyLock = LazyLock::new(|| { + tokio::sync::Semaphore::new( + ENV.PROJECT_FLAG_SCAN_DOWNLOAD_CONCURRENCY.get(), + ) +}); + thread_local! { static EVALUATION_LOG: RefCell>> = const { RefCell::new(None) }; } @@ -58,7 +67,7 @@ pub struct ProjectFlagEvaluation { pub log: Vec, } -#[derive(serde::Serialize, utoipa::ToSchema)] +#[derive(Clone, serde::Serialize, utoipa::ToSchema)] pub struct FlagContext { pub project: Project, pub team: FlagTeam, @@ -67,13 +76,13 @@ pub struct FlagContext { pub scanned_files: Vec, } -#[derive(serde::Serialize, utoipa::ToSchema)] +#[derive(Clone, serde::Serialize, utoipa::ToSchema)] pub struct FlagTeam { pub id: TeamId, pub members: Vec, } -#[derive(serde::Serialize, utoipa::ToSchema)] +#[derive(Clone, serde::Serialize, utoipa::ToSchema)] pub struct ScannedFile { #[schema(example = "AABBCCDD")] pub version_id: VersionId, @@ -94,6 +103,7 @@ pub struct ProjectRuleCall { pub scan_files_for: BTreeMap, } +#[derive(Clone)] pub struct ProjectFlagEngine { rhai: Arc, ro_pool: ReadOnlyPgPool, @@ -143,6 +153,167 @@ impl ProjectFlagEngine { }) } + pub(super) async fn load_contexts( + &self, + project_ids: &[DBProjectId], + ) -> Result> { + if project_ids.is_empty() { + return Ok(HashMap::new()); + } + let projects = + DBProject::get_many_ids(project_ids, &*self.ro_pool, &self.redis) + .await + .wrap_err("fetching projects for flag evaluation")?; + let mut organization_ids = projects + .iter() + .filter_map(|p| p.inner.organization_id) + .collect::>(); + organization_ids.sort_unstable_by_key(|id| id.0); + organization_ids.dedup(); + let organizations = DBOrganization::get_many_ids( + &organization_ids, + &*self.ro_pool, + &self.redis, + ) + .await + .wrap_err("fetching organizations for flag evaluation")? + .into_iter() + .map(|organization| (organization.id, organization)) + .collect::>(); + let mut team_ids = projects + .iter() + .map(|p| p.inner.team_id) + .chain(organizations.values().map(|o| o.team_id)) + .collect::>(); + team_ids.sort_unstable_by_key(|id| id.0); + team_ids.dedup(); + let members = DBTeamMember::get_from_team_full_many( + &team_ids, + &*self.ro_pool, + &self.redis, + ) + .await + .wrap_err("fetching team members for flag evaluation")?; + let mut user_ids = + members.iter().map(|m| m.user_id).collect::>(); + user_ids.sort_unstable_by_key(|id| id.0); + user_ids.dedup(); + let users = + DBUser::get_many_ids(&user_ids, &*self.ro_pool, &self.redis) + .await + .wrap_err("fetching team users for flag evaluation")? + .into_iter() + .map(|user| (user.id, User::from(user))) + .collect::>(); + let mut teams = HashMap::<_, Vec>::new(); + for member in members { + let user = users + .get(&member.user_id) + .cloned() + .ok_or_else(|| eyre!("team member user not found"))?; + teams + .entry(member.team_id) + .or_default() + .push(TeamMember::from_model(member, user, true)); + } + let mut contexts = HashMap::new(); + for project in projects { + let id = project.inner.id; + let team_id = project.inner.team_id; + let members = teams.get(&team_id).cloned().unwrap_or_default(); + let organization = project + .inner + .organization_id + .map(|id| { + let organization = + organizations.get(&id).cloned().ok_or_else(|| { + eyre!("project organization not found") + })?; + let members = teams + .get(&organization.team_id) + .cloned() + .unwrap_or_default(); + eyre::Ok(Organization::from(organization, members)) + }) + .transpose()?; + contexts.insert( + id, + FlagContext { + project: Project::from(project), + users: members + .iter() + .map(|member| member.user.clone()) + .collect(), + team: FlagTeam { + id: team_id.into(), + members, + }, + organization, + scanned_files: Vec::new(), + }, + ); + } + ensure!( + project_ids.iter().all(|id| contexts.contains_key(id)), + "project not found" + ); + Ok(contexts) + } + + pub(super) async fn load_files( + &self, + project_ids: &[DBProjectId], + ) -> Result>> { + load_project_files(project_ids, &self.ro_pool).await + } + + pub(super) async fn prepare_context( + &self, + mut context: FlagContext, + files: &[ProjectScanFile], + patterns: &BTreeMap, + ) -> Result { + context.scanned_files = scan_project_files( + files, + patterns, + self.file_host.as_ref(), + &DOWNLOADS, + ) + .await?; + Ok(context) + } + + pub fn evaluate( + &self, + call: &ProjectRuleCall, + context: &FlagContext, + ) -> Result { + let log = EvaluationLog::start(); + let context = rhai::serde::to_dynamic(context) + .wrap_err("serializing flag context for script")? + .try_cast::() + .ok_or_else(|| eyre!("flag context must serialize to a map"))?; + let mut scope = rhai::Scope::new(); + for (name, value) in context { + scope.push_dynamic(name.to_string(), value); + } + let output = self + .rhai + .eval_ast_with_scope::(&mut scope, &call.ast) + .wrap_err("running script")?; + let flags = output + .into_iter() + .map(|value| { + rhai::serde::from_dynamic::(&value) + .wrap_err("converting project flag to json") + }) + .collect::>>()?; + Ok(ProjectFlagEvaluation { + flags, + log: log.take(), + }) + } + pub async fn scan_flags_with_log( &self, call: &ProjectRuleCall, @@ -151,108 +322,23 @@ impl ProjectFlagEngine { tokio::time::timeout( Duration::from_millis(ENV.PROJECT_FLAG_RULE_TIMEOUT), async { - let project = - DBProject::get_id(project_id, &*self.ro_pool, &self.redis) - .await - .wrap_err("fetching project for flag evaluation")? - .ok_or_else(|| eyre!("project not found"))?; - let team_id = project.inner.team_id; - let organization = match project.inner.organization_id { - Some(id) => Some( - DBOrganization::get_id(id, &*self.ro_pool, &self.redis) - .await - .wrap_err("fetching project organization for flag evaluation")? - .ok_or_else(|| eyre!("project organization not found"))?, - ), - None => None, - }; - let mut team_ids = vec![team_id]; - if let Some(organization) = &organization { - team_ids.push(organization.team_id); - } - let members = DBTeamMember::get_from_team_full_many( - &team_ids, - &*self.ro_pool, - &self.redis, - ) - .await - .wrap_err("fetching team members for flag evaluation")?; - let mut user_ids = members.iter().map(|member| member.user_id).collect::>(); - user_ids.sort_unstable_by_key(|id| id.0); - user_ids.dedup(); - let users = DBUser::get_many_ids(&user_ids, &*self.ro_pool, &self.redis) - .await - .wrap_err("fetching team users for flag evaluation")? - .into_iter() - .map(|user| (user.id, User::from(user))) - .collect::>(); - let mut project_members = Vec::new(); - let mut organization_members = Vec::new(); - for member in members { - let member_team_id = member.team_id; - let user = users.get(&member.user_id) - .cloned() - .ok_or_else(|| eyre!("team member user not found"))?; - let member = TeamMember::from_model(member, user, true); - if member_team_id == team_id { - project_members.push(member); - } else { - organization_members.push(member); - } - } - let users = project_members.iter().map(|member| member.user.clone()).collect(); - let team = FlagTeam { - id: team_id.into(), - members: project_members, - }; - let organization = organization.map(|organization| { - Organization::from(organization, organization_members) - }); - let project = Project::from(project); - let scanned_files = scan_project_files( - project_id, - &call.scan_files_for, - &self.ro_pool, - self.file_host.as_ref(), - ) - .await?; - let context = FlagContext { - project, - team, - users, - organization, - scanned_files, - }; - let rhai = Arc::clone(&self.rhai); - let ast = call.ast.clone(); + let context = self + .load_contexts(&[project_id]) + .await? + .remove(&project_id) + .ok_or_else(|| eyre!("project not found"))?; + let files = self + .load_files(&[project_id]) + .await? + .remove(&project_id) + .unwrap_or_default(); + let context = self + .prepare_context(context, &files, &call.scan_files_for) + .await?; + let engine = self.clone(); + let call = call.clone(); tokio::task::spawn_blocking(move || { - let log = EvaluationLog::start(); - let context = rhai::serde::to_dynamic(context) - .wrap_err("serializing flag context for script")? - .try_cast::() - .ok_or_else(|| { - eyre!("flag context must serialize to a map") - })?; - let mut scope = rhai::Scope::new(); - for (name, value) in context { - scope.push_dynamic(name.to_string(), value); - } - let output = rhai - .eval_ast_with_scope::(&mut scope, &ast) - .wrap_err("running script")?; - let flags = output - .into_iter() - .map(|value| { - rhai::serde::from_dynamic::( - &value, - ) - .wrap_err("converting project flag to json") - }) - .collect::>>()?; - Ok(ProjectFlagEvaluation { - flags, - log: log.take(), - }) + engine.evaluate(&call, &context) }) .await .wrap_err("joining project flag evaluation")? diff --git a/apps/labrinth/src/queue/moderation/project_flag_files.rs b/apps/labrinth/src/queue/moderation/project_flag_files.rs index e38a14036e8..d32304c22d3 100644 --- a/apps/labrinth/src/queue/moderation/project_flag_files.rs +++ b/apps/labrinth/src/queue/moderation/project_flag_files.rs @@ -1,5 +1,6 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::io::{Cursor, Read}; +use std::sync::Arc; use eyre::{Context, Result, ensure}; use regex::Regex; @@ -44,38 +45,74 @@ impl Budget { } } -pub(super) async fn scan_project_files( - project_id: DBProjectId, - patterns: &BTreeMap, +pub(super) struct ProjectScanFile { + pub file_id: DBFileId, + pub version_id: DBVersionId, + pub filename: String, + pub url: String, + pub size: i64, +} + +pub(super) async fn load_project_files( + project_ids: &[DBProjectId], ro_pool: &ReadOnlyPgPool, - file_host: &dyn FileHost, -) -> Result> { - let files = sqlx::query!( +) -> Result>> { + let mut files = project_ids + .iter() + .copied() + .map(|id| (id, Vec::new())) + .collect::>(); + if project_ids.is_empty() { + return Ok(files); + } + let ids = project_ids.iter().map(|id| id.0).collect::>(); + let rows = sqlx::query!( r#" - SELECT f.id, f.version_id, f.filename, f.url, f.size + SELECT v.mod_id AS project_id, f.id, f.version_id, f.filename, f.url, f.size FROM files f INNER JOIN versions v ON v.id = f.version_id - WHERE v.mod_id = $1 - ORDER BY v.date_published DESC, v.id DESC, f.id + WHERE v.mod_id = ANY($1) + ORDER BY v.mod_id, v.date_published DESC, v.id DESC, f.id "#, - project_id.0 + &ids ) .fetch_all(&**ro_pool) .await .wrap_err("fetching project files for content scanning")?; + for row in rows { + files.entry(DBProjectId(row.project_id)).or_default().push( + ProjectScanFile { + file_id: DBFileId(row.id), + version_id: DBVersionId(row.version_id), + filename: row.filename, + url: row.url, + size: i64::from(row.size), + }, + ); + } + Ok(files) +} + +pub(super) async fn scan_project_files( + files: &[ProjectScanFile], + patterns: &BTreeMap, + file_host: &dyn FileHost, + downloads: &tokio::sync::Semaphore, +) -> Result> { let mut scanned_files = Vec::new(); let mut budget = Budget::default(); + let patterns = Arc::new(patterns.clone()); for file in files { ensure!( file.size >= 0 && file.size as u64 <= MAX_ARTIFACT_BYTES, "file {} exceeds declared artifact size budget", - file.id + file.file_id.0 ); ensure!( file.size as u64 <= MAX_TOTAL_BYTES - budget.bytes, "file {} exceeds remaining declared byte budget", - file.id + file.file_id.0 ); let cdn_prefix = format!("{}/", ENV.CDN_URL.trim_end_matches('/')); let key = file.url.strip_prefix(&cdn_prefix).ok_or_else(|| { @@ -89,19 +126,24 @@ pub(super) async fn scan_project_files( && key.split('/').all(|part| !matches!(part, "" | "." | "..")), "invalid project file storage key" ); + let permit = downloads + .acquire() + .await + .wrap_err("acquiring project file download permit")?; let data = file_host .read_file(&key, FileHostPublicity::Public) .await .wrap_err_with(|| { - format!("reading project file {} from storage", file.id) + format!("reading project file {} from storage", file.file_id.0) })?; + drop(permit); ensure!( data.len() as u64 <= MAX_ARTIFACT_BYTES, "project file exceeds artifact size budget" ); budget.consume(data.len() as u64)?; - let path = format!("{}/{}", file.version_id, file.filename); - let patterns = patterns.clone(); + let path = format!("{}/{}", file.version_id.0, file.filename); + let patterns = Arc::clone(&patterns); let (artifact_leaves, next_budget) = tokio::task::spawn_blocking(move || { let mut leaves = Vec::new(); @@ -120,8 +162,8 @@ pub(super) async fn scan_project_files( budget = next_budget; scanned_files.extend(artifact_leaves.into_iter().map( |(file_path, sha256, found_patterns)| ScannedFile { - version_id: DBVersionId(file.version_id).into(), - file_id: DBFileId(file.id).into(), + version_id: file.version_id.into(), + file_id: file.file_id.into(), file_path, sha256, found_patterns, diff --git a/apps/labrinth/src/queue/moderation/project_flag_run.rs b/apps/labrinth/src/queue/moderation/project_flag_run.rs index 405e96debdc..fa3f02aada6 100644 --- a/apps/labrinth/src/queue/moderation/project_flag_run.rs +++ b/apps/labrinth/src/queue/moderation/project_flag_run.rs @@ -1,21 +1,263 @@ +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + sync::Arc, + time::Duration, +}; + +use super::project_flag::{ProjectFlagEngine, ProjectRuleCall}; use eyre::{Result, WrapErr, eyre}; use futures::{StreamExt, TryStreamExt, stream}; -use rdkafka::producer::{FutureRecord, Producer}; +use rdkafka::{ + Message, + consumer::{Consumer, StreamConsumer}, + producer::{FutureRecord, Producer}, +}; +use regex::Regex; use serde::{Deserialize, Serialize}; use xredis::RedisPool; use crate::{ + database::ReadOnlyPgPool, database::models::{ - DBProjectId, ProjectFlagRunId, project_flag_item::DBProjectFlagRule, + DBProjectId, ProjectFlagRuleId, ProjectFlagRunId, + project_flag_item::{DBProjectFlagRule, PatternName}, }, + env::ENV, + file_hosting::FileHost, models::ids::ProjectId, util::kafka::{ KAFKA_DELIVERY_TIMEOUT, KAFKA_OPERATION_INTERVAL, KafkaClientState, - KafkaEvent, create_project_flag_run_producer, sentry_trace_headers, + KafkaEvent, consumer_config, create_project_flag_run_producer, + sentry_trace_headers, }, }; pub const PROJECT_FLAG_RUN_TOPIC: &str = "private.labrinth.project-flag-run.v1"; +pub const PROJECT_FLAG_RUN_CONSUMER_GROUP: &str = "project-flag-runs"; +const SCAN_BATCH_DELAY: Duration = Duration::from_secs(1); + +/// Executes run jobs without acknowledging them until result persistence is implemented. +pub async fn consume_project_flag_runs( + ro_pool: ReadOnlyPgPool, + redis: RedisPool, + file_host: Arc, +) -> Result<()> { + let engine = ProjectFlagEngine::new(ro_pool, redis.clone(), file_host)?; + let batch_size = ENV.PROJECT_FLAG_SCAN_BATCH_SIZE.get(); + let consumer: StreamConsumer = + consumer_config(PROJECT_FLAG_RUN_CONSUMER_GROUP) + .set("isolation.level", "read_committed") + .set("enable.auto.offset.store", "false") + .create() + .wrap_err("creating project flag run Kafka consumer")?; + consumer + .subscribe(&[PROJECT_FLAG_RUN_TOPIC]) + .wrap_err("subscribing to project flag run Kafka topic")?; + + tracing::warn!( + topic = PROJECT_FLAG_RUN_TOPIC, + consumer_group = PROJECT_FLAG_RUN_CONSUMER_GROUP, + batch_size, + "Started project flag run consumer; result persistence and offset commits are disabled" + ); + + let mut messages = Vec::with_capacity(batch_size); + loop { + messages.clear(); + let result = async { + messages.push(consumer.recv().await.wrap_err( + "receiving initial project flag run Kafka message", + )?); + let deadline = tokio::time::Instant::now() + SCAN_BATCH_DELAY; + while messages.len() < batch_size { + match tokio::time::timeout_at(deadline, consumer.recv()).await { + Ok(message) => messages.push(message.wrap_err( + "receiving project flag run Kafka batch message", + )?), + Err(_) => break, + } + } + + let events = messages + .iter() + .map(|message| { + let payload = message.payload().ok_or_else(|| { + eyre!("project flag run Kafka message has no payload") + })?; + serde_json::from_slice::(payload) + .wrap_err("decoding project flag run Kafka message") + }) + .collect::>>()?; + + scan_batch(&engine, &redis, events).await?; + eyre::Ok(()) + } + .await; + if let Err(error) = result { + tracing::error!(?error, "Project flag run consumer failed"); + tokio::time::sleep(KAFKA_OPERATION_INTERVAL).await; + } + } +} + +struct PreparedRule { + id: ProjectFlagRuleId, + call: ProjectRuleCall, + pattern_names: BTreeMap, +} + +struct PreparedRun { + rules: Vec, + patterns: BTreeMap, +} + +async fn scan_batch( + engine: &ProjectFlagEngine, + redis: &RedisPool, + events: Vec, +) -> Result<()> { + let mut snapshots = HashMap::new(); + let mut connection = redis + .connect() + .await + .wrap_err("connecting to Redis for project flag rule snapshots")?; + for run_id in events.iter().map(|event| event.run_id) { + if snapshots.contains_key(&run_id) { + continue; + } + let key = connection + .key() + .entity("project_flag_run_rules:v1", run_id.0); + let snapshot: Option> = redis::cmd("GET") + .arg(key) + .query_async(&mut connection) + .await + .wrap_err("fetching project flag rule snapshot")?; + let snapshot = snapshot.ok_or_else(|| { + eyre!("rule snapshot missing for run {}", run_id.0) + })?; + let rules: Vec = + serde_json::from_slice(&snapshot) + .wrap_err("decoding project flag rule snapshot")?; + snapshots.insert(run_id, rules); + } + drop(connection); + + let compile_engine = engine.clone(); + let runs = tokio::task::spawn_blocking(move || { + snapshots + .into_iter() + .map(|(run_id, rules)| { + let mut prepared = PreparedRun { + rules: Vec::new(), + patterns: BTreeMap::new(), + }; + for rule in rules { + let call = compile_engine + .make_call(&rule.params) + .wrap_err_with(|| { + format!("compiling project flag rule {}", rule.id.0) + })?; + let mut pattern_names = BTreeMap::new(); + for (name, regex) in &call.scan_files_for { + let internal = + PatternName(format!("{}:{}", rule.id.0, name.0)); + prepared + .patterns + .insert(internal.clone(), regex.clone()); + pattern_names.insert(internal, name.clone()); + } + prepared.rules.push(PreparedRule { + id: rule.id, + call, + pattern_names, + }); + } + eyre::Ok((run_id, prepared)) + }) + .collect::>>() + }) + .await + .wrap_err("joining project flag rule compilation")??; + let runs = Arc::new(runs); + let project_ids = events + .iter() + .map(|event| DBProjectId::from(event.project_id)) + .collect::>() + .into_iter() + .collect::>(); + let (contexts, files) = tokio::try_join!( + engine.load_contexts(&project_ids), + engine.load_files(&project_ids), + )?; + + let prepared = + futures::future::try_join_all(events.into_iter().map(|event| { + let contexts = &contexts; + let files = &files; + let runs = &runs; + async move { + let project_id = DBProjectId::from(event.project_id); + let context = contexts + .get(&project_id) + .ok_or_else(|| { + eyre!( + "project context missing for {}", + event.project_id + ) + })? + .clone(); + let run = runs.get(&event.run_id).ok_or_else(|| { + eyre!("rule snapshot missing for run {}", event.run_id.0) + })?; + let files = files + .get(&project_id) + .map(Vec::as_slice) + .unwrap_or_default(); + let context = engine + .prepare_context(context, files, &run.patterns) + .await + .wrap_err_with(|| { + format!( + "preparing project {} for run {}", + event.project_id, event.run_id.0 + ) + })?; + eyre::Ok((event, context)) + } + })) + .await?; + + let tasks = prepared.into_iter().map(|(event, mut context)| { + let engine = engine.clone(); + let runs = runs.clone(); + tokio::task::spawn_blocking(move || { + let original_patterns = context.scanned_files.iter() + .map(|file| file.found_patterns.clone()).collect::>(); + for rule in &runs[&event.run_id].rules { + for (file, names) in context.scanned_files.iter_mut().zip(&original_patterns) { + file.found_patterns = names.iter() + .filter_map(|name| rule.pattern_names.get(name).cloned()).collect(); + } + match engine.evaluate(&rule.call, &context) { + Ok(result) => tracing::info!( + run_id = %event.run_id.0, project_id = %event.project_id, + rule_id = rule.id.0, flags = ?result.flags, log = ?result.log, + "Evaluated project flag rule; results are not persisted" + ), + Err(error) => tracing::error!( + run_id = %event.run_id.0, project_id = %event.project_id, + rule_id = rule.id.0, ?error, "Project flag rule evaluation failed" + ), + } + } + }) + }).collect::>(); + for result in futures::future::join_all(tasks).await { + result.wrap_err("joining project flag batch evaluation")?; + } + Ok(()) +} #[derive(Debug, Serialize, Deserialize)] pub struct ProjectFlagRunEvent { diff --git a/apps/labrinth/src/util/kafka.rs b/apps/labrinth/src/util/kafka.rs index a2785db002e..0ab953629e0 100644 --- a/apps/labrinth/src/util/kafka.rs +++ b/apps/labrinth/src/util/kafka.rs @@ -87,16 +87,22 @@ pub fn producer_config() -> ClientConfig { } pub fn create_consumer(group_id: &str) -> eyre::Result { - ClientConfig::new() + consumer_config(group_id) + .create() + .wrap_err("failed to create Kafka consumer") +} + +pub fn consumer_config(group_id: &str) -> ClientConfig { + let mut config = ClientConfig::new(); + config .set("bootstrap.servers", ENV.KAFKA_BOOTSTRAP_SERVERS.0.join(",")) .set("client.id", &ENV.KAFKA_CLIENT_ID) .set("group.id", group_id) .set("enable.auto.commit", "false") .set("auto.offset.reset", "earliest") .set("max.poll.interval.ms", "1800000") - .set("broker.address.family", "v4") - .create() - .wrap_err("failed to create Kafka consumer") + .set("broker.address.family", "v4"); + config } /// Sentry distributed tracing headers for the active span, so consumers can From 660fc91c9f951d301188748c7a6bc81e85966727 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:08:01 +0000 Subject: [PATCH 09/26] persist flag run results in db --- .../20261028145153_project_flags_backend.sql | 11 +++ apps/labrinth/src/background_task.rs | 2 +- .../src/database/models/project_flag_item.rs | 58 ++++++++++++- .../src/queue/moderation/project_flag.rs | 13 +-- .../src/queue/moderation/project_flag_run.rs | 86 ++++++++++++------- 5 files changed, 134 insertions(+), 36 deletions(-) diff --git a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql index 35e370b8987..8f1045d35ad 100644 --- a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql +++ b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql @@ -23,3 +23,14 @@ CREATE TABLE project_flag_runs ( output JSONB, error TEXT ); + +CREATE TABLE project_flag_run_results ( + run_id UUID NOT NULL REFERENCES project_flag_runs(id) ON DELETE CASCADE, + project_id BIGINT NOT NULL REFERENCES mods(id) ON DELETE CASCADE, + rule_id BIGINT NOT NULL, + flags JSONB NOT NULL DEFAULT '[]'::jsonb, + log TEXT[] NOT NULL DEFAULT '{}', + error TEXT, + evaluated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (run_id, project_id, rule_id) +); diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs index ff0de237eb6..d486a405f27 100644 --- a/apps/labrinth/src/background_task.rs +++ b/apps/labrinth/src/background_task.rs @@ -93,7 +93,7 @@ impl BackgroundTask { } ProjectFlagRuns => { crate::queue::moderation::project_flag_run::consume_project_flag_runs( - ro_pool.into(), redis_pool, file_host.into_inner(), + pool, ro_pool.into(), redis_pool, file_host.into_inner(), ).await } Mail => run_email(email_queue).await, diff --git a/apps/labrinth/src/database/models/project_flag_item.rs b/apps/labrinth/src/database/models/project_flag_item.rs index 80cebbef9fb..961ef62ed0a 100644 --- a/apps/labrinth/src/database/models/project_flag_item.rs +++ b/apps/labrinth/src/database/models/project_flag_item.rs @@ -231,11 +231,67 @@ impl DBProjectFlagRule { } } +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProjectFlag { + #[serde(default)] + pub retain_if_removed: bool, + #[serde(flatten)] + pub value: serde_json::Value, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DBProjectFlag { pub id: ProjectFlagId, pub rule_id: Option, - pub what: serde_json::Value, + pub what: ProjectFlag, +} + +#[derive(Debug, Serialize)] +pub struct DBProjectFlagRunResult { + pub run_id: ProjectFlagRunId, + pub project_id: super::DBProjectId, + pub rule_id: ProjectFlagRuleId, + pub flags: Vec, + pub log: Vec, + pub error: Option, +} + +impl DBProjectFlagRunResult { + pub async fn upsert_many(results: Vec, pool: &PgPool) -> Result<()> { + if results.is_empty() { + return Ok(()); + } + let results = results + .into_iter() + .map(|result| { + ((result.run_id, result.project_id, result.rule_id), result) + }) + .collect::>() + .into_values() + .collect::>(); + let results = serde_json::to_value(results) + .wrap_err("serializing project flag run results")?; + sqlx::query!( + r#" + INSERT INTO project_flag_run_results + (run_id, project_id, rule_id, flags, log, error) + SELECT run_id, project_id, rule_id, flags, log, error + FROM jsonb_to_recordset($1) AS results( + run_id UUID, project_id BIGINT, rule_id BIGINT, + flags JSONB, log TEXT[], error TEXT + ) + ORDER BY run_id, project_id, rule_id + ON CONFLICT (run_id, project_id, rule_id) DO UPDATE + SET flags = EXCLUDED.flags, log = EXCLUDED.log, + error = EXCLUDED.error, evaluated_at = CURRENT_TIMESTAMP + "#, + results, + ) + .execute(pool) + .await + .wrap_err("persisting project flag run results")?; + Ok(()) + } } #[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/apps/labrinth/src/queue/moderation/project_flag.rs b/apps/labrinth/src/queue/moderation/project_flag.rs index d764dde7934..384ca52901a 100644 --- a/apps/labrinth/src/queue/moderation/project_flag.rs +++ b/apps/labrinth/src/queue/moderation/project_flag.rs @@ -18,7 +18,7 @@ use crate::{ ReadOnlyPgPool, models::{ DBOrganization, DBProject, DBProjectId, DBTeamMember, DBUser, - project_flag_item::{PatternName, ProjectRuleParams}, + project_flag_item::{PatternName, ProjectFlag, ProjectRuleParams}, }, }, env::ENV, @@ -63,7 +63,7 @@ impl Drop for EvaluationLog { #[derive(serde::Serialize, utoipa::ToSchema)] pub struct ProjectFlagEvaluation { - pub flags: Vec, + pub flags: Vec, pub log: Vec, } @@ -304,8 +304,11 @@ impl ProjectFlagEngine { let flags = output .into_iter() .map(|value| { - rhai::serde::from_dynamic::(&value) - .wrap_err("converting project flag to json") + let value = + rhai::serde::from_dynamic::(&value) + .wrap_err("converting project flag to json")?; + serde_json::from_value::(value) + .wrap_err("decoding project flag") }) .collect::>>()?; Ok(ProjectFlagEvaluation { @@ -352,7 +355,7 @@ impl ProjectFlagEngine { &self, call: &ProjectRuleCall, project_id: DBProjectId, - ) -> Result> { + ) -> Result> { Ok(self.scan_flags_with_log(call, project_id).await?.flags) } } diff --git a/apps/labrinth/src/queue/moderation/project_flag_run.rs b/apps/labrinth/src/queue/moderation/project_flag_run.rs index fa3f02aada6..b0b30e55597 100644 --- a/apps/labrinth/src/queue/moderation/project_flag_run.rs +++ b/apps/labrinth/src/queue/moderation/project_flag_run.rs @@ -17,11 +17,13 @@ use serde::{Deserialize, Serialize}; use xredis::RedisPool; use crate::{ - database::ReadOnlyPgPool, database::models::{ DBProjectId, ProjectFlagRuleId, ProjectFlagRunId, - project_flag_item::{DBProjectFlagRule, PatternName}, + project_flag_item::{ + DBProjectFlagRule, DBProjectFlagRunResult, PatternName, + }, }, + database::{PgPool, ReadOnlyPgPool}, env::ENV, file_hosting::FileHost, models::ids::ProjectId, @@ -36,8 +38,9 @@ pub const PROJECT_FLAG_RUN_TOPIC: &str = "private.labrinth.project-flag-run.v1"; pub const PROJECT_FLAG_RUN_CONSUMER_GROUP: &str = "project-flag-runs"; const SCAN_BATCH_DELAY: Duration = Duration::from_secs(1); -/// Executes run jobs without acknowledging them until result persistence is implemented. +/// Executes and persists run jobs; offset commits are not yet implemented. pub async fn consume_project_flag_runs( + pool: PgPool, ro_pool: ReadOnlyPgPool, redis: RedisPool, file_host: Arc, @@ -58,7 +61,7 @@ pub async fn consume_project_flag_runs( topic = PROJECT_FLAG_RUN_TOPIC, consumer_group = PROJECT_FLAG_RUN_CONSUMER_GROUP, batch_size, - "Started project flag run consumer; result persistence and offset commits are disabled" + "Started project flag run consumer; offset commits are disabled" ); let mut messages = Vec::with_capacity(batch_size); @@ -89,7 +92,7 @@ pub async fn consume_project_flag_runs( }) .collect::>>()?; - scan_batch(&engine, &redis, events).await?; + scan_batch(&engine, &redis, &pool, events).await?; eyre::Ok(()) } .await; @@ -114,6 +117,7 @@ struct PreparedRun { async fn scan_batch( engine: &ProjectFlagEngine, redis: &RedisPool, + pool: &PgPool, events: Vec, ) -> Result<()> { let mut snapshots = HashMap::new(); @@ -228,34 +232,58 @@ async fn scan_batch( })) .await?; - let tasks = prepared.into_iter().map(|(event, mut context)| { - let engine = engine.clone(); - let runs = runs.clone(); - tokio::task::spawn_blocking(move || { - let original_patterns = context.scanned_files.iter() - .map(|file| file.found_patterns.clone()).collect::>(); - for rule in &runs[&event.run_id].rules { - for (file, names) in context.scanned_files.iter_mut().zip(&original_patterns) { - file.found_patterns = names.iter() - .filter_map(|name| rule.pattern_names.get(name).cloned()).collect(); - } - match engine.evaluate(&rule.call, &context) { - Ok(result) => tracing::info!( - run_id = %event.run_id.0, project_id = %event.project_id, - rule_id = rule.id.0, flags = ?result.flags, log = ?result.log, - "Evaluated project flag rule; results are not persisted" - ), - Err(error) => tracing::error!( - run_id = %event.run_id.0, project_id = %event.project_id, - rule_id = rule.id.0, ?error, "Project flag rule evaluation failed" - ), + let tasks = prepared + .into_iter() + .map(|(event, mut context)| { + let engine = engine.clone(); + let runs = runs.clone(); + tokio::task::spawn_blocking(move || { + let original_patterns = context + .scanned_files + .iter() + .map(|file| file.found_patterns.clone()) + .collect::>(); + let mut results = Vec::new(); + for rule in &runs[&event.run_id].rules { + for (file, names) in + context.scanned_files.iter_mut().zip(&original_patterns) + { + file.found_patterns = names + .iter() + .filter_map(|name| { + rule.pattern_names.get(name).cloned() + }) + .collect(); + } + let (flags, log, error) = match engine + .evaluate(&rule.call, &context) + { + Ok(result) => (result.flags, result.log, None), + Err(error) => { + (Vec::new(), Vec::new(), Some(format!("{error:#}"))) + } + }; + results.push(DBProjectFlagRunResult { + run_id: event.run_id, + project_id: event.project_id.into(), + rule_id: rule.id, + flags, + log, + error, + }); } - } + results + }) }) - }).collect::>(); + .collect::>(); + let mut results = Vec::new(); for result in futures::future::join_all(tasks).await { - result.wrap_err("joining project flag batch evaluation")?; + results + .extend(result.wrap_err("joining project flag batch evaluation")?); } + let result_count = results.len(); + DBProjectFlagRunResult::upsert_many(results, pool).await?; + tracing::info!(result_count, "Persisted project flag run batch results"); Ok(()) } From 6b81dc1a5c1d2bde5c25b4db9ed6130c8cfcbce9 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:00:29 +0000 Subject: [PATCH 10/26] run rule result persistence --- .../20261028145153_project_flags_backend.sql | 25 ++-- .../src/database/models/project_flag_item.rs | 123 +++++++++++---- .../src/queue/moderation/project_flag_run.rs | 140 +++++++++++++----- .../internal/moderation/project_flag/mod.rs | 9 +- 4 files changed, 217 insertions(+), 80 deletions(-) diff --git a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql index 8f1045d35ad..29a5a5ee11d 100644 --- a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql +++ b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql @@ -15,22 +15,25 @@ CREATE TABLE project_flags ( ); CREATE TABLE project_flag_runs ( - id UUID PRIMARY KEY, - status TEXT NOT NULL, - started_by BIGINT REFERENCES users(id) ON DELETE SET NULL, - started_at TIMESTAMPTZ NOT NULL, - completed_at TIMESTAMPTZ, - output JSONB, - error TEXT + id UUID PRIMARY KEY, + status TEXT NOT NULL, + total_projects BIGINT NOT NULL, + started_by BIGINT REFERENCES users(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL, + completed_at TIMESTAMPTZ, + output JSONB, + error TEXT ); CREATE TABLE project_flag_run_results ( run_id UUID NOT NULL REFERENCES project_flag_runs(id) ON DELETE CASCADE, project_id BIGINT NOT NULL REFERENCES mods(id) ON DELETE CASCADE, - rule_id BIGINT NOT NULL, - flags JSONB NOT NULL DEFAULT '[]'::jsonb, - log TEXT[] NOT NULL DEFAULT '{}', + rules JSONB NOT NULL DEFAULT '{}'::jsonb, error TEXT, evaluated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (run_id, project_id, rule_id) + PRIMARY KEY (run_id, project_id) ); + +CREATE INDEX project_flag_run_results_errors + ON project_flag_run_results (run_id) + WHERE error IS NOT NULL; diff --git a/apps/labrinth/src/database/models/project_flag_item.rs b/apps/labrinth/src/database/models/project_flag_item.rs index 961ef62ed0a..c33116cd848 100644 --- a/apps/labrinth/src/database/models/project_flag_item.rs +++ b/apps/labrinth/src/database/models/project_flag_item.rs @@ -246,13 +246,18 @@ pub struct DBProjectFlag { pub what: ProjectFlag, } +#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] +pub struct ProjectFlagRuleResult { + pub flags: Vec, + pub log: Vec, + pub error: Option, +} + #[derive(Debug, Serialize)] pub struct DBProjectFlagRunResult { pub run_id: ProjectFlagRunId, pub project_id: super::DBProjectId, - pub rule_id: ProjectFlagRuleId, - pub flags: Vec, - pub log: Vec, + pub rules: BTreeMap, pub error: Option, } @@ -261,35 +266,94 @@ impl DBProjectFlagRunResult { if results.is_empty() { return Ok(()); } + let mut run_ids = results + .iter() + .map(|result| result.run_id.0) + .collect::>(); + run_ids.sort_unstable(); + run_ids.dedup(); let results = results .into_iter() - .map(|result| { - ((result.run_id, result.project_id, result.rule_id), result) + .map(|mut result| { + if result.error.is_none() { + let errors = result + .rules + .iter() + .filter_map(|(rule_id, rule)| { + rule.error + .as_ref() + .map(|error| format!("{rule_id}: {error}")) + }) + .collect::>(); + if !errors.is_empty() { + result.error = Some(errors.join("\n")); + } + } + ((result.run_id, result.project_id), result) }) .collect::>() .into_values() .collect::>(); let results = serde_json::to_value(results) .wrap_err("serializing project flag run results")?; + let mut transaction = pool + .begin() + .await + .wrap_err("starting project flag results transaction")?; + sqlx::query!( + r#" + SELECT id + FROM project_flag_runs + WHERE id = ANY($1) + ORDER BY id + FOR UPDATE + "#, + &run_ids, + ) + .fetch_all(&mut transaction) + .await + .wrap_err("locking project flag runs")?; sqlx::query!( r#" INSERT INTO project_flag_run_results - (run_id, project_id, rule_id, flags, log, error) - SELECT run_id, project_id, rule_id, flags, log, error + (run_id, project_id, rules, error) + SELECT results.run_id, results.project_id, results.rules, results.error FROM jsonb_to_recordset($1) AS results( - run_id UUID, project_id BIGINT, rule_id BIGINT, - flags JSONB, log TEXT[], error TEXT + run_id UUID, project_id BIGINT, rules JSONB, error TEXT ) - ORDER BY run_id, project_id, rule_id - ON CONFLICT (run_id, project_id, rule_id) DO UPDATE - SET flags = EXCLUDED.flags, log = EXCLUDED.log, + INNER JOIN project_flag_runs AS runs ON runs.id = results.run_id + WHERE runs.status = 'running' + ORDER BY results.run_id, results.project_id + ON CONFLICT (run_id, project_id) DO UPDATE + SET rules = EXCLUDED.rules, error = EXCLUDED.error, evaluated_at = CURRENT_TIMESTAMP "#, results, ) - .execute(pool) + .execute(&mut transaction) .await .wrap_err("persisting project flag run results")?; + sqlx::query!( + r#" + UPDATE project_flag_runs AS runs + SET status = 'done', completed_at = CURRENT_TIMESTAMP + WHERE runs.id = ANY($1) + AND runs.status = 'running' + AND ( + SELECT COUNT(*) + FROM project_flag_run_results AS results + WHERE results.run_id = runs.id + ) >= runs.total_projects + "#, + &run_ids, + ) + .execute(&mut transaction) + .await + .wrap_err("completing project flag runs")?; + transaction + .commit() + .await + .wrap_err("committing project flag run results")?; Ok(()) } } @@ -298,6 +362,8 @@ impl DBProjectFlagRunResult { pub struct DBProjectFlagRun { pub id: ProjectFlagRunId, pub status: ProjectFlagRunStatus, + pub total_projects: i64, + pub num_errors: i64, pub started_by: Option, pub started_at: DateTime, pub completed_at: Option>, @@ -318,10 +384,16 @@ impl DBProjectFlagRun { Ok(ids.into_iter().map(super::DBProjectId).collect()) } - pub async fn create(user_id: DBUserId, pool: &PgPool) -> Result { + pub async fn create( + user_id: DBUserId, + total_projects: i64, + pool: &PgPool, + ) -> Result { let run = Self { id: ProjectFlagRunId(uuid::Uuid::now_v7()), status: ProjectFlagRunStatus::Running, + total_projects, + num_errors: 0, started_by: Some(user_id), started_at: Utc::now(), completed_at: None, @@ -330,12 +402,13 @@ impl DBProjectFlagRun { }; sqlx::query!( r#" - INSERT INTO project_flag_runs (id, status, started_by, started_at) - VALUES ($1, 'running', $2, $3) + INSERT INTO project_flag_runs (id, status, started_by, started_at, total_projects) + VALUES ($1, 'running', $2, $3, $4) "#, run.id.0, user_id.0, run.started_at, + total_projects, ) .execute(pool) .await @@ -348,11 +421,7 @@ impl DBProjectFlagRun { error: Option, pool: &PgPool, ) -> Result<()> { - let status = if error.is_some() { - ProjectFlagRunStatus::Err - } else { - ProjectFlagRunStatus::Ok - }; + let status = ProjectFlagRunStatus::Done; let completed_at = Utc::now(); sqlx::query!( r#" @@ -381,9 +450,12 @@ impl DBProjectFlagRun { ) -> Result> { let rows = sqlx::query!( r#" - SELECT id, status AS "status: ProjectFlagRunStatus", started_by, + SELECT id, status AS "status: ProjectFlagRunStatus", total_projects, started_by, started_at, completed_at, - output AS "output: sqlx::types::Json", error + output AS "output: sqlx::types::Json", error, + (SELECT COUNT(*) FROM project_flag_run_results AS results + WHERE results.run_id = project_flag_runs.id AND results.error IS NOT NULL + ) AS "num_errors!" FROM project_flag_runs ORDER BY started_at DESC, id DESC LIMIT $1 OFFSET $2 @@ -400,6 +472,8 @@ impl DBProjectFlagRun { .map(|row| Self { id: ProjectFlagRunId(row.id), status: row.status, + total_projects: row.total_projects, + num_errors: row.num_errors, started_by: row.started_by.map(DBUserId), started_at: row.started_at, completed_at: row.completed_at, @@ -425,8 +499,7 @@ impl DBProjectFlagRun { #[sqlx(type_name = "text", rename_all = "snake_case")] pub enum ProjectFlagRunStatus { Running, - Ok, - Err, + Done, } #[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/apps/labrinth/src/queue/moderation/project_flag_run.rs b/apps/labrinth/src/queue/moderation/project_flag_run.rs index b0b30e55597..e48d882ece6 100644 --- a/apps/labrinth/src/queue/moderation/project_flag_run.rs +++ b/apps/labrinth/src/queue/moderation/project_flag_run.rs @@ -8,8 +8,8 @@ use super::project_flag::{ProjectFlagEngine, ProjectRuleCall}; use eyre::{Result, WrapErr, eyre}; use futures::{StreamExt, TryStreamExt, stream}; use rdkafka::{ - Message, - consumer::{Consumer, StreamConsumer}, + Message, Offset, TopicPartitionList, + consumer::{CommitMode, Consumer, StreamConsumer}, producer::{FutureRecord, Producer}, }; use regex::Regex; @@ -21,6 +21,7 @@ use crate::{ DBProjectId, ProjectFlagRuleId, ProjectFlagRunId, project_flag_item::{ DBProjectFlagRule, DBProjectFlagRunResult, PatternName, + ProjectFlagRuleResult, }, }, database::{PgPool, ReadOnlyPgPool}, @@ -38,7 +39,7 @@ pub const PROJECT_FLAG_RUN_TOPIC: &str = "private.labrinth.project-flag-run.v1"; pub const PROJECT_FLAG_RUN_CONSUMER_GROUP: &str = "project-flag-runs"; const SCAN_BATCH_DELAY: Duration = Duration::from_secs(1); -/// Executes and persists run jobs; offset commits are not yet implemented. +/// Persists each batch before acknowledging its Kafka offsets. pub async fn consume_project_flag_runs( pool: PgPool, ro_pool: ReadOnlyPgPool, @@ -53,6 +54,7 @@ pub async fn consume_project_flag_runs( .set("enable.auto.offset.store", "false") .create() .wrap_err("creating project flag run Kafka consumer")?; + let consumer = Arc::new(consumer); consumer .subscribe(&[PROJECT_FLAG_RUN_TOPIC]) .wrap_err("subscribing to project flag run Kafka topic")?; @@ -61,7 +63,7 @@ pub async fn consume_project_flag_runs( topic = PROJECT_FLAG_RUN_TOPIC, consumer_group = PROJECT_FLAG_RUN_CONSUMER_GROUP, batch_size, - "Started project flag run consumer; offset commits are disabled" + "Started project flag run consumer" ); let mut messages = Vec::with_capacity(batch_size); @@ -93,23 +95,50 @@ pub async fn consume_project_flag_runs( .collect::>>()?; scan_batch(&engine, &redis, &pool, events).await?; + let mut offsets = BTreeMap::new(); + for message in &messages { + offsets + .entry(message.partition()) + .and_modify(|offset: &mut i64| { + *offset = (*offset).max(message.offset() + 1) + }) + .or_insert(message.offset() + 1); + } + let mut positions = TopicPartitionList::new(); + for (partition, offset) in offsets { + positions + .add_partition_offset( + PROJECT_FLAG_RUN_TOPIC, + partition, + Offset::Offset(offset), + ) + .wrap_err("building project flag run consumer offsets")?; + } + let consumer = consumer.clone(); + tokio::task::spawn_blocking(move || { + consumer.commit(&positions, CommitMode::Sync) + }) + .await + .wrap_err("joining project flag run offset commit")? + .wrap_err("committing project flag run consumer offsets")?; eyre::Ok(()) } .await; if let Err(error) = result { tracing::error!(?error, "Project flag run consumer failed"); - tokio::time::sleep(KAFKA_OPERATION_INTERVAL).await; + return Err(error); } } } struct PreparedRule { id: ProjectFlagRuleId, - call: ProjectRuleCall, + call: Result, pattern_names: BTreeMap, } struct PreparedRun { + error: Option, rules: Vec, patterns: BTreeMap, } @@ -137,12 +166,16 @@ async fn scan_batch( .query_async(&mut connection) .await .wrap_err("fetching project flag rule snapshot")?; - let snapshot = snapshot.ok_or_else(|| { - eyre!("rule snapshot missing for run {}", run_id.0) - })?; - let rules: Vec = - serde_json::from_slice(&snapshot) - .wrap_err("decoding project flag rule snapshot")?; + let rules = snapshot + .ok_or_else(|| { + format!("rule snapshot missing for run {}", run_id.0) + }) + .and_then(|snapshot| { + serde_json::from_slice::>(&snapshot) + .map_err(|error| { + format!("decoding project flag rule snapshot: {error}") + }) + }); snapshots.insert(run_id, rules); } drop(connection); @@ -153,17 +186,23 @@ async fn scan_batch( .into_iter() .map(|(run_id, rules)| { let mut prepared = PreparedRun { + error: None, rules: Vec::new(), patterns: BTreeMap::new(), }; + let rules = match rules { + Ok(rules) => rules, + Err(error) => { + prepared.error = Some(error); + return eyre::Ok((run_id, prepared)); + } + }; for rule in rules { let call = compile_engine .make_call(&rule.params) - .wrap_err_with(|| { - format!("compiling project flag rule {}", rule.id.0) - })?; + .map_err(|error| format!("{error:#}")); let mut pattern_names = BTreeMap::new(); - for (name, regex) in &call.scan_files_for { + for (name, regex) in &rule.params.scan_files_for { let internal = PatternName(format!("{}:{}", rule.id.0, name.0)); prepared @@ -195,12 +234,12 @@ async fn scan_batch( engine.load_files(&project_ids), )?; - let prepared = - futures::future::try_join_all(events.into_iter().map(|event| { - let contexts = &contexts; - let files = &files; - let runs = &runs; - async move { + let prepared = futures::future::join_all(events.into_iter().map(|event| { + let contexts = &contexts; + let files = &files; + let runs = &runs; + async move { + let prepared = async { let project_id = DBProjectId::from(event.project_id); let context = contexts .get(&project_id) @@ -214,6 +253,9 @@ async fn scan_batch( let run = runs.get(&event.run_id).ok_or_else(|| { eyre!("rule snapshot missing for run {}", event.run_id.0) })?; + if let Some(error) = &run.error { + return Err(eyre!(error.clone())); + } let files = files .get(&project_id) .map(Vec::as_slice) @@ -227,23 +269,37 @@ async fn scan_batch( event.project_id, event.run_id.0 ) })?; - eyre::Ok((event, context)) + eyre::Ok(context) } - })) - .await?; + .await; + (event, prepared) + } + })) + .await; let tasks = prepared .into_iter() - .map(|(event, mut context)| { + .map(|(event, context)| { let engine = engine.clone(); let runs = runs.clone(); tokio::task::spawn_blocking(move || { + let mut context = match context { + Ok(context) => context, + Err(error) => { + return DBProjectFlagRunResult { + run_id: event.run_id, + project_id: event.project_id.into(), + rules: BTreeMap::new(), + error: Some(format!("{error:#}")), + }; + } + }; let original_patterns = context .scanned_files .iter() .map(|file| file.found_patterns.clone()) .collect::>(); - let mut results = Vec::new(); + let mut rules = BTreeMap::new(); for rule in &runs[&event.run_id].rules { for (file, names) in context.scanned_files.iter_mut().zip(&original_patterns) @@ -255,31 +311,33 @@ async fn scan_batch( }) .collect(); } - let (flags, log, error) = match engine - .evaluate(&rule.call, &context) - { + let evaluation = match &rule.call { + Ok(call) => engine.evaluate(call, &context), + Err(error) => Err(eyre!(error.clone())), + }; + let (flags, log, error) = match evaluation { Ok(result) => (result.flags, result.log, None), Err(error) => { (Vec::new(), Vec::new(), Some(format!("{error:#}"))) } }; - results.push(DBProjectFlagRunResult { - run_id: event.run_id, - project_id: event.project_id.into(), - rule_id: rule.id, - flags, - log, - error, - }); + rules.insert( + rule.id.0.to_string(), + ProjectFlagRuleResult { flags, log, error }, + ); + } + DBProjectFlagRunResult { + run_id: event.run_id, + project_id: event.project_id.into(), + rules, + error: None, } - results }) }) .collect::>(); let mut results = Vec::new(); for result in futures::future::join_all(tasks).await { - results - .extend(result.wrap_err("joining project flag batch evaluation")?); + results.push(result.wrap_err("joining project flag batch evaluation")?); } let result_count = results.len(); DBProjectFlagRunResult::upsert_many(results, pool).await?; diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs index 457edffe78f..019a39be9eb 100644 --- a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs @@ -350,9 +350,12 @@ pub async fn start_run( let project_ids = DBProjectFlagRun::queued_project_ids(&pool) .await .wrap_internal_err("fetching queued project IDs")?; - let mut run = DBProjectFlagRun::create(user.id.into(), &pool) - .await - .wrap_internal_err("creating project flag run")?; + let total_projects = i64::try_from(project_ids.len()) + .wrap_internal_err("counting queued project IDs")?; + let mut run = + DBProjectFlagRun::create(user.id.into(), total_projects, &pool) + .await + .wrap_internal_err("creating project flag run")?; // Keep dispatch alive if the caller disconnects while Kafka is publishing. let run = tokio::spawn(async move { From 2676af9c9308af309a716bda478513190bb0a9a2 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:36:26 +0000 Subject: [PATCH 11/26] route to fetch project flags --- .../20261028145153_project_flags_backend.sql | 3 + .../src/database/models/project_flag_item.rs | 32 ++ apps/labrinth/src/routes/internal/mod.rs | 19 +- .../internal/moderation/project_flag/mod.rs | 448 +----------------- .../moderation/project_flag/project.rs | 49 ++ .../internal/moderation/project_flag/rule.rs | 428 +++++++++++++++++ 6 files changed, 534 insertions(+), 445 deletions(-) create mode 100644 apps/labrinth/src/routes/internal/moderation/project_flag/project.rs create mode 100644 apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs diff --git a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql index 29a5a5ee11d..0c817e7804f 100644 --- a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql +++ b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql @@ -34,6 +34,9 @@ CREATE TABLE project_flag_run_results ( PRIMARY KEY (run_id, project_id) ); +CREATE INDEX project_flag_run_results_project_id + ON project_flag_run_results (project_id); + CREATE INDEX project_flag_run_results_errors ON project_flag_run_results (run_id) WHERE error IS NOT NULL; diff --git a/apps/labrinth/src/database/models/project_flag_item.rs b/apps/labrinth/src/database/models/project_flag_item.rs index c33116cd848..b8025163e1e 100644 --- a/apps/labrinth/src/database/models/project_flag_item.rs +++ b/apps/labrinth/src/database/models/project_flag_item.rs @@ -253,6 +253,8 @@ pub struct ProjectFlagRuleResult { pub error: Option, } +type RuleResults = BTreeMap; + #[derive(Debug, Serialize)] pub struct DBProjectFlagRunResult { pub run_id: ProjectFlagRunId, @@ -262,6 +264,36 @@ pub struct DBProjectFlagRunResult { } impl DBProjectFlagRunResult { + pub async fn get_project_flags( + project_id: super::DBProjectId, + pool: &ReadOnlyPgPool, + ) -> Result>> { + let rows = sqlx::query!( + r#" + SELECT results.rules AS "rules?: sqlx::types::Json" + FROM mods AS projects + LEFT JOIN project_flag_run_results AS results ON results.project_id = projects.id + WHERE projects.id = $1 + ORDER BY results.evaluated_at DESC, results.run_id DESC + "#, + project_id.0, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching project flags")?; + + if rows.is_empty() { + return Ok(None); + } + Ok(Some( + rows.into_iter() + .filter_map(|row| row.rules) + .flat_map(|rules| rules.0.into_values()) + .flat_map(|rule| rule.flags) + .collect(), + )) + } + pub async fn upsert_many(results: Vec, pool: &PgPool) -> Result<()> { if results.is_empty() { return Ok(()); diff --git a/apps/labrinth/src/routes/internal/mod.rs b/apps/labrinth/src/routes/internal/mod.rs index 266270a40a3..39c6b8674ea 100644 --- a/apps/labrinth/src/routes/internal/mod.rs +++ b/apps/labrinth/src/routes/internal/mod.rs @@ -103,15 +103,16 @@ pub fn config(cfg: &mut web::ServiceConfig) { pats::create_pat, pats::edit_pat, pats::delete_pat, - moderation::project_flag::list_runs, - moderation::project_flag::start_run, - moderation::project_flag::eval_rule, - moderation::project_flag::list_rules, - moderation::project_flag::get_rule_schema, - moderation::project_flag::get_rule, - moderation::project_flag::create_rule, - moderation::project_flag::update_rule, - moderation::project_flag::delete_rule, + moderation::project_flag::project::get_project_flags, + moderation::project_flag::rule::list_runs, + moderation::project_flag::rule::start_run, + moderation::project_flag::rule::eval_rule, + moderation::project_flag::rule::list_rules, + moderation::project_flag::rule::get_rule_schema, + moderation::project_flag::rule::get_rule, + moderation::project_flag::rule::create_rule, + moderation::project_flag::rule::update_rule, + moderation::project_flag::rule::delete_rule, moderation::get_projects, moderation::get_project_ids, moderation::get_project_meta, diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs index 019a39be9eb..01e0a0d01c0 100644 --- a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs @@ -1,440 +1,16 @@ -use actix_web::{HttpRequest, delete, get, post, put, web}; -use eyre::eyre; -use serde::Deserialize; -use validator::Validate; -use xredis::RedisPool; - -use crate::auth::check_is_moderator_from_headers; -use crate::database::models::project_flag_item::ProjectRuleParams; -use crate::database::models::{ - DBProject, DBProjectFlagRule, DBProjectFlagRun, ProjectFlagRuleId, -}; -use crate::database::{PgPool, ReadOnlyPgPool}; -use crate::models::ids::ProjectId; -use crate::models::pats::Scopes; - -use crate::queue::moderation::project_flag::{ - FlagContext, ProjectFlagEngine, ProjectFlagEvaluation, -}; -use crate::queue::session::AuthQueue; -use crate::routes::ApiError; -use crate::util::error::Context as _; -use crate::util::validate::{validate_name, validation_errors_to_string}; - +pub mod project; +pub mod rule; mod schema_example; pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig<'_>) { - cfg.service(list_rules) - .service(get_rule_schema) - .service(list_runs) - .service(start_run) - .service(get_rule) - .service(create_rule) - .service(update_rule) - .service(delete_rule) - .service(eval_rule); -} - -/// Get an example project flag rule context. -#[utoipa::path( - context_path = "/moderation/project-flag", - tag = "moderation", - security(("bearer_auth" = [])), - responses((status = OK, body = FlagContext)) -)] -#[get("/rules/schema")] -pub async fn get_rule_schema( - req: HttpRequest, - ro_pool: web::Data, - redis: web::Data, - session_queue: web::Data, -) -> Result, ApiError> { - check_is_moderator_from_headers( - &req, - &***ro_pool, - &redis, - &session_queue, - Scopes::SESSION_ACCESS, - ) - .await - .wrap_auth_err("authenticating moderator")?; - - Ok(web::Json( - schema_example::generate::() - .wrap_internal_err("generating project flag context example")?, - )) -} - -#[derive(Deserialize, utoipa::IntoParams)] -#[into_params(parameter_in = Query)] -pub struct ListRules { - #[serde(default = "default_limit")] - pub limit: u16, - #[serde(default)] - pub offset: u32, -} - -fn default_limit() -> u16 { - 100 -} - -#[utoipa::path( - context_path = "/moderation/project-flag", - tag = "moderation", - params(ListRules), - responses((status = OK, body = [DBProjectFlagRule])) -)] -#[get("/rules")] -pub async fn list_rules( - req: HttpRequest, - query: web::Query, - ro_pool: web::Data, - redis: web::Data, - session_queue: web::Data, -) -> Result>, ApiError> { - check_is_moderator_from_headers( - &req, - &***ro_pool, - &redis, - &session_queue, - Scopes::SESSION_ACCESS, - ) - .await - .wrap_auth_err("authenticating moderator")?; - - if query.limit == 0 || query.limit > 200 { - return Err(ApiError::Request(eyre!( - "`limit` must be between 1 and 200" - ))); - } - let rules = DBProjectFlagRule::list( - i64::from(query.limit), - i64::from(query.offset), - &ro_pool, - ) - .await - .wrap_internal_err("listing project flag rules")?; - Ok(web::Json(rules)) -} - -#[utoipa::path( - context_path = "/moderation/project-flag", - tag = "moderation", - params(("id" = ProjectFlagRuleId, Path)), - responses((status = OK, body = DBProjectFlagRule), (status = NOT_FOUND)) -)] -#[get("/rules/{id}")] -pub async fn get_rule( - req: HttpRequest, - path: web::Path, - ro_pool: web::Data, - redis: web::Data, - session_queue: web::Data, -) -> Result, ApiError> { - check_is_moderator_from_headers( - &req, - &***ro_pool, - &redis, - &session_queue, - Scopes::SESSION_ACCESS, - ) - .await - .wrap_auth_err("authenticating moderator")?; - - let rule = DBProjectFlagRule::get(path.into_inner(), &ro_pool) - .await - .wrap_internal_err("fetching project flag rule")? - .wrap_not_found_err("project flag rule not found")?; - Ok(web::Json(rule)) -} - -#[derive(Deserialize, Validate, utoipa::ToSchema)] -pub struct CreateRule { - #[validate( - length(min = 1, max = 1023), - custom(function = "validate_name") - )] - pub name: String, - #[validate(nested)] - pub params: ProjectRuleParams, -} - -#[utoipa::path( - context_path = "/moderation/project-flag", - tag = "moderation", - request_body = CreateRule, - responses((status = OK, body = DBProjectFlagRule)) -)] -#[post("/rules")] -pub async fn create_rule( - req: HttpRequest, - body: web::Json, - pool: web::Data, - redis: web::Data, - session_queue: web::Data, -) -> Result, ApiError> { - let user = check_is_moderator_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Scopes::SESSION_ACCESS, - ) - .await - .wrap_auth_err("authenticating moderator")?; - body.validate().map_err(|error| { - ApiError::Request(eyre!(validation_errors_to_string(error, None))) - })?; - - let rule = DBProjectFlagRule::create( - &body.name, - &body.params, - user.id.into(), - &pool, - ) - .await - .wrap_internal_err("creating project flag rule")?; - Ok(web::Json(rule)) -} - -#[utoipa::path( - context_path = "/moderation/project-flag", - tag = "moderation", - params(("id" = ProjectFlagRuleId, Path)), - request_body = CreateRule, - responses((status = OK, body = DBProjectFlagRule), (status = NOT_FOUND)) -)] -#[put("/rules/{id}")] -pub async fn update_rule( - req: HttpRequest, - path: web::Path, - body: web::Json, - pool: web::Data, - redis: web::Data, - session_queue: web::Data, -) -> Result, ApiError> { - let user = check_is_moderator_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Scopes::SESSION_ACCESS, - ) - .await - .wrap_auth_err("authenticating moderator")?; - body.validate().map_err(|error| { - ApiError::Request(eyre!(validation_errors_to_string(error, None))) - })?; - - let rule = DBProjectFlagRule::update( - path.into_inner(), - &body.name, - &body.params, - user.id.into(), - &pool, - ) - .await - .wrap_internal_err("updating project flag rule")? - .wrap_not_found_err("project flag rule not found")?; - Ok(web::Json(rule)) -} - -#[utoipa::path( - context_path = "/moderation/project-flag", - tag = "moderation", - params(("id" = ProjectFlagRuleId, Path)), - responses((status = NO_CONTENT), (status = NOT_FOUND)) -)] -#[delete("/rules/{id}")] -pub async fn delete_rule( - req: HttpRequest, - path: web::Path, - pool: web::Data, - redis: web::Data, - session_queue: web::Data, -) -> Result<(), ApiError> { - check_is_moderator_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Scopes::SESSION_ACCESS, - ) - .await - .wrap_auth_err("authenticating moderator")?; - - if !DBProjectFlagRule::delete(path.into_inner(), &pool) - .await - .wrap_internal_err("deleting project flag rule")? - { - return Err(ApiError::NotFound(eyre!("project flag rule not found"))); - } - Ok(()) -} - -#[derive(Deserialize, utoipa::IntoParams)] -#[into_params(parameter_in = Query)] -pub struct ListRuns { - #[serde(default = "default_limit")] - pub limit: u16, - #[serde(default)] - pub offset: u32, -} - -#[utoipa::path( - context_path = "/moderation/project-flag", - tag = "moderation", - params(ListRuns), - responses((status = OK, body = [DBProjectFlagRun])) -)] -#[get("/rules/run")] -pub async fn list_runs( - req: HttpRequest, - query: web::Query, - ro_pool: web::Data, - redis: web::Data, - session_queue: web::Data, -) -> Result>, ApiError> { - check_is_moderator_from_headers( - &req, - &***ro_pool, - &redis, - &session_queue, - Scopes::SESSION_ACCESS, - ) - .await - .wrap_auth_err("authenticating moderator")?; - - if query.limit == 0 || query.limit > 200 { - return Err(ApiError::Request(eyre!( - "`limit` must be between 1 and 200" - ))); - } - let runs = DBProjectFlagRun::list( - i64::from(query.limit), - i64::from(query.offset), - &ro_pool, - ) - .await - .wrap_internal_err("listing project flag runs")?; - Ok(web::Json(runs)) -} - -#[utoipa::path( - context_path = "/moderation/project-flag", - tag = "moderation", - security(("bearer_auth" = [])), - responses((status = OK, body = DBProjectFlagRun)) -)] -#[post("/rules/run")] -pub async fn start_run( - req: HttpRequest, - kafka: web::Data, - pool: web::Data, - redis: web::Data, - session_queue: web::Data, -) -> Result, ApiError> { - let user = check_is_moderator_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Scopes::SESSION_ACCESS, - ) - .await - .wrap_auth_err("authenticating moderator")?; - - let rules = DBProjectFlagRule::all(&pool) - .await - .wrap_internal_err("snapshotting project flag rules")?; - let project_ids = DBProjectFlagRun::queued_project_ids(&pool) - .await - .wrap_internal_err("fetching queued project IDs")?; - let total_projects = i64::try_from(project_ids.len()) - .wrap_internal_err("counting queued project IDs")?; - let mut run = - DBProjectFlagRun::create(user.id.into(), total_projects, &pool) - .await - .wrap_internal_err("creating project flag run")?; - - // Keep dispatch alive if the caller disconnects while Kafka is publishing. - let run = tokio::spawn(async move { - let empty = project_ids.is_empty(); - let result = crate::queue::moderation::project_flag_run::dispatch_project_flag_run( - run.id, rules, project_ids, &redis, &kafka, - ).await; - if let Err(error) = result { - run.finish(Some(format!("{error:#}")), &pool) - .await - .wrap_internal_err("recording project flag run dispatch failure")?; - return Err(ApiError::Internal(error)); - } - if empty { - run.finish(None, &pool) - .await - .wrap_internal_err("completing empty project flag run")?; - } - Ok(web::Json(run)) - }) - .await - .wrap_internal_err("joining project flag run dispatch")??; - Ok(run) -} - -#[derive(Deserialize, Validate, utoipa::ToSchema)] -pub struct EvalRule { - pub project_id: ProjectId, - #[validate(nested)] - pub params: ProjectRuleParams, -} - -#[utoipa::path( - context_path = "/moderation/project-flag", - tag = "moderation", - request_body = EvalRule, - responses( - (status = OK, body = ProjectFlagEvaluation), - (status = BAD_REQUEST), - (status = NOT_FOUND) - ) -)] -#[post("/rule/eval")] -pub async fn eval_rule( - req: HttpRequest, - body: web::Json, - ro_pool: web::Data, - redis: web::Data, - session_queue: web::Data, - engine: web::Data, -) -> Result, ApiError> { - check_is_moderator_from_headers( - &req, - &***ro_pool, - &redis, - &session_queue, - Scopes::SESSION_ACCESS, - ) - .await - .wrap_auth_err("authenticating moderator")?; - body.validate().map_err(|error| { - ApiError::Request(eyre!(validation_errors_to_string(error, None))) - })?; - let body = body.into_inner(); - let project_id = body.project_id.into(); - DBProject::get_id(project_id, &***ro_pool, &redis) - .await - .wrap_internal_err("fetching project for flag evaluation")? - .wrap_not_found_err("project not found")?; - let compile_engine = engine.clone(); - let call = tokio::task::spawn_blocking(move || { - compile_engine.make_call(&body.params) - }) - .await - .wrap_internal_err("joining project flag compilation")? - .wrap_request_err("compiling project flag expression")?; - let evaluation = engine - .scan_flags_with_log(&call, project_id) - .await - .wrap_request_err("evaluating project flag expression")?; - Ok(web::Json(evaluation)) + cfg.service(rule::list_rules) + .service(rule::get_rule_schema) + .service(rule::list_runs) + .service(rule::start_run) + .service(rule::get_rule) + .service(rule::create_rule) + .service(rule::update_rule) + .service(rule::delete_rule) + .service(rule::eval_rule) + .service(project::get_project_flags); } diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs new file mode 100644 index 00000000000..49debd10b33 --- /dev/null +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs @@ -0,0 +1,49 @@ +use actix_web::{HttpRequest, get, web}; +use xredis::RedisPool; + +use crate::{ + auth::check_is_moderator_from_headers, + database::{ + ReadOnlyPgPool, + models::project_flag_item::{DBProjectFlagRunResult, ProjectFlag}, + }, + models::{ids::ProjectId, pats::Scopes}, + queue::session::AuthQueue, + routes::ApiError, + util::error::Context as _, +}; + +/// Fetch all flags from persisted runs for a project. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + params(("id" = ProjectId, Path)), + responses((status = OK, body = [ProjectFlag]), (status = NOT_FOUND)) +)] +#[get("/project/{id}")] +pub async fn get_project_flags( + req: HttpRequest, + path: web::Path, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result>, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + let project_id = path.into_inner().into(); + + let flags = DBProjectFlagRunResult::get_project_flags(project_id, &ro_pool) + .await + .wrap_internal_err("fetching project flags")? + .wrap_not_found_err("project not found")?; + Ok(web::Json(flags)) +} diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs new file mode 100644 index 00000000000..171f57b5962 --- /dev/null +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs @@ -0,0 +1,428 @@ +use actix_web::{HttpRequest, delete, get, post, put, web}; +use eyre::eyre; +use serde::Deserialize; +use validator::Validate; +use xredis::RedisPool; + +use crate::auth::check_is_moderator_from_headers; +use crate::database::models::project_flag_item::ProjectRuleParams; +use crate::database::models::{ + DBProject, DBProjectFlagRule, DBProjectFlagRun, ProjectFlagRuleId, +}; +use crate::database::{PgPool, ReadOnlyPgPool}; +use crate::models::ids::ProjectId; +use crate::models::pats::Scopes; + +use crate::queue::moderation::project_flag::{ + FlagContext, ProjectFlagEngine, ProjectFlagEvaluation, +}; +use crate::queue::session::AuthQueue; +use crate::routes::ApiError; +use crate::util::error::Context as _; +use crate::util::validate::{validate_name, validation_errors_to_string}; + +use super::schema_example; + +/// Get an example project flag rule context. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + responses((status = OK, body = FlagContext)) +)] +#[get("/rules/schema")] +pub async fn get_rule_schema( + req: HttpRequest, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + Ok(web::Json( + schema_example::generate::() + .wrap_internal_err("generating project flag context example")?, + )) +} + +#[derive(Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListRules { + #[serde(default = "default_limit")] + pub limit: u16, + #[serde(default)] + pub offset: u32, +} + +fn default_limit() -> u16 { + 100 +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(ListRules), + responses((status = OK, body = [DBProjectFlagRule])) +)] +#[get("/rules")] +pub async fn list_rules( + req: HttpRequest, + query: web::Query, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result>, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if query.limit == 0 || query.limit > 200 { + return Err(ApiError::Request(eyre!( + "`limit` must be between 1 and 200" + ))); + } + let rules = DBProjectFlagRule::list( + i64::from(query.limit), + i64::from(query.offset), + &ro_pool, + ) + .await + .wrap_internal_err("listing project flag rules")?; + Ok(web::Json(rules)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(("id" = ProjectFlagRuleId, Path)), + responses((status = OK, body = DBProjectFlagRule), (status = NOT_FOUND)) +)] +#[get("/rules/{id}")] +pub async fn get_rule( + req: HttpRequest, + path: web::Path, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + let rule = DBProjectFlagRule::get(path.into_inner(), &ro_pool) + .await + .wrap_internal_err("fetching project flag rule")? + .wrap_not_found_err("project flag rule not found")?; + Ok(web::Json(rule)) +} + +#[derive(Deserialize, Validate, utoipa::ToSchema)] +pub struct CreateRule { + #[validate( + length(min = 1, max = 1023), + custom(function = "validate_name") + )] + pub name: String, + #[validate(nested)] + pub params: ProjectRuleParams, +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + request_body = CreateRule, + responses((status = OK, body = DBProjectFlagRule)) +)] +#[post("/rules")] +pub async fn create_rule( + req: HttpRequest, + body: web::Json, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + body.validate().map_err(|error| { + ApiError::Request(eyre!(validation_errors_to_string(error, None))) + })?; + + let rule = DBProjectFlagRule::create( + &body.name, + &body.params, + user.id.into(), + &pool, + ) + .await + .wrap_internal_err("creating project flag rule")?; + Ok(web::Json(rule)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(("id" = ProjectFlagRuleId, Path)), + request_body = CreateRule, + responses((status = OK, body = DBProjectFlagRule), (status = NOT_FOUND)) +)] +#[put("/rules/{id}")] +pub async fn update_rule( + req: HttpRequest, + path: web::Path, + body: web::Json, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + body.validate().map_err(|error| { + ApiError::Request(eyre!(validation_errors_to_string(error, None))) + })?; + + let rule = DBProjectFlagRule::update( + path.into_inner(), + &body.name, + &body.params, + user.id.into(), + &pool, + ) + .await + .wrap_internal_err("updating project flag rule")? + .wrap_not_found_err("project flag rule not found")?; + Ok(web::Json(rule)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(("id" = ProjectFlagRuleId, Path)), + responses((status = NO_CONTENT), (status = NOT_FOUND)) +)] +#[delete("/rules/{id}")] +pub async fn delete_rule( + req: HttpRequest, + path: web::Path, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result<(), ApiError> { + check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if !DBProjectFlagRule::delete(path.into_inner(), &pool) + .await + .wrap_internal_err("deleting project flag rule")? + { + return Err(ApiError::NotFound(eyre!("project flag rule not found"))); + } + Ok(()) +} + +#[derive(Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListRuns { + #[serde(default = "default_limit")] + pub limit: u16, + #[serde(default)] + pub offset: u32, +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + params(ListRuns), + responses((status = OK, body = [DBProjectFlagRun])) +)] +#[get("/rules/run")] +pub async fn list_runs( + req: HttpRequest, + query: web::Query, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result>, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if query.limit == 0 || query.limit > 200 { + return Err(ApiError::Request(eyre!( + "`limit` must be between 1 and 200" + ))); + } + let runs = DBProjectFlagRun::list( + i64::from(query.limit), + i64::from(query.offset), + &ro_pool, + ) + .await + .wrap_internal_err("listing project flag runs")?; + Ok(web::Json(runs)) +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + responses((status = OK, body = DBProjectFlagRun)) +)] +#[post("/rules/run")] +pub async fn start_run( + req: HttpRequest, + kafka: web::Data, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + let rules = DBProjectFlagRule::all(&pool) + .await + .wrap_internal_err("snapshotting project flag rules")?; + let project_ids = DBProjectFlagRun::queued_project_ids(&pool) + .await + .wrap_internal_err("fetching queued project IDs")?; + let total_projects = i64::try_from(project_ids.len()) + .wrap_internal_err("counting queued project IDs")?; + let mut run = + DBProjectFlagRun::create(user.id.into(), total_projects, &pool) + .await + .wrap_internal_err("creating project flag run")?; + + // Keep dispatch alive if the caller disconnects while Kafka is publishing. + let run = tokio::spawn(async move { + let empty = project_ids.is_empty(); + let result = crate::queue::moderation::project_flag_run::dispatch_project_flag_run( + run.id, rules, project_ids, &redis, &kafka, + ).await; + if let Err(error) = result { + run.finish(Some(format!("{error:#}")), &pool) + .await + .wrap_internal_err("recording project flag run dispatch failure")?; + return Err(ApiError::Internal(error)); + } + if empty { + run.finish(None, &pool) + .await + .wrap_internal_err("completing empty project flag run")?; + } + Ok(web::Json(run)) + }) + .await + .wrap_internal_err("joining project flag run dispatch")??; + Ok(run) +} + +#[derive(Deserialize, Validate, utoipa::ToSchema)] +pub struct EvalRule { + pub project_id: ProjectId, + #[validate(nested)] + pub params: ProjectRuleParams, +} + +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + request_body = EvalRule, + responses( + (status = OK, body = ProjectFlagEvaluation), + (status = BAD_REQUEST), + (status = NOT_FOUND) + ) +)] +#[post("/rule/eval")] +pub async fn eval_rule( + req: HttpRequest, + body: web::Json, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, + engine: web::Data, +) -> Result, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + body.validate().map_err(|error| { + ApiError::Request(eyre!(validation_errors_to_string(error, None))) + })?; + let body = body.into_inner(); + let project_id = body.project_id.into(); + DBProject::get_id(project_id, &***ro_pool, &redis) + .await + .wrap_internal_err("fetching project for flag evaluation")? + .wrap_not_found_err("project not found")?; + let compile_engine = engine.clone(); + let call = tokio::task::spawn_blocking(move || { + compile_engine.make_call(&body.params) + }) + .await + .wrap_internal_err("joining project flag compilation")? + .wrap_request_err("compiling project flag expression")?; + let evaluation = engine + .scan_flags_with_log(&call, project_id) + .await + .wrap_request_err("evaluating project flag expression")?; + Ok(web::Json(evaluation)) +} From 05ef0eb0d8eae7e8fcdd7c517613fcc5b0f5b03e Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:06:19 +0000 Subject: [PATCH 12/26] list runs/delete runs but keep active flags --- .../project-flag/ProjectFlagRuns.vue | 446 ++++++++++++++++++ .../src/pages/moderation/project-flag.vue | 19 +- .../20261028145153_project_flags_backend.sql | 21 +- .../src/database/models/project_flag_item.rs | 358 +++++++++++++- apps/labrinth/src/routes/internal/mod.rs | 3 + .../internal/moderation/project_flag/mod.rs | 3 + .../moderation/project_flag/project.rs | 6 +- .../internal/moderation/project_flag/rule.rs | 165 ++++++- .../labrinth/project-flags/internal.ts | 44 ++ .../modules/labrinth/project-flags/types.ts | 47 +- packages/api-client/src/modules/types.ts | 1 + 11 files changed, 1084 insertions(+), 29 deletions(-) create mode 100644 apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuns.vue diff --git a/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuns.vue b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuns.vue new file mode 100644 index 00000000000..e129b807677 --- /dev/null +++ b/apps/frontend/src/components/ui/moderation/project-flag/ProjectFlagRuns.vue @@ -0,0 +1,446 @@ + + + diff --git a/apps/frontend/src/pages/moderation/project-flag.vue b/apps/frontend/src/pages/moderation/project-flag.vue index 24804d8f9d9..17d75c70404 100644 --- a/apps/frontend/src/pages/moderation/project-flag.vue +++ b/apps/frontend/src/pages/moderation/project-flag.vue @@ -12,11 +12,20 @@ Create rule +
-
- Run history is not available yet. -
+ route.query.tab === 'runs') +const runsView = useTemplateRef>('runsView') const tabs = [ { label: 'Rules', href: '', icon: TagCategoryFlagIcon }, { label: 'Runs', href: 'runs', icon: HistoryIcon }, @@ -244,7 +255,7 @@ const hasNextPage = computed(() => (data.value?.length ?? 0) > PAGE_SIZE) const DEFAULT_RULE = `print(\`\${project.name}: \${project.summary}\`); if project.slug == "foo" { - [#{ reason: "Project has illegal slug" }] + [#{ key: "illegal_slug", reason: "Project has illegal slug" }] } else { [] }` diff --git a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql index 0c817e7804f..25a5d160350 100644 --- a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql +++ b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql @@ -8,11 +8,6 @@ CREATE TABLE project_flag_rules ( params JSONB NOT NULL ); -CREATE TABLE project_flags ( - id BIGINT PRIMARY KEY, - rule_id BIGINT REFERENCES project_flag_rules(id) ON DELETE SET NULL, - what JSONB NOT NULL -); CREATE TABLE project_flag_runs ( id UUID PRIMARY KEY, @@ -21,10 +16,26 @@ CREATE TABLE project_flag_runs ( started_by BIGINT REFERENCES users(id) ON DELETE SET NULL, started_at TIMESTAMPTZ NOT NULL, completed_at TIMESTAMPTZ, + applied_at TIMESTAMPTZ, output JSONB, error TEXT ); +CREATE TABLE project_flags ( + id BIGINT PRIMARY KEY, + run_id UUID REFERENCES project_flag_runs(id) ON DELETE CASCADE, + project_id BIGINT NOT NULL REFERENCES mods(id) ON DELETE CASCADE, + what JSONB NOT NULL, + active BOOLEAN NOT NULL DEFAULT FALSE +); + +CREATE INDEX project_flags_run_id ON project_flags (run_id); + +CREATE INDEX project_flags_project_id ON project_flags (project_id); + +CREATE UNIQUE INDEX project_flags_identity + ON project_flags (project_id, (what->>'key')); + CREATE TABLE project_flag_run_results ( run_id UUID NOT NULL REFERENCES project_flag_runs(id) ON DELETE CASCADE, project_id BIGINT NOT NULL REFERENCES mods(id) ON DELETE CASCADE, diff --git a/apps/labrinth/src/database/models/project_flag_item.rs b/apps/labrinth/src/database/models/project_flag_item.rs index b8025163e1e..eb0a03369d3 100644 --- a/apps/labrinth/src/database/models/project_flag_item.rs +++ b/apps/labrinth/src/database/models/project_flag_item.rs @@ -11,7 +11,9 @@ use serde_with::{DisplayFromStr, serde_as}; use utoipa::ToSchema; use validator::Validate; -use super::{DBUserId, ProjectFlagId, ProjectFlagRuleId, ProjectFlagRunId}; +use super::{ + DBProjectId, DBUserId, ProjectFlagId, ProjectFlagRuleId, ProjectFlagRunId, +}; #[derive( Clone, @@ -233,6 +235,7 @@ impl DBProjectFlagRule { #[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] pub struct ProjectFlag { + pub key: String, #[serde(default)] pub retain_if_removed: bool, #[serde(flatten)] @@ -242,8 +245,10 @@ pub struct ProjectFlag { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DBProjectFlag { pub id: ProjectFlagId, - pub rule_id: Option, + pub run_id: Option, + pub project_id: DBProjectId, pub what: ProjectFlag, + pub active: bool, } #[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] @@ -258,23 +263,24 @@ type RuleResults = BTreeMap; #[derive(Debug, Serialize)] pub struct DBProjectFlagRunResult { pub run_id: ProjectFlagRunId, - pub project_id: super::DBProjectId, + pub project_id: DBProjectId, pub rules: BTreeMap, pub error: Option, } -impl DBProjectFlagRunResult { +impl DBProjectFlag { pub async fn get_project_flags( - project_id: super::DBProjectId, + project_id: DBProjectId, pool: &ReadOnlyPgPool, ) -> Result>> { let rows = sqlx::query!( r#" - SELECT results.rules AS "rules?: sqlx::types::Json" + SELECT flags.what AS "what?: sqlx::types::Json" FROM mods AS projects - LEFT JOIN project_flag_run_results AS results ON results.project_id = projects.id + LEFT JOIN project_flags AS flags ON flags.project_id = projects.id + AND (flags.active OR (flags.what->>'retain_if_removed')::bool = TRUE) WHERE projects.id = $1 - ORDER BY results.evaluated_at DESC, results.run_id DESC + ORDER BY flags.what->>'key', flags.id "#, project_id.0, ) @@ -287,12 +293,128 @@ impl DBProjectFlagRunResult { } Ok(Some( rows.into_iter() - .filter_map(|row| row.rules) - .flat_map(|rules| rules.0.into_values()) - .flat_map(|rule| rule.flags) + .filter_map(|row| row.what.map(|what| what.0)) .collect(), )) } +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ProjectFlagRunResultProject { + pub id: crate::models::ids::ProjectId, + pub title: String, + pub slug: Option, + pub icon_url: Option, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ProjectFlagRunRuleResult { + pub name: Option, + #[serde(flatten)] + pub result: ProjectFlagRuleResult, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ProjectFlagRunResult { + pub project: ProjectFlagRunResultProject, + pub rules: BTreeMap, + pub error: Option, + pub evaluated_at: DateTime, +} + +#[derive(Debug, Serialize, ToSchema)] +pub struct ProjectFlagRunResults { + pub results: Vec, + pub total: i64, +} + +impl DBProjectFlagRunResult { + pub async fn list( + run_id: ProjectFlagRunId, + limit: i64, + offset: i64, + pool: &ReadOnlyPgPool, + ) -> Result> { + let run = sqlx::query!( + r#" + SELECT ( + SELECT COUNT(*) + FROM project_flag_run_results + WHERE run_id = runs.id + ) AS "total!" + FROM project_flag_runs AS runs + WHERE runs.id = $1 + "#, + run_id.0, + ) + .fetch_optional(&**pool) + .await + .wrap_err("counting project flag run results")?; + let Some(run) = run else { + return Ok(None); + }; + let rows = sqlx::query!( + r#" + SELECT projects.id, projects.name AS title, projects.slug, projects.icon_url, + results.rules AS "rules: sqlx::types::Json", + results.error, results.evaluated_at + FROM project_flag_run_results AS results + INNER JOIN mods AS projects ON projects.id = results.project_id + WHERE results.run_id = $1 + ORDER BY results.project_id + LIMIT $2 OFFSET $3 + "#, + run_id.0, + limit, + offset, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching project flag run results")?; + let mut rule_ids = rows + .iter() + .flat_map(|row| row.rules.0.keys()) + .map(|id| id.parse::()) + .collect::, _>>() + .wrap_err("parsing project flag result rule IDs")?; + rule_ids.sort_unstable(); + rule_ids.dedup(); + let names = sqlx::query!( + r#"SELECT id, name FROM project_flag_rules WHERE id = ANY($1)"#, + &rule_ids, + ) + .fetch_all(&**pool) + .await + .wrap_err("fetching project flag rule names")? + .into_iter() + .map(|row| (row.id.to_string(), row.name)) + .collect::>(); + Ok(Some(ProjectFlagRunResults { + results: rows + .into_iter() + .map(|row| ProjectFlagRunResult { + project: ProjectFlagRunResultProject { + id: DBProjectId(row.id).into(), + title: row.title, + slug: row.slug, + icon_url: row.icon_url, + }, + rules: row + .rules + .0 + .into_iter() + .map(|(id, result)| { + let name = names.get(&id).cloned(); + (id, ProjectFlagRunRuleResult { name, result }) + }) + .collect(), + error: row.error, + evaluated_at: row.evaluated_at, + }) + .collect(), + total: run.total, + })) + } pub async fn upsert_many(results: Vec, pool: &PgPool) -> Result<()> { if results.is_empty() { @@ -399,21 +521,225 @@ pub struct DBProjectFlagRun { pub started_by: Option, pub started_at: DateTime, pub completed_at: Option>, + pub applied_at: Option>, pub output: Option, pub error: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ApplyRunOutcome { + Applied(DateTime), + NotFound, + NotDone, + AlreadyApplied, + Incomplete, +} + impl DBProjectFlagRun { - pub async fn queued_project_ids( + pub async fn delete(id: ProjectFlagRunId, pool: &PgPool) -> Result { + let mut transaction = pool + .begin() + .await + .wrap_err("starting project flag run deletion")?; + let run = sqlx::query_scalar!( + r#"SELECT id FROM project_flag_runs WHERE id = $1 FOR UPDATE"#, + id.0, + ) + .fetch_optional(&mut transaction) + .await + .wrap_err("locking project flag run for deletion")?; + if run.is_none() { + return Ok(false); + } + sqlx::query!( + r#" + SELECT id FROM mods + WHERE id IN (SELECT project_id FROM project_flags WHERE run_id = $1) + ORDER BY id + FOR UPDATE + "#, + id.0, + ) + .fetch_all(&mut transaction) + .await + .wrap_err("locking projects for run deletion")?; + sqlx::query!( + r#"UPDATE project_flags SET run_id = NULL WHERE run_id = $1 AND active"#, + id.0, + ).execute(&mut transaction).await.wrap_err("detaching active project flags")?; + let result = sqlx::query!( + r#" + DELETE FROM project_flag_runs + WHERE id = $1 + "#, + id.0, + ) + .execute(&mut transaction) + .await + .wrap_err("deleting project flag run")?; + transaction + .commit() + .await + .wrap_err("committing project flag run deletion")?; + Ok(result.rows_affected() > 0) + } + + pub async fn apply( + id: ProjectFlagRunId, pool: &PgPool, - ) -> Result> { + ) -> Result { + let mut transaction = pool + .begin() + .await + .wrap_err("starting project flag apply transaction")?; + let run = sqlx::query!( + r#" + SELECT status AS "status: ProjectFlagRunStatus", applied_at, + total_projects, error + FROM project_flag_runs + WHERE id = $1 + FOR UPDATE + "#, + id.0, + ) + .fetch_optional(&mut transaction) + .await + .wrap_err("locking project flag run for application")?; + let Some(run) = run else { + return Ok(ApplyRunOutcome::NotFound); + }; + if run.applied_at.is_some() { + return Ok(ApplyRunOutcome::AlreadyApplied); + } + if run.status != ProjectFlagRunStatus::Done { + return Ok(ApplyRunOutcome::NotDone); + } + let results = sqlx::query!( + r#" + SELECT project_id, rules AS "rules: sqlx::types::Json", error + FROM project_flag_run_results + WHERE run_id = $1 + ORDER BY project_id + "#, + id.0, + ) + .fetch_all(&mut transaction) + .await + .wrap_err("loading project flag results for application")?; + if run.error.is_some() || results.len() as i64 != run.total_projects { + return Ok(ApplyRunOutcome::Incomplete); + } + let results = results + .into_iter() + .filter(|result| { + result.error.is_none() + && result.rules.0.values().all(|rule| rule.error.is_none()) + }) + .collect::>(); + let project_ids = results + .iter() + .map(|result| result.project_id) + .collect::>(); + sqlx::query!( + r#" + SELECT id FROM mods + WHERE id = ANY($1) + ORDER BY id + FOR UPDATE + "#, + &project_ids, + ) + .fetch_all(&mut transaction) + .await + .wrap_err("locking projects for project flag application")?; + let mut flags = Vec::new(); + for result in results { + let mut unique_flags = BTreeMap::::new(); + for rule in result.rules.0.into_values() { + for flag in rule.flags { + match unique_flags.entry(flag.key.clone()) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(flag); + } + std::collections::btree_map::Entry::Occupied( + mut entry, + ) => { + entry.get_mut().retain_if_removed |= + flag.retain_if_removed; + } + } + } + } + for what in unique_flags.into_values() { + flags.push(DBProjectFlag { + id: ProjectFlagId(random_base62(8) as i64), + run_id: Some(id), + project_id: DBProjectId(result.project_id), + what, + active: true, + }); + } + } + let flags = serde_json::to_value(flags) + .wrap_err("serializing applied project flags")?; + sqlx::query!( + r#" + UPDATE project_flags SET active = FALSE + WHERE project_id = ANY($1) + "#, + &project_ids, + ) + .execute(&mut transaction) + .await + .wrap_err("deactivating previous project flags")?; + sqlx::query!( + r#" + INSERT INTO project_flags (id, run_id, project_id, what, active) + SELECT flags.id, flags.run_id, flags.project_id, flags.what, TRUE + FROM jsonb_to_recordset($1) AS flags( + id BIGINT, run_id UUID, project_id BIGINT, what JSONB + ) + ORDER BY flags.project_id, flags.what->>'key' + ON CONFLICT (project_id, (what->>'key')) DO UPDATE + SET active = TRUE, + run_id = EXCLUDED.run_id, + what = EXCLUDED.what || jsonb_build_object( + 'retain_if_removed', + COALESCE((project_flags.what->>'retain_if_removed')::bool, FALSE) + OR COALESCE((EXCLUDED.what->>'retain_if_removed')::bool, FALSE) + ) + "#, + flags, + ) + .execute(&mut transaction) + .await + .wrap_err("upserting applied project flags")?; + let applied_at = sqlx::query_scalar!( + r#" + UPDATE project_flag_runs SET applied_at = CURRENT_TIMESTAMP + WHERE id = $1 + RETURNING applied_at AS "applied_at!" + "#, + id.0, + ) + .fetch_one(&mut transaction) + .await + .wrap_err("marking project flag run applied")?; + transaction + .commit() + .await + .wrap_err("committing project flag application")?; + Ok(ApplyRunOutcome::Applied(applied_at)) + } + + pub async fn queued_project_ids(pool: &PgPool) -> Result> { let ids = sqlx::query_scalar!( r#"SELECT id FROM mods WHERE status = 'processing'"#, ) .fetch_all(pool) .await .wrap_err("fetching queued project IDs")?; - Ok(ids.into_iter().map(super::DBProjectId).collect()) + Ok(ids.into_iter().map(DBProjectId).collect()) } pub async fn create( @@ -429,6 +755,7 @@ impl DBProjectFlagRun { started_by: Some(user_id), started_at: Utc::now(), completed_at: None, + applied_at: None, output: None, error: None, }; @@ -483,7 +810,7 @@ impl DBProjectFlagRun { let rows = sqlx::query!( r#" SELECT id, status AS "status: ProjectFlagRunStatus", total_projects, started_by, - started_at, completed_at, + started_at, completed_at, applied_at, output AS "output: sqlx::types::Json", error, (SELECT COUNT(*) FROM project_flag_run_results AS results WHERE results.run_id = project_flag_runs.id AND results.error IS NOT NULL @@ -509,6 +836,7 @@ impl DBProjectFlagRun { started_by: row.started_by.map(DBUserId), started_at: row.started_at, completed_at: row.completed_at, + applied_at: row.applied_at, output: row.output.map(|output| output.0), error: row.error, }) diff --git a/apps/labrinth/src/routes/internal/mod.rs b/apps/labrinth/src/routes/internal/mod.rs index 39c6b8674ea..c3a4dd79871 100644 --- a/apps/labrinth/src/routes/internal/mod.rs +++ b/apps/labrinth/src/routes/internal/mod.rs @@ -105,7 +105,10 @@ pub fn config(cfg: &mut web::ServiceConfig) { pats::delete_pat, moderation::project_flag::project::get_project_flags, moderation::project_flag::rule::list_runs, + moderation::project_flag::rule::list_run_results, moderation::project_flag::rule::start_run, + moderation::project_flag::rule::apply_run, + moderation::project_flag::rule::delete_run, moderation::project_flag::rule::eval_rule, moderation::project_flag::rule::list_rules, moderation::project_flag::rule::get_rule_schema, diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs index 01e0a0d01c0..dec4c6a9128 100644 --- a/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/mod.rs @@ -6,7 +6,10 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig<'_>) { cfg.service(rule::list_rules) .service(rule::get_rule_schema) .service(rule::list_runs) + .service(rule::list_run_results) .service(rule::start_run) + .service(rule::apply_run) + .service(rule::delete_run) .service(rule::get_rule) .service(rule::create_rule) .service(rule::update_rule) diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs index 49debd10b33..f897a63c734 100644 --- a/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/project.rs @@ -5,7 +5,7 @@ use crate::{ auth::check_is_moderator_from_headers, database::{ ReadOnlyPgPool, - models::project_flag_item::{DBProjectFlagRunResult, ProjectFlag}, + models::project_flag_item::{DBProjectFlag, ProjectFlag}, }, models::{ids::ProjectId, pats::Scopes}, queue::session::AuthQueue, @@ -13,7 +13,7 @@ use crate::{ util::error::Context as _, }; -/// Fetch all flags from persisted runs for a project. +/// Fetch current and retained flags for a project. #[utoipa::path( context_path = "/moderation/project-flag", tag = "moderation", @@ -41,7 +41,7 @@ pub async fn get_project_flags( let project_id = path.into_inner().into(); - let flags = DBProjectFlagRunResult::get_project_flags(project_id, &ro_pool) + let flags = DBProjectFlag::get_project_flags(project_id, &ro_pool) .await .wrap_internal_err("fetching project flags")? .wrap_not_found_err("project not found")?; diff --git a/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs b/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs index 171f57b5962..b56dd28c522 100644 --- a/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs +++ b/apps/labrinth/src/routes/internal/moderation/project_flag/rule.rs @@ -5,9 +5,13 @@ use validator::Validate; use xredis::RedisPool; use crate::auth::check_is_moderator_from_headers; -use crate::database::models::project_flag_item::ProjectRuleParams; +use crate::database::models::project_flag_item::{ + ApplyRunOutcome, DBProjectFlagRunResult, ProjectFlagRunResults, + ProjectRuleParams, +}; use crate::database::models::{ DBProject, DBProjectFlagRule, DBProjectFlagRun, ProjectFlagRuleId, + ProjectFlagRunId, }; use crate::database::{PgPool, ReadOnlyPgPool}; use crate::models::ids::ProjectId; @@ -260,6 +264,71 @@ pub async fn delete_rule( Ok(()) } +#[derive(Deserialize, utoipa::IntoParams)] +#[into_params(parameter_in = Query)] +pub struct ListRunResults { + #[serde(default = "default_results_limit")] + #[param(default = 25, minimum = 1, maximum = 200)] + pub limit: u16, + #[serde(default)] + #[param(default = 0)] + pub offset: u32, +} + +fn default_results_limit() -> u16 { + 25 +} + +/// List completed project evaluations for a project flag run. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + params(("id" = ProjectFlagRunId, Path), ListRunResults), + responses( + (status = OK, body = ProjectFlagRunResults), + (status = BAD_REQUEST), + (status = UNAUTHORIZED), + (status = FORBIDDEN), + (status = NOT_FOUND) + ) +)] +#[get("/rules/run/{id}/results")] +pub async fn list_run_results( + req: HttpRequest, + path: web::Path, + query: web::Query, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + check_is_moderator_from_headers( + &req, + &***ro_pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + if query.limit == 0 || query.limit > 200 { + return Err(ApiError::Request(eyre!( + "`limit` must be between 1 and 200" + ))); + } + let results = DBProjectFlagRunResult::list( + path.into_inner(), + i64::from(query.limit), + i64::from(query.offset), + &ro_pool, + ) + .await + .wrap_internal_err("listing project flag run results")? + .wrap_not_found_err("project flag run not found")?; + Ok(web::Json(results)) +} + #[derive(Deserialize, utoipa::IntoParams)] #[into_params(parameter_in = Query)] pub struct ListRuns { @@ -369,6 +438,100 @@ pub async fn start_run( Ok(run) } +/// Delete a run, its results, and its inactive flags, preserving active flags. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + params(("id" = ProjectFlagRunId, Path)), + responses((status = OK), (status = NOT_FOUND)) +)] +#[delete("/rules/run/{id}")] +pub async fn delete_run( + req: HttpRequest, + path: web::Path, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result<(), ApiError> { + check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + let id = path.into_inner(); + if !DBProjectFlagRun::delete(id, &pool) + .await + .wrap_internal_err("deleting project flag run")? + { + return Err(ApiError::NotFound(eyre!("project flag run not found"))); + } + let cleanup = async { + let mut connection = redis.connect().await?; + let key = connection.key().entity("project_flag_run_rules:v1", id.0); + redis::cmd("DEL") + .arg(key) + .query_async::(&mut connection) + .await?; + eyre::Ok(()) + } + .await; + if let Err(error) = cleanup { + tracing::warn!(run_id = %id.0, ?error, "Failed to remove deleted run rule snapshot"); + } + Ok(()) +} + +/// Apply a completed run to the current project flags. +#[utoipa::path( + context_path = "/moderation/project-flag", + tag = "moderation", + security(("bearer_auth" = [])), + params(("id" = ProjectFlagRunId, Path)), + responses((status = OK), (status = NOT_FOUND), (status = CONFLICT)) +)] +#[post("/rules/run/{id}/apply")] +pub async fn apply_run( + req: HttpRequest, + path: web::Path, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result<(), ApiError> { + check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::SESSION_ACCESS, + ) + .await + .wrap_auth_err("authenticating moderator")?; + + match DBProjectFlagRun::apply(path.into_inner(), &pool) + .await + .wrap_internal_err("applying project flag run")? + { + ApplyRunOutcome::Applied(_) => Ok(()), + ApplyRunOutcome::NotFound => { + Err(ApiError::NotFound(eyre!("project flag run not found"))) + } + ApplyRunOutcome::NotDone => { + Err(ApiError::Conflict(eyre!("project flag run is not done"))) + } + ApplyRunOutcome::AlreadyApplied => Err(ApiError::Conflict(eyre!( + "project flag run has already been applied" + ))), + ApplyRunOutcome::Incomplete => Err(ApiError::Conflict(eyre!( + "project flag run has incomplete results" + ))), + } +} + #[derive(Deserialize, Validate, utoipa::ToSchema)] pub struct EvalRule { pub project_id: ProjectId, diff --git a/packages/api-client/src/modules/labrinth/project-flags/internal.ts b/packages/api-client/src/modules/labrinth/project-flags/internal.ts index de8ba1e5cf1..a9428b2c5fc 100644 --- a/packages/api-client/src/modules/labrinth/project-flags/internal.ts +++ b/packages/api-client/src/modules/labrinth/project-flags/internal.ts @@ -3,6 +3,10 @@ import type { CreateProjectFlagRuleRequest, EvalProjectFlagRuleRequest, ListProjectFlagRulesParams, + ListProjectFlagRunsParams, + ListProjectFlagRunResultsParams, + ProjectFlagRun, + ProjectFlagRunResults, ProjectFlagEvaluation, ProjectFlagRule, ProjectFlagRuleSchemaResponse, @@ -33,6 +37,46 @@ export class LabrinthProjectFlagsInternalModule extends AbstractModule { }) } + public async listRuns(params: ListProjectFlagRunsParams = {}): Promise { + return this.client.request('/moderation/project-flag/rules/run', { + api: 'labrinth', + version: 'internal', + method: 'GET', + params, + }) + } + + public async startRun(): Promise { + return this.client.request('/moderation/project-flag/rules/run', { + api: 'labrinth', + version: 'internal', + method: 'POST', + retry: false, + }) + } + + public async deleteRun(id: string): Promise { + return this.client.request( + `/moderation/project-flag/rules/run/${encodeURIComponent(id)}`, + { api: 'labrinth', version: 'internal', method: 'DELETE', retry: false }, + ) + } + + public async getRunResults( + id: string, + params: ListProjectFlagRunResultsParams = {}, + ): Promise { + return this.client.request( + `/moderation/project-flag/rules/run/${encodeURIComponent(id)}/results`, + { + api: 'labrinth', + version: 'internal', + method: 'GET', + params, + }, + ) + } + public async get(id: number): Promise { return this.client.request(`/moderation/project-flag/rules/${id}`, { api: 'labrinth', diff --git a/packages/api-client/src/modules/labrinth/project-flags/types.ts b/packages/api-client/src/modules/labrinth/project-flags/types.ts index 02254771ca2..74dd3ebb9ba 100644 --- a/packages/api-client/src/modules/labrinth/project-flags/types.ts +++ b/packages/api-client/src/modules/labrinth/project-flags/types.ts @@ -33,10 +33,55 @@ export interface EvalProjectFlagRuleRequest { export type ProjectFlagRuleSchemaResponse = Record export interface ProjectFlagEvaluation { - flags: ProjectFlagValue[] + flags: ProjectFlag[] log: string[] } +export interface ProjectFlag { + key: string + retain_if_removed: boolean + [field: string]: ProjectFlagValue +} + +export interface ProjectFlagRun { + id: string + status: 'running' | 'done' + total_projects: number + num_errors: number + started_by: number | null + started_at: string + completed_at: string | null + applied_at: string | null + output: Record | null + error: string | null +} + +export interface ProjectFlagRuleResult { + flags: ProjectFlag[] + log: string[] + error: string | null +} + +export interface ProjectFlagRunResult { + project: { + id: string + title: string + slug: string | null + icon_url: string | null + } + rules: Record + error: string | null + evaluated_at: string +} + +export interface ProjectFlagRunResults { + results: ProjectFlagRunResult[] + total: number +} + +export type ListProjectFlagRunsParams = ListProjectFlagRulesParams +export type ListProjectFlagRunResultsParams = ListProjectFlagRulesParams + export type ProjectFlagValue = | null | boolean diff --git a/packages/api-client/src/modules/types.ts b/packages/api-client/src/modules/types.ts index bd25e5f707c..7e0b78d9013 100644 --- a/packages/api-client/src/modules/types.ts +++ b/packages/api-client/src/modules/types.ts @@ -2,6 +2,7 @@ export * from './archon/types' export * from './iso3166/types' export * from './kyros/types' export * from './labrinth/types' +export * from './labrinth/project-flags/types' export * from './launcher-meta/types' export * from './mclogs/types' export * from './paper/types' From c97cefbb9a00c5c7f07db66a5c5c8228b58e99f8 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:33:00 +0000 Subject: [PATCH 13/26] allow kafka scans to not have a run_id --- .../20261028145153_project_flags_backend.sql | 14 ++-- .../src/database/models/project_flag_item.rs | 68 ++++++++++++------- .../src/queue/moderation/project_flag_run.rs | 29 ++++---- 3 files changed, 65 insertions(+), 46 deletions(-) diff --git a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql index 25a5d160350..3ffb9e6cd27 100644 --- a/apps/labrinth/migrations/20261028145153_project_flags_backend.sql +++ b/apps/labrinth/migrations/20261028145153_project_flags_backend.sql @@ -8,7 +8,6 @@ CREATE TABLE project_flag_rules ( params JSONB NOT NULL ); - CREATE TABLE project_flag_runs ( id UUID PRIMARY KEY, status TEXT NOT NULL, @@ -22,10 +21,10 @@ CREATE TABLE project_flag_runs ( ); CREATE TABLE project_flags ( - id BIGINT PRIMARY KEY, - run_id UUID REFERENCES project_flag_runs(id) ON DELETE CASCADE, - project_id BIGINT NOT NULL REFERENCES mods(id) ON DELETE CASCADE, - what JSONB NOT NULL, + id BIGINT PRIMARY KEY, + run_id UUID REFERENCES project_flag_runs(id) ON DELETE CASCADE, + project_id BIGINT NOT NULL REFERENCES mods(id) ON DELETE CASCADE, + what JSONB NOT NULL, active BOOLEAN NOT NULL DEFAULT FALSE ); @@ -37,12 +36,13 @@ CREATE UNIQUE INDEX project_flags_identity ON project_flags (project_id, (what->>'key')); CREATE TABLE project_flag_run_results ( - run_id UUID NOT NULL REFERENCES project_flag_runs(id) ON DELETE CASCADE, + id UUID PRIMARY KEY, + run_id UUID REFERENCES project_flag_runs(id) ON DELETE CASCADE, project_id BIGINT NOT NULL REFERENCES mods(id) ON DELETE CASCADE, rules JSONB NOT NULL DEFAULT '{}'::jsonb, error TEXT, evaluated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (run_id, project_id) + UNIQUE (run_id, project_id) ); CREATE INDEX project_flag_run_results_project_id diff --git a/apps/labrinth/src/database/models/project_flag_item.rs b/apps/labrinth/src/database/models/project_flag_item.rs index eb0a03369d3..2ff7d5f5e7f 100644 --- a/apps/labrinth/src/database/models/project_flag_item.rs +++ b/apps/labrinth/src/database/models/project_flag_item.rs @@ -262,7 +262,7 @@ type RuleResults = BTreeMap; #[derive(Debug, Serialize)] pub struct DBProjectFlagRunResult { - pub run_id: ProjectFlagRunId, + pub run_id: Option, pub project_id: DBProjectId, pub rules: BTreeMap, pub error: Option, @@ -422,31 +422,47 @@ impl DBProjectFlagRunResult { } let mut run_ids = results .iter() - .map(|result| result.run_id.0) + .filter_map(|result| result.run_id.map(|id| id.0)) .collect::>(); run_ids.sort_unstable(); run_ids.dedup(); - let results = results - .into_iter() - .map(|mut result| { - if result.error.is_none() { - let errors = result - .rules - .iter() - .filter_map(|(rule_id, rule)| { - rule.error - .as_ref() - .map(|error| format!("{rule_id}: {error}")) - }) - .collect::>(); - if !errors.is_empty() { - result.error = Some(errors.join("\n")); - } + #[derive(Serialize)] + struct ResultPayload { + id: uuid::Uuid, + #[serde(flatten)] + result: DBProjectFlagRunResult, + } + + let mut run_results = std::collections::HashMap::new(); + let mut standalone_results = Vec::new(); + for mut result in results { + if result.error.is_none() { + let errors = result + .rules + .iter() + .filter_map(|(rule_id, rule)| { + rule.error + .as_ref() + .map(|error| format!("{rule_id}: {error}")) + }) + .collect::>(); + if !errors.is_empty() { + result.error = Some(errors.join("\n")); } - ((result.run_id, result.project_id), result) - }) - .collect::>() + } + if let Some(run_id) = result.run_id { + run_results.insert((run_id, result.project_id), result); + } else { + standalone_results.push(result); + } + } + let results = run_results .into_values() + .chain(standalone_results) + .map(|result| ResultPayload { + id: uuid::Uuid::now_v7(), + result, + }) .collect::>(); let results = serde_json::to_value(results) .wrap_err("serializing project flag run results")?; @@ -470,13 +486,13 @@ impl DBProjectFlagRunResult { sqlx::query!( r#" INSERT INTO project_flag_run_results - (run_id, project_id, rules, error) - SELECT results.run_id, results.project_id, results.rules, results.error + (id, run_id, project_id, rules, error) + SELECT results.id, results.run_id, results.project_id, results.rules, results.error FROM jsonb_to_recordset($1) AS results( - run_id UUID, project_id BIGINT, rules JSONB, error TEXT + id UUID, run_id UUID, project_id BIGINT, rules JSONB, error TEXT ) - INNER JOIN project_flag_runs AS runs ON runs.id = results.run_id - WHERE runs.status = 'running' + LEFT JOIN project_flag_runs AS runs ON runs.id = results.run_id + WHERE results.run_id IS NULL OR runs.status = 'running' ORDER BY results.run_id, results.project_id ON CONFLICT (run_id, project_id) DO UPDATE SET rules = EXCLUDED.rules, diff --git a/apps/labrinth/src/queue/moderation/project_flag_run.rs b/apps/labrinth/src/queue/moderation/project_flag_run.rs index e48d882ece6..a64f6aeb141 100644 --- a/apps/labrinth/src/queue/moderation/project_flag_run.rs +++ b/apps/labrinth/src/queue/moderation/project_flag_run.rs @@ -158,18 +158,21 @@ async fn scan_batch( if snapshots.contains_key(&run_id) { continue; } - let key = connection - .key() - .entity("project_flag_run_rules:v1", run_id.0); + let Some(id) = run_id else { + let rules = DBProjectFlagRule::all(pool) + .await + .wrap_err("loading rules for standalone project scans")?; + snapshots.insert(None, Ok(rules)); + continue; + }; + let key = connection.key().entity("project_flag_run_rules:v1", id.0); let snapshot: Option> = redis::cmd("GET") .arg(key) .query_async(&mut connection) .await .wrap_err("fetching project flag rule snapshot")?; let rules = snapshot - .ok_or_else(|| { - format!("rule snapshot missing for run {}", run_id.0) - }) + .ok_or_else(|| format!("rule snapshot missing for run {}", id.0)) .and_then(|snapshot| { serde_json::from_slice::>(&snapshot) .map_err(|error| { @@ -250,9 +253,9 @@ async fn scan_batch( ) })? .clone(); - let run = runs.get(&event.run_id).ok_or_else(|| { - eyre!("rule snapshot missing for run {}", event.run_id.0) - })?; + let run = runs + .get(&event.run_id) + .ok_or_else(|| eyre!("project scan rules not found"))?; if let Some(error) = &run.error { return Err(eyre!(error.clone())); } @@ -265,8 +268,8 @@ async fn scan_batch( .await .wrap_err_with(|| { format!( - "preparing project {} for run {}", - event.project_id, event.run_id.0 + "preparing project {} for flag scanning", + event.project_id ) })?; eyre::Ok(context) @@ -348,7 +351,7 @@ async fn scan_batch( #[derive(Debug, Serialize, Deserialize)] pub struct ProjectFlagRunEvent { pub project_id: ProjectId, - pub run_id: ProjectFlagRunId, + pub run_id: Option, } /// Stores a permanent JSON rules snapshot under the run ID and atomically @@ -411,7 +414,7 @@ pub async fn dispatch_project_flag_run( let project_id = ProjectId::from(project_id); let event = KafkaEvent::new( PROJECT_FLAG_RUN_TOPIC, - ProjectFlagRunEvent { project_id, run_id }, + ProjectFlagRunEvent { project_id, run_id: Some(run_id) }, ); let payload = serde_json::to_vec(&event) .wrap_err("serializing project flag run event")?; From 7961dc1b4da8e418f79b2f074409561ab496c859 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:27:44 +0000 Subject: [PATCH 14/26] sqlx prepare --- ...10dee885caa54ae8082b6cf9417ce8dd17597.json | 104 ++++++++++++ ...8569aa16c555283d5fb3bc9849e15fa17ef39.json | 14 ++ ...d307f5bf46989fdbae4d0829141301f61b36d.json | 98 ++++++++++++ ...0dce39f0d8deaca7775e9d5ff8f0d41a044b0.json | 14 ++ ...9723b6889d7e462ca1c7e87320caed28f9a7c.json | 64 ++++++++ ...793f4eeb9fedaa2cfd6ac7bdd99f10b14c511.json | 148 ++++++++++++++++++ ...9ab18ca959bce216193d8c1aa7cb61d9c1ffc.json | 14 ++ ...ea32ead13b547476e3900ee767d4fedd3cb68.json | 101 ++++++++++++ ...facb4b5b6b6eeb3519ac614e2aea07dad5dfb.json | 100 ++++++++++++ ...4d2a0476843c808a03f54a7e628dfa479ec0b.json | 23 +++ ...1a24d798a699f43f9c2bef8198244efc57038.json | 83 ++++++++++ ...8770dc4d19ce2968a4a97be5f4705710191b1.json | 28 ++++ ...87fb2fa4d56739ad38d02d3b305777b4d9046.json | 28 ++++ ...c04dd48751a393877b9923e047eef5bb1d9e4.json | 14 ++ ...1d969860f2758c6ef2354997d8451abdc4468.json | 104 ++++++++++++ ...35f18d45db9476dcdeeeb6ffef851f30a582.json} | 6 +- ...e96bcee32af66a8a754a5e1dcac03929a224f.json | 28 ++++ ...47944e4d9193d8fbd43696f958e21018fb376.json | 40 +++++ ...8d561f635ffcc0891f4c2e8dbfadedc8fe651.json | 88 +++++++++++ ...3062e95de8eca1cc09d1caeab92114ff65d0b.json | 64 ++++++++ ...ca1260cc250ea4455402f84e9f1530c31df87.json | 26 +++ ...9a25a281089126f997db20ba3a00d04294e01.json | 14 ++ ...2b2aca7f4b5805c5d3529b5951926a0e53c02.json | 28 ++++ ...8d4c2c10d4c95bdc084a9e87288b87d37fd1a.json | 14 ++ ...0e0b2f1028be4ef44d663c0ea09127ad9f1b7.json | 66 ++++++++ ...8fff08a5b6c7cff4cc9de9acd7b9085616b4b.json | 132 ++++++++++++++++ ...bac85b1c4cec7216aa11ded2032c5a80b889e.json | 88 +++++++++++ ...72c89491848279a33298d20218cfeea9dc2b4.json | 102 ++++++++++++ ...0ae5a1ffad6e9af8e0435180f743bcbfb2a18.json | 17 ++ ...775390aa222914065a0777a67091767032250.json | 28 ++++ ...bb9e44c2a1d6b167904e6b8b6af3800440fd2.json | 28 ++++ ...624e2983d83bc1800ab2ba08622afa4ea9066.json | 52 ++++++ ...edf55b6de4c4a124216006512dc05572ed78e.json | 28 ++++ ...f2fadc2f1012f04a681d137fb0e33dc3afea2.json | 14 ++ ...d9a40ea223bf3407fb1f84b5263198f67c32b.json | 17 ++ 35 files changed, 1814 insertions(+), 3 deletions(-) create mode 100644 apps/labrinth/.sqlx/query-052ac329820a9e2fa9951f4b39a10dee885caa54ae8082b6cf9417ce8dd17597.json create mode 100644 apps/labrinth/.sqlx/query-05344e329201e94246c428d34ad8569aa16c555283d5fb3bc9849e15fa17ef39.json create mode 100644 apps/labrinth/.sqlx/query-059765ac84ce39c47f99bcf415bd307f5bf46989fdbae4d0829141301f61b36d.json create mode 100644 apps/labrinth/.sqlx/query-0a1106008e49501cc33f418974c0dce39f0d8deaca7775e9d5ff8f0d41a044b0.json create mode 100644 apps/labrinth/.sqlx/query-0fe059fc4a9430c807795fd7b909723b6889d7e462ca1c7e87320caed28f9a7c.json create mode 100644 apps/labrinth/.sqlx/query-0fe6c80b9d078cf9457ba72716c793f4eeb9fedaa2cfd6ac7bdd99f10b14c511.json create mode 100644 apps/labrinth/.sqlx/query-11769eb1c8764857579505a52cd9ab18ca959bce216193d8c1aa7cb61d9c1ffc.json create mode 100644 apps/labrinth/.sqlx/query-13d3731070096afa00698998459ea32ead13b547476e3900ee767d4fedd3cb68.json create mode 100644 apps/labrinth/.sqlx/query-15ca266ff7c1f2f7f6dfe10defafacb4b5b6b6eeb3519ac614e2aea07dad5dfb.json create mode 100644 apps/labrinth/.sqlx/query-18b4cd2bb06995aebdd157900b14d2a0476843c808a03f54a7e628dfa479ec0b.json create mode 100644 apps/labrinth/.sqlx/query-2026e6f5ceac9e6591acf88fc741a24d798a699f43f9c2bef8198244efc57038.json create mode 100644 apps/labrinth/.sqlx/query-2c5eaf5fe4a303de0dc848e84b28770dc4d19ce2968a4a97be5f4705710191b1.json create mode 100644 apps/labrinth/.sqlx/query-2f1ddd1e833270ca93ced1d38da87fb2fa4d56739ad38d02d3b305777b4d9046.json create mode 100644 apps/labrinth/.sqlx/query-325c48f77ec92f11c104c5c4353c04dd48751a393877b9923e047eef5bb1d9e4.json create mode 100644 apps/labrinth/.sqlx/query-3ea3b382fd37491079984f165801d969860f2758c6ef2354997d8451abdc4468.json rename apps/labrinth/.sqlx/{query-43a668954d69d7d450f73077d6fa89fd9d47eedd1e627079ddc3cb89bc9ec4f2.json => query-4b5ae530df1467e490fb7c2c02ef35f18d45db9476dcdeeeb6ffef851f30a582.json} (92%) create mode 100644 apps/labrinth/.sqlx/query-4d434f2ec68539ec88460fb325be96bcee32af66a8a754a5e1dcac03929a224f.json create mode 100644 apps/labrinth/.sqlx/query-4f71596f731158bdf72079623b147944e4d9193d8fbd43696f958e21018fb376.json create mode 100644 apps/labrinth/.sqlx/query-63c0675524e625a090cfb56b1cb8d561f635ffcc0891f4c2e8dbfadedc8fe651.json create mode 100644 apps/labrinth/.sqlx/query-672f0342b4986d71d559a00907e3062e95de8eca1cc09d1caeab92114ff65d0b.json create mode 100644 apps/labrinth/.sqlx/query-7285d8fc34604cc1364decbbc5dca1260cc250ea4455402f84e9f1530c31df87.json create mode 100644 apps/labrinth/.sqlx/query-7934e9d059f935fbe13924b27299a25a281089126f997db20ba3a00d04294e01.json create mode 100644 apps/labrinth/.sqlx/query-94ce372258b35283e392b19c1242b2aca7f4b5805c5d3529b5951926a0e53c02.json create mode 100644 apps/labrinth/.sqlx/query-99a623492d62c03e0c9f4dfe5358d4c2c10d4c95bdc084a9e87288b87d37fd1a.json create mode 100644 apps/labrinth/.sqlx/query-a9450319c2576b9a3076fed184e0e0b2f1028be4ef44d663c0ea09127ad9f1b7.json create mode 100644 apps/labrinth/.sqlx/query-aae6384e06abe8d7f6583fe8fe78fff08a5b6c7cff4cc9de9acd7b9085616b4b.json create mode 100644 apps/labrinth/.sqlx/query-b29282c47f6768fdb629d1add77bac85b1c4cec7216aa11ded2032c5a80b889e.json create mode 100644 apps/labrinth/.sqlx/query-b69bb764f5b415b75e830fb11d272c89491848279a33298d20218cfeea9dc2b4.json create mode 100644 apps/labrinth/.sqlx/query-bcfe1a5dc34a9f65e2a8de9b1b00ae5a1ffad6e9af8e0435180f743bcbfb2a18.json create mode 100644 apps/labrinth/.sqlx/query-be342f33880f6990e76255ad01c775390aa222914065a0777a67091767032250.json create mode 100644 apps/labrinth/.sqlx/query-c1e150a9a958dcb9fc06ee3b997bb9e44c2a1d6b167904e6b8b6af3800440fd2.json create mode 100644 apps/labrinth/.sqlx/query-c92428782f51d3dec6584b55455624e2983d83bc1800ab2ba08622afa4ea9066.json create mode 100644 apps/labrinth/.sqlx/query-e19110b3c4230a260b0007d8d35edf55b6de4c4a124216006512dc05572ed78e.json create mode 100644 apps/labrinth/.sqlx/query-f66ec4d671f5a24f10369557778f2fadc2f1012f04a681d137fb0e33dc3afea2.json create mode 100644 apps/labrinth/.sqlx/query-fe94b8e26665ba77421fdb4d436d9a40ea223bf3407fb1f84b5263198f67c32b.json diff --git a/apps/labrinth/.sqlx/query-052ac329820a9e2fa9951f4b39a10dee885caa54ae8082b6cf9417ce8dd17597.json b/apps/labrinth/.sqlx/query-052ac329820a9e2fa9951f4b39a10dee885caa54ae8082b6cf9417ce8dd17597.json new file mode 100644 index 00000000000..967f55f5dcd --- /dev/null +++ b/apps/labrinth/.sqlx/query-052ac329820a9e2fa9951f4b39a10dee885caa54ae8082b6cf9417ce8dd17597.json @@ -0,0 +1,104 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE project_flag_rules\n SET name = $2, params = $3, updated_by = $4, updated_at = $5\n WHERE id = $1\n RETURNING id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Varchar", + "Jsonb", + "Int8", + "Timestamptz" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "052ac329820a9e2fa9951f4b39a10dee885caa54ae8082b6cf9417ce8dd17597" +} diff --git a/apps/labrinth/.sqlx/query-05344e329201e94246c428d34ad8569aa16c555283d5fb3bc9849e15fa17ef39.json b/apps/labrinth/.sqlx/query-05344e329201e94246c428d34ad8569aa16c555283d5fb3bc9849e15fa17ef39.json new file mode 100644 index 00000000000..5af52b36bea --- /dev/null +++ b/apps/labrinth/.sqlx/query-05344e329201e94246c428d34ad8569aa16c555283d5fb3bc9849e15fa17ef39.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE project_flag_runs AS runs\n SET status = 'done', completed_at = CURRENT_TIMESTAMP\n WHERE runs.id = ANY($1)\n AND runs.status = 'running'\n AND (\n SELECT COUNT(*)\n FROM project_flag_run_results AS results\n WHERE results.run_id = runs.id\n ) >= runs.total_projects\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "05344e329201e94246c428d34ad8569aa16c555283d5fb3bc9849e15fa17ef39" +} diff --git a/apps/labrinth/.sqlx/query-059765ac84ce39c47f99bcf415bd307f5bf46989fdbae4d0829141301f61b36d.json b/apps/labrinth/.sqlx/query-059765ac84ce39c47f99bcf415bd307f5bf46989fdbae4d0829141301f61b36d.json new file mode 100644 index 00000000000..dd7871e2484 --- /dev/null +++ b/apps/labrinth/.sqlx/query-059765ac84ce39c47f99bcf415bd307f5bf46989fdbae4d0829141301f61b36d.json @@ -0,0 +1,98 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n FROM project_flag_rules\n ORDER BY id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "059765ac84ce39c47f99bcf415bd307f5bf46989fdbae4d0829141301f61b36d" +} diff --git a/apps/labrinth/.sqlx/query-0a1106008e49501cc33f418974c0dce39f0d8deaca7775e9d5ff8f0d41a044b0.json b/apps/labrinth/.sqlx/query-0a1106008e49501cc33f418974c0dce39f0d8deaca7775e9d5ff8f0d41a044b0.json new file mode 100644 index 00000000000..d970933f6e5 --- /dev/null +++ b/apps/labrinth/.sqlx/query-0a1106008e49501cc33f418974c0dce39f0d8deaca7775e9d5ff8f0d41a044b0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO project_flag_run_results\n (id, run_id, project_id, rules, error)\n SELECT results.id, results.run_id, results.project_id, results.rules, results.error\n FROM jsonb_to_recordset($1) AS results(\n id UUID, run_id UUID, project_id BIGINT, rules JSONB, error TEXT\n )\n LEFT JOIN project_flag_runs AS runs ON runs.id = results.run_id\n WHERE results.run_id IS NULL OR runs.status = 'running'\n ORDER BY results.run_id, results.project_id\n ON CONFLICT (run_id, project_id) DO UPDATE\n SET rules = EXCLUDED.rules,\n error = EXCLUDED.error, evaluated_at = CURRENT_TIMESTAMP\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "0a1106008e49501cc33f418974c0dce39f0d8deaca7775e9d5ff8f0d41a044b0" +} diff --git a/apps/labrinth/.sqlx/query-0fe059fc4a9430c807795fd7b909723b6889d7e462ca1c7e87320caed28f9a7c.json b/apps/labrinth/.sqlx/query-0fe059fc4a9430c807795fd7b909723b6889d7e462ca1c7e87320caed28f9a7c.json new file mode 100644 index 00000000000..176c2abd4b8 --- /dev/null +++ b/apps/labrinth/.sqlx/query-0fe059fc4a9430c807795fd7b909723b6889d7e462ca1c7e87320caed28f9a7c.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, name, version_number, date_published\n FROM versions\n WHERE id = ANY($1)\n ORDER BY date_published DESC, id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "versions", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "version_number", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "versions", + "name": "version_number" + } + } + }, + { + "ordinal": 3, + "name": "date_published", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "versions", + "name": "date_published" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "0fe059fc4a9430c807795fd7b909723b6889d7e462ca1c7e87320caed28f9a7c" +} diff --git a/apps/labrinth/.sqlx/query-0fe6c80b9d078cf9457ba72716c793f4eeb9fedaa2cfd6ac7bdd99f10b14c511.json b/apps/labrinth/.sqlx/query-0fe6c80b9d078cf9457ba72716c793f4eeb9fedaa2cfd6ac7bdd99f10b14c511.json new file mode 100644 index 00000000000..355c8aec4d3 --- /dev/null +++ b/apps/labrinth/.sqlx/query-0fe6c80b9d078cf9457ba72716c793f4eeb9fedaa2cfd6ac7bdd99f10b14c511.json @@ -0,0 +1,148 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, title, status, link, exceptions, proof, flame_project_id,\n inserted_at, inserted_by, updated_at, updated_by\n FROM moderation_external_licenses\n WHERE id = ANY($1)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "title", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "title" + } + } + }, + { + "ordinal": 2, + "name": "status", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "status" + } + } + }, + { + "ordinal": 3, + "name": "link", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "link" + } + } + }, + { + "ordinal": 4, + "name": "exceptions", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "exceptions" + } + } + }, + { + "ordinal": 5, + "name": "proof", + "type_info": "Text", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "proof" + } + } + }, + { + "ordinal": 6, + "name": "flame_project_id", + "type_info": "Int4", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "flame_project_id" + } + } + }, + { + "ordinal": 7, + "name": "inserted_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "inserted_at" + } + } + }, + { + "ordinal": 8, + "name": "inserted_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "inserted_by" + } + } + }, + { + "ordinal": 9, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "updated_at" + } + } + }, + { + "ordinal": 10, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "moderation_external_licenses", + "name": "updated_by" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "0fe6c80b9d078cf9457ba72716c793f4eeb9fedaa2cfd6ac7bdd99f10b14c511" +} diff --git a/apps/labrinth/.sqlx/query-11769eb1c8764857579505a52cd9ab18ca959bce216193d8c1aa7cb61d9c1ffc.json b/apps/labrinth/.sqlx/query-11769eb1c8764857579505a52cd9ab18ca959bce216193d8c1aa7cb61d9c1ffc.json new file mode 100644 index 00000000000..071a15182d6 --- /dev/null +++ b/apps/labrinth/.sqlx/query-11769eb1c8764857579505a52cd9ab18ca959bce216193d8c1aa7cb61d9c1ffc.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE project_flags SET run_id = NULL WHERE run_id = $1 AND active", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "11769eb1c8764857579505a52cd9ab18ca959bce216193d8c1aa7cb61d9c1ffc" +} diff --git a/apps/labrinth/.sqlx/query-13d3731070096afa00698998459ea32ead13b547476e3900ee767d4fedd3cb68.json b/apps/labrinth/.sqlx/query-13d3731070096afa00698998459ea32ead13b547476e3900ee767d4fedd3cb68.json new file mode 100644 index 00000000000..e0d73a701f2 --- /dev/null +++ b/apps/labrinth/.sqlx/query-13d3731070096afa00698998459ea32ead13b547476e3900ee767d4fedd3cb68.json @@ -0,0 +1,101 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n FROM project_flag_rules\n ORDER BY created_at DESC, id DESC\n LIMIT $1 OFFSET $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "13d3731070096afa00698998459ea32ead13b547476e3900ee767d4fedd3cb68" +} diff --git a/apps/labrinth/.sqlx/query-15ca266ff7c1f2f7f6dfe10defafacb4b5b6b6eeb3519ac614e2aea07dad5dfb.json b/apps/labrinth/.sqlx/query-15ca266ff7c1f2f7f6dfe10defafacb4b5b6b6eeb3519ac614e2aea07dad5dfb.json new file mode 100644 index 00000000000..8ab51c69893 --- /dev/null +++ b/apps/labrinth/.sqlx/query-15ca266ff7c1f2f7f6dfe10defafacb4b5b6b6eeb3519ac614e2aea07dad5dfb.json @@ -0,0 +1,100 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n FROM project_flag_rules\n WHERE id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "15ca266ff7c1f2f7f6dfe10defafacb4b5b6b6eeb3519ac614e2aea07dad5dfb" +} diff --git a/apps/labrinth/.sqlx/query-18b4cd2bb06995aebdd157900b14d2a0476843c808a03f54a7e628dfa479ec0b.json b/apps/labrinth/.sqlx/query-18b4cd2bb06995aebdd157900b14d2a0476843c808a03f54a7e628dfa479ec0b.json new file mode 100644 index 00000000000..b09439b73cb --- /dev/null +++ b/apps/labrinth/.sqlx/query-18b4cd2bb06995aebdd157900b14d2a0476843c808a03f54a7e628dfa479ec0b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT (\n SELECT COUNT(*)\n FROM project_flag_run_results\n WHERE run_id = runs.id\n ) AS \"total!\"\n FROM project_flag_runs AS runs\n WHERE runs.id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "18b4cd2bb06995aebdd157900b14d2a0476843c808a03f54a7e628dfa479ec0b" +} diff --git a/apps/labrinth/.sqlx/query-2026e6f5ceac9e6591acf88fc741a24d798a699f43f9c2bef8198244efc57038.json b/apps/labrinth/.sqlx/query-2026e6f5ceac9e6591acf88fc741a24d798a699f43f9c2bef8198244efc57038.json new file mode 100644 index 00000000000..3d0255b806a --- /dev/null +++ b/apps/labrinth/.sqlx/query-2026e6f5ceac9e6591acf88fc741a24d798a699f43f9c2bef8198244efc57038.json @@ -0,0 +1,83 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT source_version.mod_id AS project_id,\n ofs.file_path AS \"file_path!\", CONVERT_FROM(ofs.sha1, 'UTF8') AS \"sha1!\",\n source_file.version_id, platform_file.version_id AS platform_version_id,\n platform_version.mod_id AS platform_project_id\n FROM files source_file\n INNER JOIN versions source_version ON source_version.id = source_file.version_id\n INNER JOIN override_file_sources ofs ON ofs.file_id = source_file.id\n INNER JOIN hashes h ON h.algorithm = 'sha1' AND h.hash = ofs.sha1\n INNER JOIN files platform_file ON platform_file.id = h.file_id\n INNER JOIN versions platform_version ON platform_version.id = platform_file.version_id\n WHERE source_version.mod_id = ANY($1)\n ORDER BY source_version.mod_id, source_file.version_id, ofs.file_path,\n platform_file.version_id, platform_version.mod_id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "project_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "mod_id" + } + } + }, + { + "ordinal": 1, + "name": "file_path!", + "type_info": "Text", + "origin": { + "Table": { + "table": "override_file_sources", + "name": "file_path" + } + } + }, + { + "ordinal": 2, + "name": "sha1!", + "type_info": "Text", + "origin": "Expression" + }, + { + "ordinal": 3, + "name": "version_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "files", + "name": "version_id" + } + } + }, + { + "ordinal": 4, + "name": "platform_version_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "files", + "name": "version_id" + } + } + }, + { + "ordinal": 5, + "name": "platform_project_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "mod_id" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + null, + false, + false, + false + ] + }, + "hash": "2026e6f5ceac9e6591acf88fc741a24d798a699f43f9c2bef8198244efc57038" +} diff --git a/apps/labrinth/.sqlx/query-2c5eaf5fe4a303de0dc848e84b28770dc4d19ce2968a4a97be5f4705710191b1.json b/apps/labrinth/.sqlx/query-2c5eaf5fe4a303de0dc848e84b28770dc4d19ce2968a4a97be5f4705710191b1.json new file mode 100644 index 00000000000..1edaa099c14 --- /dev/null +++ b/apps/labrinth/.sqlx/query-2c5eaf5fe4a303de0dc848e84b28770dc4d19ce2968a4a97be5f4705710191b1.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM project_flag_runs WHERE id = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "2c5eaf5fe4a303de0dc848e84b28770dc4d19ce2968a4a97be5f4705710191b1" +} diff --git a/apps/labrinth/.sqlx/query-2f1ddd1e833270ca93ced1d38da87fb2fa4d56739ad38d02d3b305777b4d9046.json b/apps/labrinth/.sqlx/query-2f1ddd1e833270ca93ced1d38da87fb2fa4d56739ad38d02d3b305777b4d9046.json new file mode 100644 index 00000000000..6b740cb604a --- /dev/null +++ b/apps/labrinth/.sqlx/query-2f1ddd1e833270ca93ced1d38da87fb2fa4d56739ad38d02d3b305777b4d9046.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT flags.what AS \"what?: sqlx::types::Json\"\n FROM mods AS projects\n LEFT JOIN project_flags AS flags ON flags.project_id = projects.id\n AND (flags.active OR (flags.what->>'retain_if_removed')::bool = TRUE)\n WHERE projects.id = $1\n ORDER BY flags.what->>'key', flags.id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "what?: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flags", + "name": "what" + } + } + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "2f1ddd1e833270ca93ced1d38da87fb2fa4d56739ad38d02d3b305777b4d9046" +} diff --git a/apps/labrinth/.sqlx/query-325c48f77ec92f11c104c5c4353c04dd48751a393877b9923e047eef5bb1d9e4.json b/apps/labrinth/.sqlx/query-325c48f77ec92f11c104c5c4353c04dd48751a393877b9923e047eef5bb1d9e4.json new file mode 100644 index 00000000000..9bd2e62deb2 --- /dev/null +++ b/apps/labrinth/.sqlx/query-325c48f77ec92f11c104c5c4353c04dd48751a393877b9923e047eef5bb1d9e4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM project_flag_runs\n WHERE id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "325c48f77ec92f11c104c5c4353c04dd48751a393877b9923e047eef5bb1d9e4" +} diff --git a/apps/labrinth/.sqlx/query-3ea3b382fd37491079984f165801d969860f2758c6ef2354997d8451abdc4468.json b/apps/labrinth/.sqlx/query-3ea3b382fd37491079984f165801d969860f2758c6ef2354997d8451abdc4468.json new file mode 100644 index 00000000000..89161cdf027 --- /dev/null +++ b/apps/labrinth/.sqlx/query-3ea3b382fd37491079984f165801d969860f2758c6ef2354997d8451abdc4468.json @@ -0,0 +1,104 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO project_flag_rules\n (id, name, params, created_by, created_at, updated_by, updated_at)\n VALUES ($1, $2, $3, $4, $5, $4, $5)\n RETURNING id, name, params AS \"params: sqlx::types::Json\",\n created_by, created_at, updated_by, updated_at\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "params: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "params" + } + } + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_by" + } + } + }, + { + "ordinal": 4, + "name": "created_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "created_at" + } + } + }, + { + "ordinal": 5, + "name": "updated_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_by" + } + } + }, + { + "ordinal": 6, + "name": "updated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "updated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Varchar", + "Jsonb", + "Int8", + "Timestamptz" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + false + ] + }, + "hash": "3ea3b382fd37491079984f165801d969860f2758c6ef2354997d8451abdc4468" +} diff --git a/apps/labrinth/.sqlx/query-43a668954d69d7d450f73077d6fa89fd9d47eedd1e627079ddc3cb89bc9ec4f2.json b/apps/labrinth/.sqlx/query-4b5ae530df1467e490fb7c2c02ef35f18d45db9476dcdeeeb6ffef851f30a582.json similarity index 92% rename from apps/labrinth/.sqlx/query-43a668954d69d7d450f73077d6fa89fd9d47eedd1e627079ddc3cb89bc9ec4f2.json rename to apps/labrinth/.sqlx/query-4b5ae530df1467e490fb7c2c02ef35f18d45db9476dcdeeeb6ffef851f30a582.json index b11e945aafa..b95ad779930 100644 --- a/apps/labrinth/.sqlx/query-43a668954d69d7d450f73077d6fa89fd9d47eedd1e627079ddc3cb89bc9ec4f2.json +++ b/apps/labrinth/.sqlx/query-4b5ae530df1467e490fb7c2c02ef35f18d45db9476dcdeeeb6ffef851f30a582.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n\t\t\tSELECT project_id, type AS \"disclosure_type!\", metadata, updated_at, updated_by, set_by_moderator, deleted_at, lock_status\n\t\t\tFROM project_disclosures\n\t\t\tWHERE project_id = $1 AND ($2 OR deleted_at IS NULL)\n\t\t\tORDER BY updated_at DESC\n\t\t\t", + "query": "\n\t\t\tSELECT project_id, type AS \"disclosure_type!\", metadata, updated_at, updated_by, set_by_moderator, deleted_at, lock_status\n\t\t\tFROM project_disclosures\n\t\t\tWHERE project_id = ANY($1) AND ($2 OR deleted_at IS NULL)\n\t\t\tORDER BY updated_at DESC\n\t\t\t", "describe": { "columns": [ { @@ -94,7 +94,7 @@ ], "parameters": { "Left": [ - "Int8", + "Int8Array", "Bool" ] }, @@ -109,5 +109,5 @@ false ] }, - "hash": "43a668954d69d7d450f73077d6fa89fd9d47eedd1e627079ddc3cb89bc9ec4f2" + "hash": "4b5ae530df1467e490fb7c2c02ef35f18d45db9476dcdeeeb6ffef851f30a582" } diff --git a/apps/labrinth/.sqlx/query-4d434f2ec68539ec88460fb325be96bcee32af66a8a754a5e1dcac03929a224f.json b/apps/labrinth/.sqlx/query-4d434f2ec68539ec88460fb325be96bcee32af66a8a754a5e1dcac03929a224f.json new file mode 100644 index 00000000000..d03ee52bcf2 --- /dev/null +++ b/apps/labrinth/.sqlx/query-4d434f2ec68539ec88460fb325be96bcee32af66a8a754a5e1dcac03929a224f.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id FROM mods\n WHERE id IN (SELECT project_id FROM project_flags WHERE run_id = $1)\n ORDER BY id\n FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "mods", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4d434f2ec68539ec88460fb325be96bcee32af66a8a754a5e1dcac03929a224f" +} diff --git a/apps/labrinth/.sqlx/query-4f71596f731158bdf72079623b147944e4d9193d8fbd43696f958e21018fb376.json b/apps/labrinth/.sqlx/query-4f71596f731158bdf72079623b147944e4d9193d8fbd43696f958e21018fb376.json new file mode 100644 index 00000000000..25f0d323031 --- /dev/null +++ b/apps/labrinth/.sqlx/query-4f71596f731158bdf72079623b147944e4d9193d8fbd43696f958e21018fb376.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name FROM project_flag_rules WHERE id = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "project_flag_rules", + "name": "name" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "4f71596f731158bdf72079623b147944e4d9193d8fbd43696f958e21018fb376" +} diff --git a/apps/labrinth/.sqlx/query-63c0675524e625a090cfb56b1cb8d561f635ffcc0891f4c2e8dbfadedc8fe651.json b/apps/labrinth/.sqlx/query-63c0675524e625a090cfb56b1cb8d561f635ffcc0891f4c2e8dbfadedc8fe651.json new file mode 100644 index 00000000000..742b503facb --- /dev/null +++ b/apps/labrinth/.sqlx/query-63c0675524e625a090cfb56b1cb8d561f635ffcc0891f4c2e8dbfadedc8fe651.json @@ -0,0 +1,88 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, project_id, flame_project, attribution, attributed_at, attributed_by\n FROM project_attribution_groups\n WHERE project_id = ANY($1)\n ORDER BY project_id, id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "project_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "project_id" + } + } + }, + { + "ordinal": 2, + "name": "flame_project", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "flame_project" + } + } + }, + { + "ordinal": 3, + "name": "attribution", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "attribution" + } + } + }, + { + "ordinal": 4, + "name": "attributed_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "attributed_at" + } + } + }, + { + "ordinal": 5, + "name": "attributed_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_groups", + "name": "attributed_by" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + true, + true, + true, + true + ] + }, + "hash": "63c0675524e625a090cfb56b1cb8d561f635ffcc0891f4c2e8dbfadedc8fe651" +} diff --git a/apps/labrinth/.sqlx/query-672f0342b4986d71d559a00907e3062e95de8eca1cc09d1caeab92114ff65d0b.json b/apps/labrinth/.sqlx/query-672f0342b4986d71d559a00907e3062e95de8eca1cc09d1caeab92114ff65d0b.json new file mode 100644 index 00000000000..5867ff937b3 --- /dev/null +++ b/apps/labrinth/.sqlx/query-672f0342b4986d71d559a00907e3062e95de8eca1cc09d1caeab92114ff65d0b.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT status AS \"status: ProjectFlagRunStatus\", applied_at,\n total_projects, error\n FROM project_flag_runs\n WHERE id = $1\n FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "status: ProjectFlagRunStatus", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "status" + } + } + }, + { + "ordinal": 1, + "name": "applied_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "applied_at" + } + } + }, + { + "ordinal": 2, + "name": "total_projects", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "total_projects" + } + } + }, + { + "ordinal": 3, + "name": "error", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "error" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + true, + false, + true + ] + }, + "hash": "672f0342b4986d71d559a00907e3062e95de8eca1cc09d1caeab92114ff65d0b" +} diff --git a/apps/labrinth/.sqlx/query-7285d8fc34604cc1364decbbc5dca1260cc250ea4455402f84e9f1530c31df87.json b/apps/labrinth/.sqlx/query-7285d8fc34604cc1364decbbc5dca1260cc250ea4455402f84e9f1530c31df87.json new file mode 100644 index 00000000000..f8095f9d8bf --- /dev/null +++ b/apps/labrinth/.sqlx/query-7285d8fc34604cc1364decbbc5dca1260cc250ea4455402f84e9f1530c31df87.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM mods WHERE status = 'processing'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "mods", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "7285d8fc34604cc1364decbbc5dca1260cc250ea4455402f84e9f1530c31df87" +} diff --git a/apps/labrinth/.sqlx/query-7934e9d059f935fbe13924b27299a25a281089126f997db20ba3a00d04294e01.json b/apps/labrinth/.sqlx/query-7934e9d059f935fbe13924b27299a25a281089126f997db20ba3a00d04294e01.json new file mode 100644 index 00000000000..d9d07efe690 --- /dev/null +++ b/apps/labrinth/.sqlx/query-7934e9d059f935fbe13924b27299a25a281089126f997db20ba3a00d04294e01.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO project_flags (id, run_id, project_id, what, active)\n SELECT flags.id, flags.run_id, flags.project_id, flags.what, TRUE\n FROM jsonb_to_recordset($1) AS flags(\n id BIGINT, run_id UUID, project_id BIGINT, what JSONB\n )\n ORDER BY flags.project_id, flags.what->>'key'\n ON CONFLICT (project_id, (what->>'key')) DO UPDATE\n SET active = TRUE,\n run_id = EXCLUDED.run_id,\n what = EXCLUDED.what || jsonb_build_object(\n 'retain_if_removed',\n COALESCE((project_flags.what->>'retain_if_removed')::bool, FALSE)\n OR COALESCE((EXCLUDED.what->>'retain_if_removed')::bool, FALSE)\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "7934e9d059f935fbe13924b27299a25a281089126f997db20ba3a00d04294e01" +} diff --git a/apps/labrinth/.sqlx/query-94ce372258b35283e392b19c1242b2aca7f4b5805c5d3529b5951926a0e53c02.json b/apps/labrinth/.sqlx/query-94ce372258b35283e392b19c1242b2aca7f4b5805c5d3529b5951926a0e53c02.json new file mode 100644 index 00000000000..fb3bf0837f1 --- /dev/null +++ b/apps/labrinth/.sqlx/query-94ce372258b35283e392b19c1242b2aca7f4b5805c5d3529b5951926a0e53c02.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id FROM mods\n WHERE id = ANY($1)\n ORDER BY id\n FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "mods", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false + ] + }, + "hash": "94ce372258b35283e392b19c1242b2aca7f4b5805c5d3529b5951926a0e53c02" +} diff --git a/apps/labrinth/.sqlx/query-99a623492d62c03e0c9f4dfe5358d4c2c10d4c95bdc084a9e87288b87d37fd1a.json b/apps/labrinth/.sqlx/query-99a623492d62c03e0c9f4dfe5358d4c2c10d4c95bdc084a9e87288b87d37fd1a.json new file mode 100644 index 00000000000..1576fce8d11 --- /dev/null +++ b/apps/labrinth/.sqlx/query-99a623492d62c03e0c9f4dfe5358d4c2c10d4c95bdc084a9e87288b87d37fd1a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM project_flag_rules\n WHERE id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "99a623492d62c03e0c9f4dfe5358d4c2c10d4c95bdc084a9e87288b87d37fd1a" +} diff --git a/apps/labrinth/.sqlx/query-a9450319c2576b9a3076fed184e0e0b2f1028be4ef44d663c0ea09127ad9f1b7.json b/apps/labrinth/.sqlx/query-a9450319c2576b9a3076fed184e0e0b2f1028be4ef44d663c0ea09127ad9f1b7.json new file mode 100644 index 00000000000..9f83cb088cc --- /dev/null +++ b/apps/labrinth/.sqlx/query-a9450319c2576b9a3076fed184e0e0b2f1028be4ef44d663c0ea09127ad9f1b7.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT paf.group_id AS \"group_id!\", paf.name AS \"name!\",\n CONVERT_FROM(paf.sha1, 'UTF8') AS \"sha1!\",\n paf.moderation_external_license_id,\n COALESCE(ARRAY_AGG(DISTINCT aev.id) FILTER (WHERE aev.id IS NOT NULL), '{}') AS \"version_ids!: Vec\"\n FROM project_attribution_files paf\n INNER JOIN project_attribution_groups pag ON pag.id = paf.group_id\n LEFT JOIN override_file_sources ofs ON ofs.sha1 = paf.sha1\n LEFT JOIN files f ON f.id = ofs.file_id\n LEFT JOIN versions v ON v.id = f.version_id AND v.mod_id = pag.project_id\n LEFT JOIN attribution_enforced_versions aev ON aev.id = v.id\n WHERE paf.group_id = ANY($1)\n GROUP BY paf.group_id, paf.name, paf.sha1, paf.moderation_external_license_id\n ORDER BY paf.group_id, paf.name, paf.sha1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "group_id!", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_files", + "name": "group_id" + } + } + }, + { + "ordinal": 1, + "name": "name!", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_attribution_files", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "sha1!", + "type_info": "Text", + "origin": "Expression" + }, + { + "ordinal": 3, + "name": "moderation_external_license_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_attribution_files", + "name": "moderation_external_license_id" + } + } + }, + { + "ordinal": 4, + "name": "version_ids!: Vec", + "type_info": "Int8Array", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + null, + true, + null + ] + }, + "hash": "a9450319c2576b9a3076fed184e0e0b2f1028be4ef44d663c0ea09127ad9f1b7" +} diff --git a/apps/labrinth/.sqlx/query-aae6384e06abe8d7f6583fe8fe78fff08a5b6c7cff4cc9de9acd7b9085616b4b.json b/apps/labrinth/.sqlx/query-aae6384e06abe8d7f6583fe8fe78fff08a5b6c7cff4cc9de9acd7b9085616b4b.json new file mode 100644 index 00000000000..eceef4e251c --- /dev/null +++ b/apps/labrinth/.sqlx/query-aae6384e06abe8d7f6583fe8fe78fff08a5b6c7cff4cc9de9acd7b9085616b4b.json @@ -0,0 +1,132 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, status AS \"status: ProjectFlagRunStatus\", total_projects, started_by,\n started_at, completed_at, applied_at,\n output AS \"output: sqlx::types::Json\", error,\n (SELECT COUNT(*) FROM project_flag_run_results AS results\n WHERE results.run_id = project_flag_runs.id AND results.error IS NOT NULL\n ) AS \"num_errors!\"\n FROM project_flag_runs\n ORDER BY started_at DESC, id DESC\n LIMIT $1 OFFSET $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "status: ProjectFlagRunStatus", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "status" + } + } + }, + { + "ordinal": 2, + "name": "total_projects", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "total_projects" + } + } + }, + { + "ordinal": 3, + "name": "started_by", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "started_by" + } + } + }, + { + "ordinal": 4, + "name": "started_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "started_at" + } + } + }, + { + "ordinal": 5, + "name": "completed_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "completed_at" + } + } + }, + { + "ordinal": 6, + "name": "applied_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "applied_at" + } + } + }, + { + "ordinal": 7, + "name": "output: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "output" + } + } + }, + { + "ordinal": 8, + "name": "error", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "error" + } + } + }, + { + "ordinal": 9, + "name": "num_errors!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true, + false, + true, + true, + true, + true, + null + ] + }, + "hash": "aae6384e06abe8d7f6583fe8fe78fff08a5b6c7cff4cc9de9acd7b9085616b4b" +} diff --git a/apps/labrinth/.sqlx/query-b29282c47f6768fdb629d1add77bac85b1c4cec7216aa11ded2032c5a80b889e.json b/apps/labrinth/.sqlx/query-b29282c47f6768fdb629d1add77bac85b1c4cec7216aa11ded2032c5a80b889e.json new file mode 100644 index 00000000000..e5a95a69e7b --- /dev/null +++ b/apps/labrinth/.sqlx/query-b29282c47f6768fdb629d1add77bac85b1c4cec7216aa11ded2032c5a80b889e.json @@ -0,0 +1,88 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT v.mod_id AS project_id, f.id, f.version_id, f.filename, f.url, f.size\n FROM files f\n INNER JOIN versions v ON v.id = f.version_id\n WHERE v.mod_id = ANY($1)\n ORDER BY v.mod_id, v.date_published DESC, v.id DESC, f.id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "project_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "mod_id" + } + } + }, + { + "ordinal": 1, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "files", + "name": "id" + } + } + }, + { + "ordinal": 2, + "name": "version_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "files", + "name": "version_id" + } + } + }, + { + "ordinal": 3, + "name": "filename", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "files", + "name": "filename" + } + } + }, + { + "ordinal": 4, + "name": "url", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "files", + "name": "url" + } + } + }, + { + "ordinal": 5, + "name": "size", + "type_info": "Int4", + "origin": { + "Table": { + "table": "files", + "name": "size" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false + ] + }, + "hash": "b29282c47f6768fdb629d1add77bac85b1c4cec7216aa11ded2032c5a80b889e" +} diff --git a/apps/labrinth/.sqlx/query-b69bb764f5b415b75e830fb11d272c89491848279a33298d20218cfeea9dc2b4.json b/apps/labrinth/.sqlx/query-b69bb764f5b415b75e830fb11d272c89491848279a33298d20218cfeea9dc2b4.json new file mode 100644 index 00000000000..54f20ff89c4 --- /dev/null +++ b/apps/labrinth/.sqlx/query-b69bb764f5b415b75e830fb11d272c89491848279a33298d20218cfeea9dc2b4.json @@ -0,0 +1,102 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT projects.id, projects.name AS title, projects.slug, projects.icon_url,\n results.rules AS \"rules: sqlx::types::Json\",\n results.error, results.evaluated_at\n FROM project_flag_run_results AS results\n INNER JOIN mods AS projects ON projects.id = results.project_id\n WHERE results.run_id = $1\n ORDER BY results.project_id\n LIMIT $2 OFFSET $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "mods", + "name": "id" + } + } + }, + { + "ordinal": 1, + "name": "title", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "mods", + "name": "name" + } + } + }, + { + "ordinal": 2, + "name": "slug", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "mods", + "name": "slug" + } + } + }, + { + "ordinal": 3, + "name": "icon_url", + "type_info": "Varchar", + "origin": { + "Table": { + "table": "mods", + "name": "icon_url" + } + } + }, + { + "ordinal": 4, + "name": "rules: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_run_results", + "name": "rules" + } + } + }, + { + "ordinal": 5, + "name": "error", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_flag_run_results", + "name": "error" + } + } + }, + { + "ordinal": 6, + "name": "evaluated_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_run_results", + "name": "evaluated_at" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + true, + false, + true, + false + ] + }, + "hash": "b69bb764f5b415b75e830fb11d272c89491848279a33298d20218cfeea9dc2b4" +} diff --git a/apps/labrinth/.sqlx/query-bcfe1a5dc34a9f65e2a8de9b1b00ae5a1ffad6e9af8e0435180f743bcbfb2a18.json b/apps/labrinth/.sqlx/query-bcfe1a5dc34a9f65e2a8de9b1b00ae5a1ffad6e9af8e0435180f743bcbfb2a18.json new file mode 100644 index 00000000000..1e7c7c44920 --- /dev/null +++ b/apps/labrinth/.sqlx/query-bcfe1a5dc34a9f65e2a8de9b1b00ae5a1ffad6e9af8e0435180f743bcbfb2a18.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE project_flag_runs\n SET status = $2, completed_at = $3, error = $4\n WHERE id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Timestamptz", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bcfe1a5dc34a9f65e2a8de9b1b00ae5a1ffad6e9af8e0435180f743bcbfb2a18" +} diff --git a/apps/labrinth/.sqlx/query-be342f33880f6990e76255ad01c775390aa222914065a0777a67091767032250.json b/apps/labrinth/.sqlx/query-be342f33880f6990e76255ad01c775390aa222914065a0777a67091767032250.json new file mode 100644 index 00000000000..8fdbdc8e028 --- /dev/null +++ b/apps/labrinth/.sqlx/query-be342f33880f6990e76255ad01c775390aa222914065a0777a67091767032250.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE project_flag_runs SET applied_at = CURRENT_TIMESTAMP\n WHERE id = $1\n RETURNING applied_at AS \"applied_at!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "applied_at!", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "applied_at" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "be342f33880f6990e76255ad01c775390aa222914065a0777a67091767032250" +} diff --git a/apps/labrinth/.sqlx/query-c1e150a9a958dcb9fc06ee3b997bb9e44c2a1d6b167904e6b8b6af3800440fd2.json b/apps/labrinth/.sqlx/query-c1e150a9a958dcb9fc06ee3b997bb9e44c2a1d6b167904e6b8b6af3800440fd2.json new file mode 100644 index 00000000000..3bdddf8099d --- /dev/null +++ b/apps/labrinth/.sqlx/query-c1e150a9a958dcb9fc06ee3b997bb9e44c2a1d6b167904e6b8b6af3800440fd2.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM versions WHERE mod_id = ANY($1) ORDER BY date_published DESC, id DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "versions", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c1e150a9a958dcb9fc06ee3b997bb9e44c2a1d6b167904e6b8b6af3800440fd2" +} diff --git a/apps/labrinth/.sqlx/query-c92428782f51d3dec6584b55455624e2983d83bc1800ab2ba08622afa4ea9066.json b/apps/labrinth/.sqlx/query-c92428782f51d3dec6584b55455624e2983d83bc1800ab2ba08622afa4ea9066.json new file mode 100644 index 00000000000..a14c246b734 --- /dev/null +++ b/apps/labrinth/.sqlx/query-c92428782f51d3dec6584b55455624e2983d83bc1800ab2ba08622afa4ea9066.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT project_id, rules AS \"rules: sqlx::types::Json\", error\n FROM project_flag_run_results\n WHERE run_id = $1\n ORDER BY project_id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "project_id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "project_flag_run_results", + "name": "project_id" + } + } + }, + { + "ordinal": 1, + "name": "rules: sqlx::types::Json", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "project_flag_run_results", + "name": "rules" + } + } + }, + { + "ordinal": 2, + "name": "error", + "type_info": "Text", + "origin": { + "Table": { + "table": "project_flag_run_results", + "name": "error" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "c92428782f51d3dec6584b55455624e2983d83bc1800ab2ba08622afa4ea9066" +} diff --git a/apps/labrinth/.sqlx/query-e19110b3c4230a260b0007d8d35edf55b6de4c4a124216006512dc05572ed78e.json b/apps/labrinth/.sqlx/query-e19110b3c4230a260b0007d8d35edf55b6de4c4a124216006512dc05572ed78e.json new file mode 100644 index 00000000000..5ac24edd279 --- /dev/null +++ b/apps/labrinth/.sqlx/query-e19110b3c4230a260b0007d8d35edf55b6de4c4a124216006512dc05572ed78e.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id\n FROM project_flag_runs\n WHERE id = ANY($1)\n ORDER BY id\n FOR UPDATE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid", + "origin": { + "Table": { + "table": "project_flag_runs", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e19110b3c4230a260b0007d8d35edf55b6de4c4a124216006512dc05572ed78e" +} diff --git a/apps/labrinth/.sqlx/query-f66ec4d671f5a24f10369557778f2fadc2f1012f04a681d137fb0e33dc3afea2.json b/apps/labrinth/.sqlx/query-f66ec4d671f5a24f10369557778f2fadc2f1012f04a681d137fb0e33dc3afea2.json new file mode 100644 index 00000000000..226a9110aa7 --- /dev/null +++ b/apps/labrinth/.sqlx/query-f66ec4d671f5a24f10369557778f2fadc2f1012f04a681d137fb0e33dc3afea2.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE project_flags SET active = FALSE\n WHERE project_id = ANY($1)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "f66ec4d671f5a24f10369557778f2fadc2f1012f04a681d137fb0e33dc3afea2" +} diff --git a/apps/labrinth/.sqlx/query-fe94b8e26665ba77421fdb4d436d9a40ea223bf3407fb1f84b5263198f67c32b.json b/apps/labrinth/.sqlx/query-fe94b8e26665ba77421fdb4d436d9a40ea223bf3407fb1f84b5263198f67c32b.json new file mode 100644 index 00000000000..5840bd9b02a --- /dev/null +++ b/apps/labrinth/.sqlx/query-fe94b8e26665ba77421fdb4d436d9a40ea223bf3407fb1f84b5263198f67c32b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO project_flag_runs (id, status, started_by, started_at, total_projects)\n VALUES ($1, 'running', $2, $3, $4)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int8", + "Timestamptz", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "fe94b8e26665ba77421fdb4d436d9a40ea223bf3407fb1f84b5263198f67c32b" +} From 13efdcde49c40646c800a02f3e66cee303a8ca80 Mon Sep 17 00:00:00 2001 From: tdgao Date: Thu, 5 Nov 2026 16:52:15 -0600 Subject: [PATCH 15/26] feat: surface project flags in modview --- .../project-review/flag-list/flag-card.vue | 47 ++++ .../project-review/flag-list/index.vue | 27 +++ .../src/components/project-review/index.vue | 12 +- .../project-review/project-info/section.vue | 10 +- .../project-review/review-panel/anchor.vue | 2 + .../review-panel/flags-count-badge.vue | 21 ++ .../project-review/review-panel/flags.vue | 56 +++++ .../project-review/review-panel/index.vue | 3 + .../project-review/review-panel/popover.vue | 2 +- .../re-review-findings-count-badge.vue | 4 +- .../providers/project-review/review-flags.ts | 97 ++++++++ .../labrinth/project-flags/internal.ts | 8 + .../modules/labrinth/project-flags/types.ts | 9 +- packages/assets/generated-icons.ts | 217 +++++++++--------- 14 files changed, 400 insertions(+), 115 deletions(-) create mode 100644 apps/frontend/src/components/project-review/flag-list/flag-card.vue create mode 100644 apps/frontend/src/components/project-review/flag-list/index.vue create mode 100644 apps/frontend/src/components/project-review/review-panel/flags-count-badge.vue create mode 100644 apps/frontend/src/components/project-review/review-panel/flags.vue create mode 100644 apps/frontend/src/providers/project-review/review-flags.ts diff --git a/apps/frontend/src/components/project-review/flag-list/flag-card.vue b/apps/frontend/src/components/project-review/flag-list/flag-card.vue new file mode 100644 index 00000000000..38a597789d5 --- /dev/null +++ b/apps/frontend/src/components/project-review/flag-list/flag-card.vue @@ -0,0 +1,47 @@ + + + diff --git a/apps/frontend/src/components/project-review/flag-list/index.vue b/apps/frontend/src/components/project-review/flag-list/index.vue new file mode 100644 index 00000000000..9634efacdcc --- /dev/null +++ b/apps/frontend/src/components/project-review/flag-list/index.vue @@ -0,0 +1,27 @@ + + + diff --git a/apps/frontend/src/components/project-review/index.vue b/apps/frontend/src/components/project-review/index.vue index a28cf5962d3..21672abb249 100644 --- a/apps/frontend/src/components/project-review/index.vue +++ b/apps/frontend/src/components/project-review/index.vue @@ -70,6 +70,7 @@ + @@ -105,6 +106,7 @@ import { import { isStaff } from '~/helpers/users.js' import { injectProjectReviewPageContext } from '~/providers/project-review' import { previousReviewLinkUrls } from '~/providers/project-review/project-links' +import { createReviewFlags, provideReviewFlags } from '~/providers/project-review/review-flags' import { createReviewMessages, provideReviewMessages, @@ -126,6 +128,7 @@ import { import Description from './description/index.vue' import Disclosures from './disclosures/index.vue' +import FlagList from './flag-list/index.vue' import Gallery from './gallery/index.vue' import IssueList from './issue-list/index.vue' import IssuePicker from './issue-list/issue-picker.vue' @@ -177,6 +180,7 @@ const visibleTabs = computed((previousTabs) => { ) }) const reviewProjectId = computed(() => project.value?.id) +const reviewFlags = provideReviewFlags(createReviewFlags(reviewProjectId)) const session = provideReviewSession(createReviewSession()) const settings = useModerationSettings() const panels = provideReviewPanels( @@ -298,6 +302,7 @@ const rightPanelTabs = computed(() => [ value: 'issues', label: `Issues (${issueCount.value})`, }, + { value: 'flags', label: `Flags (${reviewFlags.flags.value.length})` }, ]) watch( @@ -313,7 +318,12 @@ watch(projectId, () => { }) useShortcut('cycle-conversation', () => { - const tabs = ['issues', 'thread', ...(hasPreviousStructuredIssues.value ? ['re-review'] : [])] + const tabs = [ + 'issues', + 'flags', + 'thread', + ...(hasPreviousStructuredIssues.value ? ['re-review'] : []), + ] activeReviewTab.value = tabs[(tabs.indexOf(activeReviewTab.value) + 1) % tabs.length] shortcuts.run('reveal-right') }) diff --git a/apps/frontend/src/components/project-review/project-info/section.vue b/apps/frontend/src/components/project-review/project-info/section.vue index 556137c6943..0fd3a63d4de 100644 --- a/apps/frontend/src/components/project-review/project-info/section.vue +++ b/apps/frontend/src/components/project-review/project-info/section.vue @@ -12,10 +12,11 @@
+
@@ -28,9 +29,11 @@ import { computed } from 'vue' import type { ReviewTarget } from '~/providers/project-review/review' +import { injectReviewFlags } from '~/providers/project-review/review-flags' import { injectReviewPanels } from '~/providers/project-review/review-panels' import { injectReviewPreviousIssues } from '~/providers/project-review/review-previous-issues' +import FlagsCountBadge from '../review-panel/flags-count-badge.vue' import FindingsCountBadge from '../review-panel/findings-count-badge.vue' import ReviewPanel from '../review-panel/index.vue' import ReReviewFindingsCountBadge from '../review-panel/re-review-findings-count-badge.vue' @@ -41,6 +44,11 @@ const props = defineProps<{ }>() const panels = injectReviewPanels() const previousIssues = injectReviewPreviousIssues() +const reviewFlags = injectReviewFlags(null) +const flagCount = computed(() => { + const binding = props.target ? panels.resolve(props.target) : undefined + return binding ? (reviewFlags?.forPanel(binding.key).length ?? 0) : 0 +}) const reReviewFindingCount = computed(() => { const binding = props.target ? panels.resolve(props.target) : undefined return binding ? previousIssues.reReviewFindingCount(binding) : 0 diff --git a/apps/frontend/src/components/project-review/review-panel/anchor.vue b/apps/frontend/src/components/project-review/review-panel/anchor.vue index 5634ba05012..466ff1628da 100644 --- a/apps/frontend/src/components/project-review/review-panel/anchor.vue +++ b/apps/frontend/src/components/project-review/review-panel/anchor.vue @@ -19,6 +19,7 @@ >
+
@@ -36,6 +37,7 @@ import { injectReviewPanels } from '~/providers/project-review/review-panels' import { injectReviewPreviousIssues } from '~/providers/project-review/review-previous-issues' import { injectReviewContext } from './context' +import FlagsCountBadge from './flags-count-badge.vue' import FindingsCountBadge from './findings-count-badge.vue' import Highlight from './highlight.vue' import ReReviewFindingsCountBadge from './re-review-findings-count-badge.vue' diff --git a/apps/frontend/src/components/project-review/review-panel/flags-count-badge.vue b/apps/frontend/src/components/project-review/review-panel/flags-count-badge.vue new file mode 100644 index 00000000000..76de6571820 --- /dev/null +++ b/apps/frontend/src/components/project-review/review-panel/flags-count-badge.vue @@ -0,0 +1,21 @@ + + + diff --git a/apps/frontend/src/components/project-review/review-panel/flags.vue b/apps/frontend/src/components/project-review/review-panel/flags.vue new file mode 100644 index 00000000000..74b0a9876ac --- /dev/null +++ b/apps/frontend/src/components/project-review/review-panel/flags.vue @@ -0,0 +1,56 @@ + + + diff --git a/apps/frontend/src/components/project-review/review-panel/index.vue b/apps/frontend/src/components/project-review/review-panel/index.vue index 455615302a4..b759f5eab0b 100644 --- a/apps/frontend/src/components/project-review/review-panel/index.vue +++ b/apps/frontend/src/components/project-review/review-panel/index.vue @@ -43,6 +43,7 @@ +