diff --git a/.gitattributes b/.gitattributes index 05b1a132398..9b5e5779f02 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,6 @@ **/ModuleBindings/** linguist-generated=true eol=lf /docs/llms/** linguist-generated=true /docs/llms/*-details.json linguist-generated=false +/tools/stack-bench/** text eol=lf +/tools/stack-bench/**/*.woff2 -text -diff +/tools/stack-bench/qualification-evidence/**/*.json -text whitespace=cr-at-eol diff --git a/.gitignore b/.gitignore index 1f3b49ecd2d..2c2f6b629ef 100644 --- a/.gitignore +++ b/.gitignore @@ -267,3 +267,6 @@ nul # Any local file *.local + +# Local working notes and temporary builds +/local-notes/ diff --git a/codex-plugin/plugins/spacetimedb/skills/cli/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/cli/SKILL.md index e7a51487fc1..2b7b0c63728 100644 --- a/codex-plugin/plugins/spacetimedb/skills/cli/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/cli/SKILL.md @@ -43,13 +43,15 @@ spacetime build --debug # faster iteration, slower runtime # Dev mode (auto-rebuild, auto-publish, generates bindings) spacetime dev -spacetime dev --client-lang typescript --module-bindings-path ./client/src/module_bindings +spacetime dev my-database --server local --yes --delete-data=never --client-lang typescript --module-bindings-path ./client/src/module_bindings # Generate client bindings spacetime generate --lang typescript|csharp|rust --out-dir ./bindings --module-path ./server spacetime generate --lang unrealcpp --uproject-dir ./MyGame --module-path ./server --unreal-module-name MyGame ``` +`dev` stays running and watches module changes. `--run "npm run dev"` starts a client command; `--server-only` omits it. `--module-path` selects the module directory when no publish targets exist in `spacetime.json`. Once targets exist, use their configured paths and omit that flag. Separate build/publish/generate commands remain useful for one-shot deployment. + ### Publishing & Deployment ```bash @@ -114,7 +116,7 @@ spacetime server add myserver --url https://my-spacetime.example.com # Set default server spacetime server set-default local -# Test connectivity +# Check connectivity spacetime server ping local # Start local instance @@ -172,10 +174,7 @@ spacetime server ping ``` ### "Schema conflict" -```bash -# Clear data and republish -spacetime publish my-db --delete-data=always --yes -``` +`--delete-data=never` rejects incompatible schema updates without clearing data. A compatible migration preserves existing data; `--delete-data=always` destroys it and is only appropriate for an intentional reset. ### "Build failed" ```bash diff --git a/codex-plugin/plugins/spacetimedb/skills/concepts/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/concepts/SKILL.md index 8ea6e8f9cfb..e152e5d9e72 100644 --- a/codex-plugin/plugins/spacetimedb/skills/concepts/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/concepts/SKILL.md @@ -101,8 +101,8 @@ Lifecycle: Write → Compile → Publish (`spacetime publish`) → Hot-swap (rep ## Identity -- **Identity**: A long-lived, globally unique identifier for a user. -- **ConnectionId**: Identifies a specific client connection. +- **Identity**: A long-lived, globally unique identifier for a user, derived from the token's issuer and subject claims. The same token yields the same identity on every connection. +- **ConnectionId**: Identifies one client connection. A new connection gets a new connection ID; a disconnect ends the connection, not the identity. - Always use `ctx.sender` / `ctx.Sender` / `ctx.sender()` for authorization. SpacetimeDB works with many OIDC providers, including SpacetimeAuth (built-in), Auth0, Clerk, Keycloak, Google, and GitHub. diff --git a/codex-plugin/plugins/spacetimedb/skills/cpp-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/cpp-server/SKILL.md index aebb82edf06..3facad75b0e 100644 --- a/codex-plugin/plugins/spacetimedb/skills/cpp-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/cpp-server/SKILL.md @@ -138,6 +138,8 @@ SPACETIMEDB_CLIENT_DISCONNECTED(on_disconnect, ReducerContext ctx) { } ``` +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.sender()`) across connections, while each connection has its own connection ID (`ctx.connection_id()`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + ## Authentication & Timestamps ```cpp diff --git a/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md index 1d08ba89fe7..b1037f599c3 100644 --- a/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md @@ -162,7 +162,9 @@ public static void OnConnect(ReducerContext ctx) { ... } public static void OnDisconnect(ReducerContext ctx) { ... } ``` -`ctx.ConnectionId` is `ConnectionId?`, including in connection lifecycle reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.Sender`) across connections, while each connection has its own connection ID (`ctx.ConnectionId`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + +`ctx.ConnectionId` is typed `ConnectionId?`. It is present inside connection lifecycle reducers and reducers invoked over a connection, and null in `Init` and scheduled reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. ## Views diff --git a/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md index 0e282242263..ec8e772c9c9 100644 --- a/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md @@ -150,6 +150,8 @@ pub fn on_connect(ctx: &ReducerContext) { ... } pub fn on_disconnect(ctx: &ReducerContext) { ... } ``` +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.sender()`) across connections, while each connection has its own connection ID (`ctx.connection_id()`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + The current connection ID is available through `ctx.connection_id()` (not a public field) and may be absent outside connection-scoped calls. ## Views diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md index 3a31183133f..dff16e1dd3c 100644 --- a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md @@ -18,7 +18,7 @@ Generated bindings convert snake_case names to camelCase, including row fields: ## React: main.tsx ```typescript -import React, { useEffect, useMemo } from 'react'; +import React, { useMemo } from 'react'; import ReactDOM from 'react-dom/client'; import { SpacetimeDBProvider } from 'spacetimedb/react'; import { DbConnection } from './module_bindings'; @@ -30,6 +30,7 @@ function Root() { DbConnection.builder() .withUri(SPACETIMEDB_URI) .withDatabaseName(MODULE_NAME) + // Reuse the token issued on the previous connection. .withToken(localStorage.getItem('auth_token') || undefined), [] ); @@ -46,27 +47,18 @@ ReactDOM.createRoot(document.getElementById('root')!).render(); ## React: App.tsx ```typescript +import { useEffect } from 'react'; import { useTable, useSpacetimeDB } from 'spacetimedb/react'; import { DbConnection, tables } from './module_bindings'; function App() { - const { isActive, identity: myIdentity, token, getConnection } = useSpacetimeDB(); + const { identity: myIdentity, token, getConnection } = useSpacetimeDB(); const conn = getConnection() as DbConnection | null; - // Save auth token + // Persist the issued token for the next page load. useEffect(() => { if (token) localStorage.setItem('auth_token', token); }, [token]); - // Subscribe when connected. Prefer typed query builders over raw SQL - useEffect(() => { - if (!conn || !isActive) return; - conn.subscriptionBuilder() - .onApplied(() => setSubscribed(true)) - .subscribe([tables.entity, tables.record]); - // Or with filters: tables.entity.where(r => r.active.eq(true)) - // Or raw SQL: 'SELECT * FROM entity' - }, [conn, isActive]); - - // Reactive data. Returns [rows, isReady] + // useTable owns the subscription and cleanup. Returns [rows, isReady]. const [entities, entitiesReady] = useTable(tables.entity); const [records, recordsReady] = useTable(tables.record); @@ -80,8 +72,8 @@ function App() { } ); - // Call reducers with object syntax - conn?.reducers.addRecord({ data }).catch(console.error); + // A callback for a UI event; defining it does not call the reducer during render. + const addRecord = (data: string) => conn?.reducers.addRecord({ data }).catch(console.error); // Compare identities const isMe = row.owner.toHexString() === myIdentity?.toHexString(); @@ -111,7 +103,9 @@ conn.db.user.onUpdate((ctx, oldUser, newUser) => console.log('Updated:', newUser ## Gotchas -- **`useTable` rows are `readonly`.** Copy before sorting/mutating, or it fails to type-check: +- **Subscription rows have no presentation order.** A server view's array order does not + define client cache iteration order. `useTable` rows are `readonly`; a sorted copy can + express the application's display order: `const [rows] = useTable(tables.message); const sorted = [...rows].sort(...)`. -- **bigint in JSX.** ids/counts from `t.u64()`/`t.i64()` columns are `bigint`, which React - cannot render. Wrap it: `{Number(row.id)}` or `{String(count)}`. +- **64-bit display values.** `{String(row.id)}` preserves the full `bigint` value. + Conversion to `Number` can lose precision outside JavaScript's safe integer range. diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md index 8563a5e9808..c4c01045ca9 100644 --- a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md @@ -98,6 +98,11 @@ Every column is a `t` builder value: Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. +`.primaryKey()` and `.unique()` apply to one column. For uniqueness across +multiple columns, use a surrogate key, the multi-column index below, and a +reducer that rejects an existing index match before inserting. An index alone +does not enforce uniqueness. + Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` @@ -130,7 +135,9 @@ export { default } from './schema'; // re-export the schema for the module ent ## Reducers -Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name: +Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp` +becomes `signUp` in generated clients and `sign_up` in `spacetime call` and +`describe`: ```typescript export const createEntity = spacetimedb.reducer( @@ -178,10 +185,26 @@ export const onConnect = spacetimedb.clientConnected((ctx) => { ... }); export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ... }); ``` -`ctx.connectionId` is `ConnectionId | null`, including in lifecycle contexts. Guard it before passing it to a helper or using it as a table key. +Connection hooks run once per connection. `ctx.connectionId` identifies that connection. +`ctx.sender` identifies the authenticated caller and stays the same when the client +reconnects with the same token. + +Use connection IDs for connection state, such as presence. A reload or network loss can +cause a disconnect. A disconnect alone does not mean the user signed out or their +application login expired. Keep connection cleanup separate from session revocation. + +`ctx.sender` is a SpacetimeDB identity, not necessarily an application account. Separate +identities can authenticate to the same application account. + +`ctx.connectionId` is typed `ConnectionId | null`. It is present inside connection lifecycle hooks and reducers invoked over a connection, and `null` in `init` and scheduled reducers. Guard it before passing it to a helper or using it as a table key. ## Reducer Context API +Each reducer call runs in one database transaction. An error that escapes the +reducer rolls back its database changes. `ctx.sender` identifies the caller; +application roles and permissions are not inferred from that identity. Table +visibility and view filters control reads, not authorization to call reducers. + `ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. Let exported callbacks infer their context type. In helpers, use `ReducerCtx>`; do not annotate a context as `any`, because that erases table row types and can make `bigint` expressions infer as `number`. ```typescript @@ -282,9 +305,22 @@ const Shape = t.enum('Shape', { A client subscribing to a view receives only the rows it returns. Use a per-user view (keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on (e.g. a membership row) automatically drops the rows it was exposing from that client. +Use index accessors in views. Do not scan a whole table with `.iter()` when an +indexed lookup can select the required rows. `t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). +A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from +`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is +not a `ReducerCtx`, so a helper shared between a reducer and a view must accept +either: + +```typescript +import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server'; +type S = InferSchema; +function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... } +``` + Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. ```typescript @@ -292,7 +328,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg export const activeUsers = spacetimedb.anonymousView( { name: 'active_users', public: true }, t.array(entity.rowType), - (ctx) => [...ctx.db.entity.iter()].filter(e => e.active) + (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree') ); // Per-user view (varies by ctx.sender): diff --git a/crates/bindings-typescript/src/lib/query.ts b/crates/bindings-typescript/src/lib/query.ts index bb93b0e6ce3..0c17e3d145a 100644 --- a/crates/bindings-typescript/src/lib/query.ts +++ b/crates/bindings-typescript/src/lib/query.ts @@ -248,19 +248,21 @@ export type NamespacedQueryBuilder = * A runtime reference to a table. This materializes the RowExpr for us. * TODO: Maybe add the full SchemaDef to the type signature depending on how joins will work. */ -export type TableRef = Readonly<{ - type: 'table'; - sourceName: TableDef['sourceName']; - accessorName: string; - cols: RowExpr; - indexedCols: IndexedRowExpr; - tableDef: TableDef; +// Keep this named so TypeScript diagnostics show `TableRef` instead of its +// expanded structure. +export interface TableRef { + readonly type: 'table'; + readonly sourceName: TableDef['sourceName']; + readonly accessorName: string; + readonly cols: RowExpr; + readonly indexedCols: IndexedRowExpr; + readonly tableDef: TableDef; // Delegated UntypedTableDef properties for compatibility. - columns: TableDef['columns']; - indexes: TableDef['indexes']; - rowType: TableDef['rowType']; - constraints: any; -}>; + readonly columns: TableDef['columns']; + readonly indexes: TableDef['indexes']; + readonly rowType: TableDef['rowType']; + readonly constraints: any; +} class TableRefImpl implements TableRef, From diff --git a/crates/bindings-typescript/tests/table_ref_error_message.test.ts b/crates/bindings-typescript/tests/table_ref_error_message.test.ts new file mode 100644 index 00000000000..54274fd5455 --- /dev/null +++ b/crates/bindings-typescript/tests/table_ref_error_message.test.ts @@ -0,0 +1,78 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as ts from 'typescript'; +import { describe, expect, it } from 'vitest'; + +const bindingsRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..' +); + +function runTypecheck(source: string) { + const tmpDir = mkdtempSync(path.join(tmpdir(), 'stdb-tableref-diag-')); + const reproPath = path.join(tmpDir, 'repro.ts'); + writeFileSync(reproPath, source); + + try { + const options: ts.CompilerOptions = { + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + strict: true, + noEmit: true, + skipLibCheck: true, + forceConsistentCasingInFileNames: true, + allowImportingTsExtensions: true, + noImplicitAny: true, + moduleResolution: ts.ModuleResolutionKind.Bundler, + useDefineForClassFields: true, + verbatimModuleSyntax: true, + isolatedModules: true, + }; + + const host = ts.createCompilerHost(options); + const program = ts.createProgram( + [reproPath, path.join(bindingsRoot, 'src/server/sys.d.ts')], + options, + host + ); + const diagnostics = ts.getPreEmitDiagnostics(program); + return diagnostics.map(d => + ts.flattenDiagnosticMessageText(d.messageText, '\n') + ); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +} + +describe('TableRef diagnostics', () => { + const source = ` +import { t } from ${JSON.stringify(path.join(bindingsRoot, 'src/server/index.ts'))}; +import { table } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/table.ts'))}; +import { createTableRefFromDef } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/query.ts'))}; +import type { AllUnique } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/constraints.ts'))}; + +const cartItem = table( + { name: 'cart_item' }, + { id: t.u64().primaryKey().autoInc(), accountId: t.u64(), quantity: t.u32() } +); + +const ref = createTableRefFromDef(cartItem as any, 'cartItem'); +type Boom = AllUnique; +declare const b: Boom; +`; + + it('names the type instead of dumping its structure', () => { + const messages = runTypecheck(source); + const constraintError = messages.find(m => + m.includes("does not satisfy the constraint 'UntypedTableDef'") + ); + + expect(constraintError).toBeDefined(); + // The name, not the shape. + expect(constraintError).toContain('TableRef<'); + expect(constraintError).not.toContain('type: "table"'); + expect(constraintError).not.toContain('accessorName'); + }, 15000); +}); diff --git a/crates/cli/build.rs b/crates/cli/build.rs index c5bd4303464..7e7d755cae5 100644 --- a/crates/cli/build.rs +++ b/crates/cli/build.rs @@ -6,6 +6,7 @@ use std::process::Command; use toml::Value; fn main() { + println!("cargo:rerun-if-env-changed=SPACETIMEDB_NIX_BUILD_GIT_COMMIT"); let git_hash = find_git_hash(); println!("cargo:rustc-env=GIT_HASH={git_hash}"); @@ -110,6 +111,7 @@ fn generate_template_files() { // Embed skill files from skills/*/SKILL.md let skills_dir = repo_root.join("skills"); + println!("cargo:rerun-if-changed={}", skills_dir.display()); let skill_names = discover_skill_names(&skills_dir); generated_code.push_str("pub fn get_skill(name: &str) -> Option<&'static str> {\n"); diff --git a/crates/cli/src/subcommands/dev.rs b/crates/cli/src/subcommands/dev.rs index 95cc579182f..ef2465cff0a 100644 --- a/crates/cli/src/subcommands/dev.rs +++ b/crates/cli/src/subcommands/dev.rs @@ -85,6 +85,12 @@ pub fn cli() -> Command { ) .arg(common_args::server().help("The nickname, host name or URL of the server to publish to")) .arg(common_args::yes()) + .arg( + Arg::new("ready-file") + .long("ready-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Write this file after the initial build cycle succeeds and file watching starts. This is a startup receipt, not ongoing health."), + ) .arg(common_args::clear_database()) .arg( Arg::new("template") @@ -150,6 +156,13 @@ struct DatabaseRow { } pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { + if let Some(path) = args.get_one::("ready-file") { + match fs::remove_file(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("Failed to clear development readiness file"), + } + } let project_path = args.get_one::("project-path").unwrap(); let module_path_from_cli = args.get_one::("module-path"); let module_bindings_path = args.get_one::("module-bindings-path").unwrap(); @@ -760,7 +773,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E let loaded_config_dir = loaded_config.as_ref().map(|lc| lc.config_dir.clone()); generate_build_and_publish( - &config, + &mut config, &project_dir, loaded_config_dir.as_deref(), &spacetimedb_dir, @@ -861,6 +874,10 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E watcher.watch(watch_dir, RecursiveMode::Recursive)?; } + if let Some(path) = args.get_one::("ready-file") { + fs::write(path, "ready\n").context("Failed to write development readiness file")?; + } + let mut debounce_timer; loop { // Use recv_timeout so we can periodically check if the client process exited @@ -876,7 +893,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E println!("\n{}", "File change detected, rebuilding...".yellow()); match generate_build_and_publish( - &config, + &mut config, &project_dir, loaded_config_dir.as_deref(), &spacetimedb_dir, @@ -1024,7 +1041,7 @@ fn upsert_env_db_names_and_hosts(env_path: &Path, server_host_url: &str, databas #[allow(clippy::too_many_arguments)] async fn generate_build_and_publish( - config: &Config, + config: &mut Config, project_dir: &Path, config_dir: Option<&Path>, spacetimedb_dir: &Path, @@ -1181,7 +1198,8 @@ async fn generate_build_and_publish( publish_entry.insert("break-clients".to_string(), json!(true)); } - publish::exec_from_entry(config.clone(), publish_entry, config_dir, clear_database, yes).await?; + // Preserve a token created during publish for logs and later rebuilds. + publish::exec_from_entry(config, publish_entry, config_dir, clear_database, yes).await?; } println!("{}", "Published successfully!".green().bold()); @@ -2043,12 +2061,17 @@ mod tests { // Verify that --skip-publish and --skip-generate flags are registered let cmd = cli(); - let matches = cmd - .clone() - .get_matches_from(vec!["dev", "--skip-publish", "--skip-generate"]); + let matches = cmd.clone().get_matches_from(vec![ + "dev", + "--skip-publish", + "--skip-generate", + "--ready-file", + "ready", + ]); assert!(matches.get_flag("skip_publish")); assert!(matches.get_flag("skip_generate")); + assert_eq!(matches.get_one::("ready-file"), Some(&PathBuf::from("ready"))); } #[test] diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index df51f6f62de..c279c7eec61 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -547,7 +547,7 @@ pub async fn exec_with_options( } pub async fn exec_from_entry( - mut config: Config, + config: &mut Config, entry: HashMap, config_dir: Option<&std::path::Path>, clear_database: ClearMode, @@ -563,7 +563,7 @@ pub async fn exec_from_entry( let yes = if force { YesFlags::all() } else { YesFlags::default() }; execute_publish_configs( - &mut config, + config, vec![command_config], true, config_dir, diff --git a/crates/codegen/build.rs b/crates/codegen/build.rs index cd3bca12faf..cff0d3ed9bf 100644 --- a/crates/codegen/build.rs +++ b/crates/codegen/build.rs @@ -3,6 +3,9 @@ use std::process::Command; // https://stackoverflow.com/questions/43753491/include-git-commit-hash-as-string-into-rust-program #[allow(clippy::disallowed_macros)] fn main() { + // Any rerun-if directive disables Cargo's default package scan; restore it. + println!("cargo:rerun-if-changed=."); + println!("cargo:rerun-if-env-changed=SPACETIMEDB_NIX_BUILD_GIT_COMMIT"); let git_hash = find_git_hash(); println!("cargo:rustc-env=GIT_HASH={git_hash}"); } diff --git a/crates/fs-utils/src/lib.rs b/crates/fs-utils/src/lib.rs index c4d1a6ba0c1..42e5a51ae25 100644 --- a/crates/fs-utils/src/lib.rs +++ b/crates/fs-utils/src/lib.rs @@ -35,12 +35,36 @@ pub fn atomic_write(file_path: &Path, data: String) -> anyhow::Result<()> { .write(true) .create_new(true) .open(&temp_path); - if let Ok(file) = opened { - temp_file = file; - break; + match opened { + Ok(file) => { + temp_file = file; + break; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), } } temp_file.write_all(data.as_bytes())?; std::fs::rename(&temp_path, file_path)?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::atomic_write; + + #[test] + fn atomic_write_replaces_contents_and_returns_open_errors() { + let dir = tempdir::TempDir::new("atomic-write").unwrap(); + let path = dir.path().join("config"); + atomic_write(&path, "before".into()).unwrap(); + atomic_write(&path, "after".into()).unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "after"); + + let error = atomic_write(&dir.path().join("missing/config"), "data".into()).unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::NotFound + ); + } +} diff --git a/skills/cli/SKILL.md b/skills/cli/SKILL.md index e7a51487fc1..2b7b0c63728 100644 --- a/skills/cli/SKILL.md +++ b/skills/cli/SKILL.md @@ -43,13 +43,15 @@ spacetime build --debug # faster iteration, slower runtime # Dev mode (auto-rebuild, auto-publish, generates bindings) spacetime dev -spacetime dev --client-lang typescript --module-bindings-path ./client/src/module_bindings +spacetime dev my-database --server local --yes --delete-data=never --client-lang typescript --module-bindings-path ./client/src/module_bindings # Generate client bindings spacetime generate --lang typescript|csharp|rust --out-dir ./bindings --module-path ./server spacetime generate --lang unrealcpp --uproject-dir ./MyGame --module-path ./server --unreal-module-name MyGame ``` +`dev` stays running and watches module changes. `--run "npm run dev"` starts a client command; `--server-only` omits it. `--module-path` selects the module directory when no publish targets exist in `spacetime.json`. Once targets exist, use their configured paths and omit that flag. Separate build/publish/generate commands remain useful for one-shot deployment. + ### Publishing & Deployment ```bash @@ -114,7 +116,7 @@ spacetime server add myserver --url https://my-spacetime.example.com # Set default server spacetime server set-default local -# Test connectivity +# Check connectivity spacetime server ping local # Start local instance @@ -172,10 +174,7 @@ spacetime server ping ``` ### "Schema conflict" -```bash -# Clear data and republish -spacetime publish my-db --delete-data=always --yes -``` +`--delete-data=never` rejects incompatible schema updates without clearing data. A compatible migration preserves existing data; `--delete-data=always` destroys it and is only appropriate for an intentional reset. ### "Build failed" ```bash diff --git a/skills/concepts/SKILL.md b/skills/concepts/SKILL.md index 8ea6e8f9cfb..e152e5d9e72 100644 --- a/skills/concepts/SKILL.md +++ b/skills/concepts/SKILL.md @@ -101,8 +101,8 @@ Lifecycle: Write → Compile → Publish (`spacetime publish`) → Hot-swap (rep ## Identity -- **Identity**: A long-lived, globally unique identifier for a user. -- **ConnectionId**: Identifies a specific client connection. +- **Identity**: A long-lived, globally unique identifier for a user, derived from the token's issuer and subject claims. The same token yields the same identity on every connection. +- **ConnectionId**: Identifies one client connection. A new connection gets a new connection ID; a disconnect ends the connection, not the identity. - Always use `ctx.sender` / `ctx.Sender` / `ctx.sender()` for authorization. SpacetimeDB works with many OIDC providers, including SpacetimeAuth (built-in), Auth0, Clerk, Keycloak, Google, and GitHub. diff --git a/skills/cpp-server/SKILL.md b/skills/cpp-server/SKILL.md index aebb82edf06..3facad75b0e 100644 --- a/skills/cpp-server/SKILL.md +++ b/skills/cpp-server/SKILL.md @@ -138,6 +138,8 @@ SPACETIMEDB_CLIENT_DISCONNECTED(on_disconnect, ReducerContext ctx) { } ``` +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.sender()`) across connections, while each connection has its own connection ID (`ctx.connection_id()`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + ## Authentication & Timestamps ```cpp diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index 1d08ba89fe7..b1037f599c3 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -162,7 +162,9 @@ public static void OnConnect(ReducerContext ctx) { ... } public static void OnDisconnect(ReducerContext ctx) { ... } ``` -`ctx.ConnectionId` is `ConnectionId?`, including in connection lifecycle reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.Sender`) across connections, while each connection has its own connection ID (`ctx.ConnectionId`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + +`ctx.ConnectionId` is typed `ConnectionId?`. It is present inside connection lifecycle reducers and reducers invoked over a connection, and null in `Init` and scheduled reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. ## Views diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index 0e282242263..ec8e772c9c9 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -150,6 +150,8 @@ pub fn on_connect(ctx: &ReducerContext) { ... } pub fn on_disconnect(ctx: &ReducerContext) { ... } ``` +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.sender()`) across connections, while each connection has its own connection ID (`ctx.connection_id()`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + The current connection ID is available through `ctx.connection_id()` (not a public field) and may be absent outside connection-scoped calls. ## Views diff --git a/skills/typescript-client/SKILL.md b/skills/typescript-client/SKILL.md index 3a31183133f..dff16e1dd3c 100644 --- a/skills/typescript-client/SKILL.md +++ b/skills/typescript-client/SKILL.md @@ -18,7 +18,7 @@ Generated bindings convert snake_case names to camelCase, including row fields: ## React: main.tsx ```typescript -import React, { useEffect, useMemo } from 'react'; +import React, { useMemo } from 'react'; import ReactDOM from 'react-dom/client'; import { SpacetimeDBProvider } from 'spacetimedb/react'; import { DbConnection } from './module_bindings'; @@ -30,6 +30,7 @@ function Root() { DbConnection.builder() .withUri(SPACETIMEDB_URI) .withDatabaseName(MODULE_NAME) + // Reuse the token issued on the previous connection. .withToken(localStorage.getItem('auth_token') || undefined), [] ); @@ -46,27 +47,18 @@ ReactDOM.createRoot(document.getElementById('root')!).render(); ## React: App.tsx ```typescript +import { useEffect } from 'react'; import { useTable, useSpacetimeDB } from 'spacetimedb/react'; import { DbConnection, tables } from './module_bindings'; function App() { - const { isActive, identity: myIdentity, token, getConnection } = useSpacetimeDB(); + const { identity: myIdentity, token, getConnection } = useSpacetimeDB(); const conn = getConnection() as DbConnection | null; - // Save auth token + // Persist the issued token for the next page load. useEffect(() => { if (token) localStorage.setItem('auth_token', token); }, [token]); - // Subscribe when connected. Prefer typed query builders over raw SQL - useEffect(() => { - if (!conn || !isActive) return; - conn.subscriptionBuilder() - .onApplied(() => setSubscribed(true)) - .subscribe([tables.entity, tables.record]); - // Or with filters: tables.entity.where(r => r.active.eq(true)) - // Or raw SQL: 'SELECT * FROM entity' - }, [conn, isActive]); - - // Reactive data. Returns [rows, isReady] + // useTable owns the subscription and cleanup. Returns [rows, isReady]. const [entities, entitiesReady] = useTable(tables.entity); const [records, recordsReady] = useTable(tables.record); @@ -80,8 +72,8 @@ function App() { } ); - // Call reducers with object syntax - conn?.reducers.addRecord({ data }).catch(console.error); + // A callback for a UI event; defining it does not call the reducer during render. + const addRecord = (data: string) => conn?.reducers.addRecord({ data }).catch(console.error); // Compare identities const isMe = row.owner.toHexString() === myIdentity?.toHexString(); @@ -111,7 +103,9 @@ conn.db.user.onUpdate((ctx, oldUser, newUser) => console.log('Updated:', newUser ## Gotchas -- **`useTable` rows are `readonly`.** Copy before sorting/mutating, or it fails to type-check: +- **Subscription rows have no presentation order.** A server view's array order does not + define client cache iteration order. `useTable` rows are `readonly`; a sorted copy can + express the application's display order: `const [rows] = useTable(tables.message); const sorted = [...rows].sort(...)`. -- **bigint in JSX.** ids/counts from `t.u64()`/`t.i64()` columns are `bigint`, which React - cannot render. Wrap it: `{Number(row.id)}` or `{String(count)}`. +- **64-bit display values.** `{String(row.id)}` preserves the full `bigint` value. + Conversion to `Number` can lose precision outside JavaScript's safe integer range. diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 8563a5e9808..c4c01045ca9 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -98,6 +98,11 @@ Every column is a `t` builder value: Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. +`.primaryKey()` and `.unique()` apply to one column. For uniqueness across +multiple columns, use a surrogate key, the multi-column index below, and a +reducer that rejects an existing index match before inserting. An index alone +does not enforce uniqueness. + Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` @@ -130,7 +135,9 @@ export { default } from './schema'; // re-export the schema for the module ent ## Reducers -Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name: +Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp` +becomes `signUp` in generated clients and `sign_up` in `spacetime call` and +`describe`: ```typescript export const createEntity = spacetimedb.reducer( @@ -178,10 +185,26 @@ export const onConnect = spacetimedb.clientConnected((ctx) => { ... }); export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ... }); ``` -`ctx.connectionId` is `ConnectionId | null`, including in lifecycle contexts. Guard it before passing it to a helper or using it as a table key. +Connection hooks run once per connection. `ctx.connectionId` identifies that connection. +`ctx.sender` identifies the authenticated caller and stays the same when the client +reconnects with the same token. + +Use connection IDs for connection state, such as presence. A reload or network loss can +cause a disconnect. A disconnect alone does not mean the user signed out or their +application login expired. Keep connection cleanup separate from session revocation. + +`ctx.sender` is a SpacetimeDB identity, not necessarily an application account. Separate +identities can authenticate to the same application account. + +`ctx.connectionId` is typed `ConnectionId | null`. It is present inside connection lifecycle hooks and reducers invoked over a connection, and `null` in `init` and scheduled reducers. Guard it before passing it to a helper or using it as a table key. ## Reducer Context API +Each reducer call runs in one database transaction. An error that escapes the +reducer rolls back its database changes. `ctx.sender` identifies the caller; +application roles and permissions are not inferred from that identity. Table +visibility and view filters control reads, not authorization to call reducers. + `ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. Let exported callbacks infer their context type. In helpers, use `ReducerCtx>`; do not annotate a context as `any`, because that erases table row types and can make `bigint` expressions infer as `number`. ```typescript @@ -282,9 +305,22 @@ const Shape = t.enum('Shape', { A client subscribing to a view receives only the rows it returns. Use a per-user view (keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on (e.g. a membership row) automatically drops the rows it was exposing from that client. +Use index accessors in views. Do not scan a whole table with `.iter()` when an +indexed lookup can select the required rows. `t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). +A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from +`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is +not a `ReducerCtx`, so a helper shared between a reducer and a view must accept +either: + +```typescript +import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server'; +type S = InferSchema; +function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... } +``` + Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. ```typescript @@ -292,7 +328,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg export const activeUsers = spacetimedb.anonymousView( { name: 'active_users', public: true }, t.array(entity.rowType), - (ctx) => [...ctx.db.entity.iter()].filter(e => e.active) + (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree') ); // Per-user view (varies by ctx.sender): diff --git a/tools/llm-sequential-upgrade/.gitignore b/tools/llm-sequential-upgrade/.gitignore index 14aa619a63d..35223d12b8f 100644 --- a/tools/llm-sequential-upgrade/.gitignore +++ b/tools/llm-sequential-upgrade/.gitignore @@ -27,4 +27,4 @@ telemetry/metrics.jsonl **/telemetry/**/metadata.json # Sequential-upgrade run output lives in the external spacetimedb-ai-test-results repo -sequential-upgrade/sequential-upgrade-*/ +sequential-upgrade/ diff --git a/tools/llm-sequential-upgrade/CLAUDE.md b/tools/llm-sequential-upgrade/CLAUDE.md index b83f19f48a9..97069bc1645 100644 --- a/tools/llm-sequential-upgrade/CLAUDE.md +++ b/tools/llm-sequential-upgrade/CLAUDE.md @@ -2,15 +2,7 @@ Your job is to **generate, build, deploy, and fix** a fully working chat app. Verification happens in a separate session — you do NOT test in the browser. ---- - -## Path Convention - -All file paths are **relative to the `llm-sequential-upgrade/` directory** unless stated otherwise. `../` means going up to `tools/`. - -Examples: -- `backends/spacetime.md` → `llm-sequential-upgrade/backends/spacetime.md` -- `../llm-oneshot/apps/chat-app/prompts/composed/01_basic.md` → `tools/llm-oneshot/apps/chat-app/prompts/composed/01_basic.md` +You work only inside the app directory, which is your working directory. Everything you need is either in the launch prompt (the language setup and feature spec) or in the app directory's own `CLAUDE.md` (backend setup and deploy steps, plus phases and SDK reference at the richer rules levels). Files outside the app directory are not available and are not needed. --- @@ -24,7 +16,7 @@ Depending on the mode passed in the launch prompt: | **upgrade** | Add new features from the next level prompt to existing code | | **fix** | Read BUG_REPORT.md, fix the listed bugs, redeploy | -**CRITICAL:** Read `backends/.md` first — it has all setup, build, and deploy instructions. +**CRITICAL:** Read the app directory's `CLAUDE.md` first — it has all setup, build, and deploy instructions. --- @@ -38,22 +30,15 @@ POSIX: `mkdir -p` not `New-Item`, `sleep` not `Start-Sleep`, `2>/dev/null` not ` ## Anti-Contamination -Do NOT read any files under: -- `../llm-oneshot/apps/chat-app/typescript/` (reference implementations) -- `../llm-oneshot/apps/chat-app/staging/` -- Any other AI-generated app code in this workspace - -Only read files you created, the backend instructions, and the feature prompts. +Only read files you created, the app directory's `CLAUDE.md`, and `BUG_REPORT.md` when fixing. Do not look for reference implementations, other generated apps, or grading material anywhere on the machine. --- ## Generate / Upgrade -1. Read `backends/.md` for pre-flight checks, phases, and deploy steps -2. Read the language setup: `../llm-oneshot/apps/chat-app/prompts/language/typescript-.md` -3. Read the feature prompt: `../llm-oneshot/apps/chat-app/prompts/composed/_.md` -4. Follow the phases in the backend file, in order -5. Output `DEPLOY_COMPLETE` when the dev server is confirmed running +1. Follow the app directory's `CLAUDE.md`, including its phases in order when it has them +2. Build from the language setup and feature spec included in the launch prompt +3. Output `DEPLOY_COMPLETE` (generate) or `UPGRADE_COMPLETE` (upgrade) when the dev server is confirmed running For **upgrade**: only add the NEW features from the target level. Do not rewrite existing working features. diff --git a/tools/llm-sequential-upgrade/grade.sh b/tools/llm-sequential-upgrade/grade.sh index 3b8e6b129f1..3f295d4549a 100644 --- a/tools/llm-sequential-upgrade/grade.sh +++ b/tools/llm-sequential-upgrade/grade.sh @@ -89,7 +89,7 @@ $CLAUDE_CMD -p "Grade the sequential upgrade app at: $APP_DIR_NATIVE Backend: $GRADE_BACKEND -Follow CLAUDE.md Phases 6-8: +Follow GRADING.md (setup, rubric, and result format): 1. Open http://localhost:$VITE_PORT in Chrome and verify the app loads 2. Test each feature using the test plans in test-plans/feature-*.md 3. Score each feature 0-3 based on browser observations diff --git a/tools/llm-sequential-upgrade/run.sh b/tools/llm-sequential-upgrade/run.sh index 34e58c3d9b4..5f71e62b50a 100644 --- a/tools/llm-sequential-upgrade/run.sh +++ b/tools/llm-sequential-upgrade/run.sh @@ -814,7 +814,8 @@ fi # ─── Run Claude Code ───────────────────────────────────────────────────────── # Run from the APP directory so CLAUDE.md auto-discovery picks up the -# backend-specific file, not the parent llm-sequential-upgrade/CLAUDE.md. +# backend-specific file. Ancestor CLAUDE.md files, including +# llm-sequential-upgrade/CLAUDE.md, are loaded too. cd "$APP_DIR" @@ -860,11 +861,16 @@ if [[ -n "$MODEL" ]]; then fi # Build args as an array so empty optional flags (model/resume) can't break the invocation. +# Read and Edit are limited to the app directory; the language and feature content is +# inlined into the prompt above, so nothing outside it is needed. Bash remains +# available, so this is not filesystem isolation. +# --setting-sources keeps the operator's user settings, plugins, and hooks out of the agent. CLAUDE_ARGS=( - --print --verbose --output-format text --dangerously-skip-permissions + --print --verbose --output-format text + --permission-mode acceptEdits + --allowedTools Bash + --setting-sources project,local --add-dir "$APP_DIR" - --add-dir "$SCRIPT_DIR" - --add-dir "$SCRIPT_DIR/../llm-oneshot/apps/chat-app/prompts" --session-id "$SESSION_ID" ) [[ -n "$MODEL" ]] && CLAUDE_ARGS+=(--model "$MODEL") @@ -974,4 +980,3 @@ if node "$SCRIPT_DIR_NATIVE/parse-telemetry.mjs" "$RUN_DIR_NATIVE" "--logs-file= else echo "WARNING: Telemetry parsing failed. Raw logs at: $SHARED_TELEMETRY_DIR/logs.jsonl" fi - diff --git a/tools/stack-bench/.gitignore b/tools/stack-bench/.gitignore new file mode 100644 index 00000000000..a16e5569f30 --- /dev/null +++ b/tools/stack-bench/.gitignore @@ -0,0 +1,14 @@ +media/ +dist/ +local-notes/ +*.local.md +.loop-test/ +results/ +snapshot-l*/ +grader/.candidates/ +.spacetime-data/ +.spacetime-data.bak-*/ +grader/.mutation-report.json +tracks/*/overview.html +transcripts/ +operator.env diff --git a/tools/stack-bench/GETTING-STARTED.md b/tools/stack-bench/GETTING-STARTED.md new file mode 100644 index 00000000000..403105e4eed --- /dev/null +++ b/tools/stack-bench/GETTING-STARTED.md @@ -0,0 +1,86 @@ +# Start your first Stack Bench run + +## Requirements + +- A clean Git clone of this repository. +- Docker running Linux containers, with Docker Compose and BuildKit. +- Internet access for the first image and package downloads. +- At least 4 CPUs and 8 GiB of memory available to Docker. For the first source + build, plan for 16 GiB of memory and 60 GiB of free Docker storage. + +Docker stores the images and results on its configured disk. Choose that disk in +Docker Desktop before the first build if the system drive is short of space. +These are planning allowances, not measured minimums for every run. + +Use one installation per Docker daemon. The demo reuses the appliance state volume, +container names, and local ports. Do not run it alongside another Stack Bench +installation on the same daemon. + +## Run the model-free demo + +From the repository root: + +```sh +docker compose -f tools/stack-bench/appliance/demo.compose.yaml run --build --rm demo +``` + +Wait for startup, then open [Stack Bench](http://localhost:7331). The first source +build includes the server and SDK and can take substantial time. Leave the command +running until it reports completion or an error. + +The demo tests supplied reference apps on three stacks. It makes no model calls +and needs no provider credentials. Open its campaign to see progress and results. +This checks the runner; it does not measure a coding model. + +## Start a model run + +1. Configure a provider credential using the + [credential setup instructions](appliance/README.md#provider-credentials). + There is no separate dashboard password. +2. Open **New run**. Select the workload, level, stacks, model, reasoning effort, + SDK skills, dev workflow, repetitions, repairs, and limits + ([defaults](dashboard/README.md#pages)). **Production-quality app** is on by default. It adds: “Build a production-quality + application suitable for real users, not a prototype or demo.” You can turn it + off before review. +3. Review the configuration, attempt count, and cost cap. Select **Start**. +4. Open the run to follow its status. Keep excluded and incomplete attempts in + your review. Provisional results are not qualified comparisons. + +The demo and model runs use the same local dashboard. Starting model work can +consume provider credit or account usage; the demo does not. + +The production-quality option changes the request, not the checks. It is recorded +with the run. Earlier frozen runs retain their original prompts. Compare results +with the same setting, or label the prompt difference. +For CLI/API run preparation, set `productionQuality` to `false` to opt out; omitted +values default to `true`. Direct coding runs also accept `--no-production-quality`. + +## Read and keep results + +Open a run, then an attempt, to inspect checks, logs, and transcripts. The campaign's +**Files** menu links to available report files. Check completion and feature +completion are different measures; the dashboard lets you select either. + +Results remain in the `stack-bench-state` Docker volume. Follow +[results and export](appliance/README.md#results-and-cleanup) to copy a report and +its evidence to the host before you remove that volume. + +## Stop and clean up + +Use the run's **Stop** control to cancel its active work. Stopping a run is not a +pause. Wait for cleanup to finish and inspect any cleanup error. + +When no run is active, stop the Stack Bench dashboard and cache containers in +Docker Desktop. This preserves results. Do not delete the state volume to stop +the dashboard. For interrupted work, follow [recovery](appliance/RECOVERY.md). + +## Use the CLI instead + +The controller's `job options`, `job prepare`, and `job start` commands run the +same flow without the browser. `prepare` returns the same review the dashboard +shows; `start` accepts only that review. See the +[run-setup interface](dashboard/README.md#workload-setup-and-ai-access) for the +selection fields, and the [appliance guide](appliance/README.md#run-a-campaign) +for running an authored campaign file directly. + +For development and grading internals, use the [documentation index](docs/README.md). diff --git a/tools/stack-bench/README.md b/tools/stack-bench/README.md new file mode 100644 index 00000000000..fb51263b530 --- /dev/null +++ b/tools/stack-bench/README.md @@ -0,0 +1,66 @@ +# Stack Bench + +Stack Bench compares how coding agents build the same application with different +technology stacks. It runs each attempt in an isolated container, tests real +behavior, supports optional bounded repairs, and keeps the evidence behind every result. + +**New here? [Start your first run](GETTING-STARTED.md).** The model-free demo needs +only Git and Docker: + +```sh +docker compose -f tools/stack-bench/appliance/demo.compose.yaml run --build --rm demo +``` + +Then open [localhost:7331](http://localhost:7331). + +## What it does + +1. Compiles a versioned campaign that fixes the product request, model, stacks, + checks, budgets, repetitions, and parallelism. +2. Verifies the runner before model work starts. +3. Gives the coding agent only the current product work and selected stack + material. The agent cannot see the grader, checks, scores, or comparison data. +4. Grades the running app through separate browser sessions and stack operations. +5. When repairs are enabled, returns conclusive app failures within the declared budget. +6. Records check completion, token usage, cost, duration, source identity, and + supporting evidence. Weighted scores remain separate from completion. + +Only compatible attempts with validated evidence become comparison data. Provider +failures, harness failures, and incomplete measurements remain visible but separate. +Results are provisional until the selected checks have current +[qualification](reference-apps/README.md). + +## Scope + +- **Stacks:** SpacetimeDB, PostgreSQL, and MongoDB. Self-hosted Convex supports + the ecommerce dependency L1–L3 path. +- **Tracks:** [ecommerce](tracks/ecommerce/LEVELS.md) (storefront and warehouse) + and [chat](tracks/chat/LEVELS.md). +- **Modes:** sequential levels, where each level must pass before the next, or a + dependency graph, where each feature opens once its parents pass. +- **Agents:** Claude Code, Codex, and OpenRouter, through the + [appliance](appliance/README.md#provider-credentials). + +Campaign execution runs in the Linux Docker appliance. Source checks also run +on a development machine. + +## Documentation + +Start with [Getting started](GETTING-STARTED.md), then the +[appliance guide](appliance/README.md) for campaigns. Everything else is in the +[documentation index](docs/README.md). + +## Layout + +- `tracks/` owns product requests, feature definitions, checks, and scenarios. +- `conditions/` owns guidance and repair feedback. +- `backends/` owns stack material sent to the coding agent. +- `src/stacks/` owns runtime stack adapters. +- `commands/` and `src/` own the CLI and reusable benchmark logic. +- `grader/`, `linter/`, and `reference-apps/` own validation. +- Retained qualification records go in `qualification-evidence/`, cited by path + from each calibration. None are retained while qualification is pending. +- `appliance/` owns deployment. `dashboard/` is an optional interface. + +Prompt selection and scoring selection stay separate. A behavior can be measured +without being named in the product request. diff --git a/tools/stack-bench/appliance/Controller.Dockerfile b/tools/stack-bench/appliance/Controller.Dockerfile new file mode 100644 index 00000000000..682dc01d356 --- /dev/null +++ b/tools/stack-bench/appliance/Controller.Dockerfile @@ -0,0 +1,123 @@ +# syntax=docker/dockerfile:1.7 + +FROM docker:29.6.2-cli@sha256:feb2d49bd65f274b3e4b4620beabe2f4691e5287e496da9fbc9830ed5f780676 AS docker-cli +FROM mcr.microsoft.com/playwright:v1.62.1-noble@sha256:c091b21d9fae78c76e85cd4356431e9b018402f172a214fc7d7a5e9a7e29d8ac AS source +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN apt-get update && apt-get install -y --no-install-recommends lsof=4.95.0-1build3 util-linux \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace +# Git metadata is mounted only for trusted source export/verification. It is never +# copied into an image layer. Release builds require a clean normal Git checkout. +RUN --mount=type=bind,target=/checkout \ + git -c safe.directory=/checkout -C /checkout archive HEAD | tar -x -C /workspace +WORKDIR /workspace/tools/stack-bench +RUN npm ci --ignore-scripts --no-audit --no-fund && npm run build \ + && node dist/src/references/reference-fixtures.js \ + && node dist/commands/check-calibration.js +# Normalize checkout text as Git does on Windows; explicit .gitattributes still apply. +RUN --mount=type=bind,target=/checkout \ + GIT_OPTIONAL_LOCKS=0 GIT_CONFIG_COUNT=2 GIT_CONFIG_KEY_0=safe.directory GIT_CONFIG_VALUE_0=/checkout \ + GIT_CONFIG_KEY_1=core.autocrlf GIT_CONFIG_VALUE_1=input \ + node --input-type=module -e 'import {writeFileSync} from "node:fs"; import {releaseSourceIdentity,binarySourceIdentity} from "./dist/src/releases/release-source.js"; writeFileSync("/workspace/stack-bench-source.json",JSON.stringify(releaseSourceIdentity("/checkout"))); writeFileSync("/workspace/stack-bench-binary-source.json",JSON.stringify(binarySourceIdentity("/checkout")));' + +FROM rust:1.93-slim-bookworm@sha256:8f8609d448e821fbc0e44241bc5ca4ce49663cc6306ff1a17f655a0e2a7cd084 AS binary-build +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev build-essential clang cmake perl git curl python3 \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /workspace +COPY --from=source /workspace/ ./ +# The existing source-archive path in cli/build.rs avoids retaining Git metadata. +RUN --mount=type=cache,id=stack-bench-rust-target,target=/target \ + --mount=type=cache,id=stack-bench-cargo-registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=stack-bench-cargo-git,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,id=stack-bench-rustup,target=/usr/local/rustup,sharing=locked \ + export SPACETIMEDB_NIX_BUILD_GIT_COMMIT="$(python3 -c 'import json; print(json.load(open("stack-bench-source.json"))["revision"])')" \ + && CARGO_TARGET_DIR=/target cargo build --release --locked \ + -p spacetimedb-cli --bin spacetimedb-cli \ + -p spacetimedb-standalone --bin spacetimedb-standalone \ + && mkdir -p /binaries \ + && cp /target/release/spacetimedb-cli /target/release/spacetimedb-standalone /binaries/ + +FROM source AS stack-bench-build +COPY --from=binary-build /binaries/ ./container/bin/ +RUN node dist/container/binary-provenance.js record-snapshot \ + --root /workspace/tools/stack-bench --source-file /workspace/stack-bench-binary-source.json \ + && npm prune --omit=dev --ignore-scripts --no-audit --no-fund + +# build-linux-cli.sh uses this same build, rather than a second Rust recipe. +FROM scratch AS binary-export +COPY --from=stack-bench-build /workspace/tools/stack-bench/container/bin/ /bin/ +COPY --from=stack-bench-build /workspace/tools/stack-bench/container/spacetimedb-binaries.json /spacetimedb-binaries.json + +FROM source AS sdk-build +WORKDIR /workspace +RUN corepack enable \ + && pnpm --filter spacetimedb install --frozen-lockfile --ignore-scripts \ + && pnpm --filter spacetimedb run build \ + && test -f crates/bindings-typescript/dist/server/index.d.ts \ + && test -f crates/bindings-typescript/dist/server/index.mjs + +FROM mcr.microsoft.com/playwright:v1.62.1-noble@sha256:c091b21d9fae78c76e85cd4356431e9b018402f172a214fc7d7a5e9a7e29d8ac + +ENV NODE_ENV=production \ + PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + +COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=docker-cli /usr/local/libexec/docker/cli-plugins /usr/local/libexec/docker/cli-plugins +ADD --checksum=sha256:4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71 \ + https://github.com/sigstore/cosign/releases/download/v3.1.3/cosign-linux-amd64 \ + /usr/local/bin/cosign + +RUN apt-get update \ + && apt-get install -y --no-install-recommends lsof=4.95.0-1build3 nftables util-linux \ + && chmod 0555 /usr/local/bin/cosign \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/stack-bench +LABEL org.opencontainers.image.title="Stack Bench controller" + +COPY --from=stack-bench-build /workspace/tools/stack-bench/ ./ +COPY --from=stack-bench-build /workspace/stack-bench-source.json ./source-identity.json +RUN chmod 0555 /opt/stack-bench/dist/container/browser-pipe.js +COPY --from=source /workspace/skills/ /skills/ +COPY --from=source /workspace/crates/bindings-typescript/ /opt/stack-bench-embedded-deps/bindings-typescript/ +COPY --from=sdk-build /workspace/crates/bindings-typescript/dist/ /opt/stack-bench-embedded-deps/bindings-typescript/dist/ +COPY --from=source /workspace/licenses/BSL.txt /opt/stack-bench-embedded-deps/BSL.txt + +RUN node dist/container/binary-provenance.js verify \ + --root /opt/stack-bench --source-sha256 "$(node -p "require('./source-identity.json').binarySourceSha256")" \ + && test "$(node -p "require('playwright/package.json').version")" = "1.62.1" \ + && rm -rf tests dist/tests \ + && node --input-type=module -e 'import {AGENT_ADAPTER_REGISTRY,agentAdapterIdentity} from "./dist/src/agents/agent-adapters.js"; for (const id of AGENT_ADAPTER_REGISTRY.ids) agentAdapterIdentity(AGENT_ADAPTER_REGISTRY.get(id));' \ + && node --input-type=module -e 'import {resolveGuidanceProfile} from "./dist/src/campaigns/condition-compiler.js"; for (const id of ["neutral-dev","neutral-managed-dev"]) resolveGuidanceProfile(id,["mongodb","postgres","spacetime"]);' \ + && install -m 0555 container/bin/spacetimedb-cli \ + /opt/stack-bench-embedded-deps/spacetimedb-cli \ + && install -m 0555 container/bin/spacetimedb-standalone \ + /opt/stack-bench-embedded-deps/spacetimedb-standalone \ + && rm /opt/stack-bench-embedded-deps/bindings-typescript/LICENSE.txt \ + && mv /opt/stack-bench-embedded-deps/BSL.txt \ + /opt/stack-bench-embedded-deps/bindings-typescript/LICENSE.txt \ + && chmod 0444 /opt/stack-bench-embedded-deps/bindings-typescript/LICENSE.txt \ + && chmod 0555 /opt/stack-bench-embedded-deps/spacetimedb-cli \ + /opt/stack-bench-embedded-deps/spacetimedb-standalone \ + && cd /opt/stack-bench-embedded-deps/bindings-typescript \ + && pack_name="$(npm pack --pack-destination /opt/stack-bench-embedded-deps --silent)" \ + && mv "/opt/stack-bench-embedded-deps/$pack_name" /opt/stack-bench-embedded-deps/spacetimedb.tgz \ + && tar -tzf /opt/stack-bench-embedded-deps/spacetimedb.tgz | grep -Fxq package/dist/server/index.d.ts \ + && tar -tzf /opt/stack-bench-embedded-deps/spacetimedb.tgz | grep -Fxq package/dist/server/index.mjs \ + && cd /opt/stack-bench \ + && node dist/appliance/dependency-volume.js manifest \ + --source /opt/stack-bench-embedded-deps \ + --out /opt/stack-bench/dependency-manifest.json \ + && node dist/appliance/dependency-volume.js verify \ + --target /opt/stack-bench-embedded-deps \ + --manifest /opt/stack-bench/dependency-manifest.json \ + && rm -rf results .spacetime-data .loop-test + +# Direct launches use the image's pinned dependencies; Compose can override them. +ENV SPACETIME_BIN=/opt/stack-bench-embedded-deps/spacetimedb-cli \ + STDB_PACKAGE=/opt/stack-bench-embedded-deps/bindings-typescript + +ENTRYPOINT ["node", "/opt/stack-bench/dist/appliance/controller.js"] +CMD ["--help"] diff --git a/tools/stack-bench/appliance/Controller.Dockerfile.dockerignore b/tools/stack-bench/appliance/Controller.Dockerfile.dockerignore new file mode 100644 index 00000000000..6541c1bd7ac --- /dev/null +++ b/tools/stack-bench/appliance/Controller.Dockerfile.dockerignore @@ -0,0 +1,46 @@ +* +!tools +!tools/stack-bench +!tools/stack-bench/** +!skills +!skills/** +!crates +!crates/** +!licenses +!licenses/BSL.txt +!Cargo.toml +!Cargo.lock +!rust-toolchain.toml +!.cargo +!.cargo/** +!.git +!.git/** +!.gitattributes +!.gitignore +!templates +!templates/** +!package.json +!pnpm-lock.yaml +!pnpm-workspace.yaml + +**/node_modules +**/dist +**/target +tools/stack-bench/results +tools/stack-bench/.spacetime-data* +tools/stack-bench/.loop-test +tools/stack-bench/*.local.md +tools/stack-bench/**/*.local.md +tools/stack-bench/transcripts +tools/stack-bench/local-notes +tools/stack-bench/media +tools/stack-bench/snapshot-l* +tools/stack-bench/grader/.candidates +tools/stack-bench/grader/.mutation-report.json +tools/stack-bench/tracks/*/overview.html + +# These are generated by the Docker build, never host inputs. +tools/stack-bench/container/bin +tools/stack-bench/operator.env +.git/hooks +.git/logs diff --git a/tools/stack-bench/appliance/DESIGN.md b/tools/stack-bench/appliance/DESIGN.md new file mode 100644 index 00000000000..ba697ff2615 --- /dev/null +++ b/tools/stack-bench/appliance/DESIGN.md @@ -0,0 +1,109 @@ +# Stack Bench appliance design + +The demo runs through one Docker appliance on a computer with Linux-container +Docker and Compose. The runtime targets `linux/amd64`. A clean image build and +an appliance rehearsal remain release gates. Source tests do not prove that +full delivery path. + +## Runtime ownership + +The trusted controller owns definitions, grading, results, provider secrets, +and the Docker socket. It runs in Docker's host network so its HTTP checks can +reach the app and SpacetimeDB ports published on loopback. The dashboard uses a +bridge and publishes only `127.0.0.1:7331`. Dashboard actions start a fresh +controller through Compose; the dashboard does not execute a campaign in its +own network namespace. + +Each real attempt owns one backend container and one bridge. The backend is +the network namespace anchor. Its coding container, browser, smoke container, +and provider broker join that exact namespace. They do not share another +attempt's loopback or backend. PostgreSQL and MongoDB have private per-attempt +credentials. Their native database ports are not published on the host. + +The controller writes a private lease and claims capacity and port locks before +it creates resources. It records creation authority before Docker create and +exact IDs before start. A short-lived helper installs native nftables rules +with `NET_ADMIN`, then exits. Only after that step does native backend startup +run. Generated code and the browser cannot change the firewall. + +The anchor uses Docker's init process to reap stopped children. SpacetimeDB +restart replaces its server process inside the same namespace. An anchor +restart or replacement invalidates the recorded namespace start time and +requires recovery of the whole attempt. + +## Network access + +The namespace permits: + +- its own loopback services and Docker DNS; +- the trusted npm cache at its exact IPv4 address and TCP port 4873; +- public IPv4 HTTP and HTTPS traffic; +- replies to connections already established by trusted host checks. + +It blocks host, private, link-local, and other-attempt destinations outside the +cache exception. Outbound IPv6 is blocked except loopback. Public HTTP and +HTTPS access is intentional; this is not an offline sandbox or a provider +hostname allowlist. + +The trusted npm cache temporarily joins each owned bridge. Teardown detaches +only that bridge membership. Cache publishing and user registration are +disabled. Teardown does not reset or stop the shared cache. + +The browser has no Docker socket, provider secret, app mount, or TCP control +port. Playwright carries its control pipes through `docker exec`. Page scripts +run in the attempt namespace. The controller can inspect the app over its +loopback-published port without exposing its grader endpoints to page scripts. + +## Files and credentials + +The `stack-bench-state` named volume holds work, results, secrets, and private +recovery records. Setup asks Docker for the volume's native mountpoint. The +controller mounts it at that same path. Child bind mounts therefore name the +same bytes on Docker Desktop and Linux. There is no host-directory translation +layer and no required host `/var/lib/stack-bench` directory. + +The coding container receives its app workspace, transcript directory, and +only the selected stack's declared dependency mounts. It does not receive the +repository, grader, scenarios, recipes, results, Docker socket, or provider +credential file. It uses a read-only root filesystem, resource limits, and +`no-new-privileges`. Trusted setup can switch users; the coding agent runs with +its reduced identity. + +The controller gives each provider broker a private credential file and gives +the agent a short-lived session token. The broker runs as a separate container +and records usage for cost reconciliation. Its private files and identity are +retained if the broker cannot be stopped. Public artifacts omit +lease tokens and creation tokens, and public diagnostics redact credentials. + +A read-only dependency volume holds the exact SpacetimeDB CLI, server, and SDK +embedded in the controller image. Its initializer verifies a checksum marker +and refuses incompatible existing contents. Setup pulls the pinned native +PostgreSQL and MongoDB images when they are absent. + +## Parallel runs and recovery + +A campaign's `parallelism` is its requested worker count. Admission finds unused +run indices and valid host ports for each dispatched attempt's stack. Linux `flock` +protects that claim across controller processes. There is no separate manual pool. +Campaign children receive private, single-use delegated authority. Each reservation +is released after its attempt proves cleanup. Explicit capacity policy controls +waiting or failure; requested parallelism never changes. Dashboard and CLI Stop +share owner checks. See [execution jobs](../docs/execution-jobs.md) for submission. + +Every attempt runs its own model-free smoke after lease activation and before +agent execution. Parent admission cannot substitute an earlier smoke result. +Bounded Docker checks have proven private routes, browser control pipes, native +backend activation, concurrent attempts, and exact cleanup. They do not replace +reference grading qualification or a clean release rehearsal. + +Normal teardown removes exact owned sidecar and backend IDs, owned database +volumes, and the owned network. Recovery can also find a resource created before +its ID was saved, but only when its private creation label still matches. +Failed cleanup keeps authority and locks for an authenticated retry. Never +remove a same-name resource merely because its name resembles Stack Bench. + +The plan owns requested work and rules. Campaign state owns scheduling. +Attempt artifacts own results. Reports and the dashboard show those records; +they do not change grades. Release identity and qualification requirements are +specified in [RELEASE.md](RELEASE.md). Operator steps are in [README.md](README.md), +and interruption handling is in [RECOVERY.md](RECOVERY.md). diff --git a/tools/stack-bench/appliance/README.md b/tools/stack-bench/appliance/README.md new file mode 100644 index 00000000000..601662efd5e --- /dev/null +++ b/tools/stack-bench/appliance/README.md @@ -0,0 +1,650 @@ +# Stack Bench appliance + +The demo uses one Docker appliance. Docker runs the controller, coding sessions, +backends, browser grader, and package cache. The host needs Git and Docker; it +does not need Node, Rust, PostgreSQL, MongoDB, or local SpacetimeDB binaries. + +## Run the demo + +From the repository root of a clean checkout, with Docker running: + +```sh +docker compose -f tools/stack-bench/appliance/demo.compose.yaml run --build --rm demo +``` + +When startup completes, open [localhost:7331](http://localhost:7331). Docker +cannot open the host browser for you. This command builds the images, prepares +state, starts the dashboard, and runs a model-free example: nine checks across +three reference stacks in parallel. No provider credential or model spend is +needed. Reference results demonstrate the runner; they do not measure a coding +agent's ability. The dashboard stays running after the example finishes. You can +stop its container in Docker Desktop; results remain in the state volume. + +The bootstrap stores its resolved environment at +`/state/controller-home/demo.env` in the `stack-bench-state` volume. Campaign +evidence also stays in that volume. The first source build can take substantial +time. Requirements are below; manual setup and paid campaigns follow. + +## Requirements + +- Docker must run Linux containers for `linux/amd64`, with BuildKit and the + Compose plugin. The current tool checks used Docker Engine 29.6.2 and Compose + 5.3.1. Other versions are not yet part of the release proof. +- Preflight requires 4 CPUs, 8 GiB total Docker memory allocation, and 10 GiB + free result storage. This is a conservative startup policy, not a measured + minimum for every workload. Choose parallel capacity for the available + resources and the work being run. +- For the first source build, reserve additional disk space and build time. + 16 GiB RAM and 60 GiB free Docker storage are conservative planning allowances, + not measured minimums. The release rehearsal must record actual build use. +- Use a clean, normal Git clone of the delivered branch. Release builds reject + changed release inputs. A host Git worktree whose `.git` points outside the + build context is not a release checkout. +- Docker needs internet access to the pinned base images, build packages, + public npm registry, and selected model provider. Model work also needs a + supported provider credential and a declared spend limit. + +The controller uses the Docker socket to manage containers. Its state uses the +`stack-bench-state` named volume. The controller mounts that volume at the Docker +daemon's own mountpoint, so child bind mounts use the same path on Docker Desktop +and a Linux host. No host `/var/lib/stack-bench` directory is required. + +For each release, verify the clean-branch image build and one-command demo on a +fresh machine. A passing source test alone does not prove the whole appliance. + +## Advanced: manual setup and paid campaigns + +If you already ran the demo, its images and state are ready. To use the CLI +commands below, copy its saved setup into a local environment file once. Run +from the repository root: + +```sh +docker run --rm --mount type=volume,source=stack-bench-state,target=/state,readonly --entrypoint cat stack-bench-controller:local /state/controller-home/demo.env > tools/stack-bench/operator.env +``` + +In Windows PowerShell 5, replace `>` with `| Out-File -Encoding utf8`. +Then go to [provider credentials](#provider-credentials) for model work or +[validate the appliance](#validate-the-appliance) for model-free checks. +Do not repeat the image builds or setup below after a successful demo. + +For a new manual installation, build and prepare state as follows. +From the repository root: + +```sh +docker build --platform linux/amd64 -t stack-bench-build:local tools/stack-bench/container +docker build --platform linux/amd64 -f tools/stack-bench/appliance/Controller.Dockerfile -t stack-bench-controller:local . +docker run --rm --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock stack-bench-controller:local setup > tools/stack-bench/operator.env +``` + +The controller build exports the clean Git revision, builds the native binaries +and SDK, and records their source and checksums in the image. It does not consume +ignored host binaries. BuildKit caches the Rust target and package downloads. +The first build can take substantial time; complete it before the demo. + +Commands below are single lines so they can be pasted into PowerShell or a +POSIX shell. Docker must be running before `setup` or Compose commands. + +`setup` creates the state volume and directories, resolves both local images to +immutable content IDs, and writes a UTF-8 Compose environment file. It installs +the pinned PostgreSQL, MongoDB and Convex images when they are absent. It also installs +the prepared plans, including the four-stack ecommerce L1–L3 plan. It keeps existing plans and model credentials. Keep `operator.env` locally; it is ignored by Git. In Windows +PowerShell 5, use `| Out-File -Encoding utf8 tools/stack-bench/operator.env` instead +of `>` so the environment file is not UTF-16. + +### Provider credentials + +For a model-free check, no provider secret is needed. To configure model work, +write the subscription token through stdin: + +```sh +docker run --rm -i --mount type=volume,source=stack-bench-state,target=/state stack-bench-controller:local set-secret claude_subscription_token +``` + +Supply the token on stdin and close input. For API billing, use secret name +`anthropic_api_key` and set `STACK_BENCH_AGENT_AUTH=api-key` in `operator.env`. +The secret stays in a private volume file; it is not a command argument or +part of the environment file. The Docker socket is not needed by `set-secret`. +After pasting the token and pressing Enter, close stdin with Ctrl+D in a POSIX +terminal, or Ctrl+Z followed by Enter in Windows PowerShell. + +Run the remaining commands from `tools/stack-bench`. Compose uses the state +volume's results directory as its working directory, so `plans/...` and +`campaigns/...` refer to durable state inside Docker. + +### OpenAI credentials + +Select the `codex` agent adapter and an explicit billing mode in `operator.env`: + +- `STACK_BENCH_AGENT_AUTH=openai-api-key` uses + `STACK_BENCH_OPENAI_API_KEY_FILE`. Store the key with `set-secret openai_api_key`. +- `STACK_BENCH_AGENT_AUTH=openai-account` uses `STACK_BENCH_CODEX_AUTH_FILE`. + Use `codex login` with file credential storage, then send only its `auth.json` + through standard input to `set-secret codex_auth`. Do not copy your Codex home. + +Setup writes the two file paths below the state volume's `secrets` directory. +Use the same Docker `set-secret` command shown above with the selected secret name. +API usage and account plan usage are separate billing modes. There is no fallback +from an account to an API key. See the [official authentication documentation](https://learn.chatgpt.com/docs/auth). + +Account mode takes an access-token snapshot in the trusted controller. Neither +the login file nor its refresh token reaches generated commands. This version +does not refresh account tokens. An expired token fails before coding starts; +expiry during a request stops that request as a provider failure. Log in again +and replace the stored file before another attempt. +OpenAI receipts use observed tokens and the plan's frozen rates. They are a +comparison cost, not an account-plan invoice. Use rates that cover the selected +model and context range. The initial broker supports text and local function +or custom tools. It rejects hosted tools, images, remote files, stored prompts, +and server-side conversation references because these need other cost bounds. + +Rebuild the build image to include the pinned Codex CLI before using this adapter. +The local mock check verified the CLI request and usage stream, not live account +access. The pinned CLI reports missing model metadata and uses fallback settings +for the selected model. Confirm those settings in a qualification run before +using this adapter for a published comparison. + +Account mode supports `gpt-5.3-codex`, `gpt-5.4`, `gpt-5.4-2026-03-05`, +`gpt-5.6-sol`, and `gpt-6-astra`. +The broker reserves each request against the documented 128,000-token output +bound. Unknown account models fail before a provider request. API mode uses an +explicit `max_output_tokens` limit. Both modes still require explicit campaign +pricing. See the [GPT-5.3-Codex model limits](https://developers.openai.com/api/docs/models/gpt-5.3-codex) +and [GPT-5.4 model limits](https://developers.openai.com/api/docs/models/gpt-5.4). + +### OpenRouter credentials and routing + +Select the `openrouter` adapter. It uses the same Codex coding runtime as the +`codex` adapter. Store its API key with `set-secret openrouter_api_key`, then set +`STACK_BENCH_AGENT_AUTH=openrouter-api-key` and +`STACK_BENCH_OPENROUTER_API_KEY_FILE` in `operator.env`. + +Each campaign agent selection must fix `model`, `providerRoute`, and `maxOutputTokens`, for example: + +```json +{ "adapter": "openrouter", "adapterVersion": "1.0.0", + "model": "openai/gpt-5.3-codex", "providerRoute": "openai", "maxOutputTokens": 8192 } +``` + +The example identifies a model and route; it is not live qualification evidence. +Use a model with Responses API and local tool support. The broker fixes one provider route, disables fallback, and rejects route changes +during continuation. A base provider slug can include several endpoint variants; +use a specific endpoint slug when that distinction matters. Receipts retain the +provider reported by OpenRouter. +See [OpenRouter routing](https://openrouter.ai/docs/guides/routing/provider-selection) +and the [Responses API](https://openrouter.ai/docs/api/reference/responses/overview). + +Use `--agent-adapter openrouter --provider-route openai --max-output-tokens 8192` +for standalone preflight. +Preflight checks local setup and credentials without a provider request. + +Set the output token limit within the selected model endpoint's documented cap +and the broker's 128,000-token safety cap. This is a per-request bound. +Declare token price ceilings and a cost limit in the campaign. The broker sets +routing price limits, rejects request fees, and records reported `usage.cost`. +These receipts are OpenRouter charges, not costs inferred from token counts. +Missing cost evidence cannot produce an exact receipt. API key billing is the +supported mode; account subscriptions and BYOK are not supported. +Rebuild the coding and controller images before use. Mock tests do not prove +live model access or model quality; qualify the selected model and endpoint +before publishing comparisons. + +## Validate the appliance + +Run commands from `tools/stack-bench` on the runner. + +Check the Compose configuration: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml config --quiet +``` + +Run preflight for the exact planned scope: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm controller preflight --backend spacetime,postgres,mongodb --track ecommerce --levels 1 --run-index 0 --agent-adapter reference-fixture --guidance neutral +``` + +This preflight selects the model-free reference adapter and needs no provider +credentials. For model work, use the planned agent adapter and its configured +credential. This preflight verifies the runner, images, dependencies, ports, +and storage without creating an attempt. Each campaign attempt then runs its +own smoke check inside its activated private network, before the agent starts. +Standalone appliance preflight is read-only. The reference trial below exercises +the automatic smoke checks without model calls. + +## Check the delivered runtime + +This command starts real containers and grades the shipped reference app. It +makes no model calls. Run it before collecting model results: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm controller campaign trial plans/reference-check.json --out campaigns/reference-check +``` + +Inspect its status and evidence before using the appliance for model work. +A reference fixture pass checks the runtime path; it is not evidence that an +agent implemented the product. + +## Inspect a campaign + +To correct grading after a grader fix, use the saved execution in a separate +output directory. This runs no coding agent and has no provider cost: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm controller run --grade-from campaigns/example/attempts/attempt-id/execution-1 --out regrades/attempt-id +``` + +This path accepts a completed single-level sequential run. It verifies the +source checkpoint, keeps the original check scope and account aliases, and +rejects changes to the product request or contract. Use the original build +image and dependency bundle. Startup must reproduce the saved source without +changes. Repeat `--check ` to regrade only affected checks from +the original scope. The separate `regrade.json`, grading bundle, and cleanup evidence do +not replace the original run or create another build sample. For saved dependency +candidates, select `--grade-level` and affected checks as described in the +[dependency replay method](../docs/grading-coverage.md#replay-a-saved-dependency-candidate). +An interrupted dependency run can replay an earlier candidate only after authenticated +recovery proves cleanup and that candidate has its complete source-bound grade bundle. +The diagnostic preserves the interrupted parent status; it does not finish the campaign. + +The campaign file is the run authority. Store it below +`plans/` in the state volume. + +`setup` installs `plans/paid-l1.json` from the supplied +[`campaign.paid-l1.json`](campaign.paid-l1.json), binds it to the local controller +and build image IDs, and freezes it for execution. It runs one fresh L1 build per stack, +three in parallel, with no repairs or retries and a $10 limit per attempt +($30 maximum across the three attempts). It uses Sonnet 5 and includes the +SpacetimeDB skills. Its results are provisional. The example already sets +`parallelism: 3`; change a draft and freeze it before execution if needed. +Inspect its model, stacks, repetitions, spend limits, and pricing before launch. +For a longer study, [`campaign.paid-l1-l3.json`](campaign.paid-l1-l3.json) is a +draft three-stack progression pilot. It selects L1 through L3, six repairs total +per attempt, no execution retries, a 120-minute attempt limit, and a $30 +per-attempt cap ($90 maximum). These are proposed limits, not a cost estimate. +It must be bound to the selected image identities, frozen, and installed under +`plans/` before execution; setup does not install it automatically. Earlier +levels must pass before later levels start. See the +[study method](../docs/study-method.md) for collection and analysis rules. +`appliance/campaign.example.json` is a model-free reference plan; changing its +title does not make it a coding-agent campaign. + +Use a new manifest ID and output directory for a new comparison. Do not edit a +running campaign's plan. Setup preserves an existing `paid-l1.json`; it does not +silently replace or rebind it after an image rebuild. A frozen plan copied from +another machine binds that machine's image identities and cannot run unchanged. + +Compile and inspect it without model work: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm controller campaign show plans/paid-l1.json +``` + +A test plan selects the model, stacks, work, checks, budgets, repetitions, +parallelism, pricing, controller image, and build image. When the run starts, +Stack Bench records these settings with the results. This prevents settings +from changing during a campaign. + +The manifest also defines how repair work is selected and limited: + +```json +"repair": { "selection": "feature", "budget": { "perFeature": 1 } } +``` + +Dependency mode supports `feature` or `batch` selection. The budget must name +at least one limit; each is a non-negative integer with no upper cap: + +- `total`: repairs across the whole attempt. `0` runs the initial grade and + advances passed branches without any repair. +- `perFeature`: repairs that may include one feature. +- `perDepth`: `{ "count": N, "carry": true | false }`. Each opened depth adds + `count` repairs; `carry` keeps unused depth repairs available later. + +When several features have failed, the next repair goes to the first of them +by dependency depth, then by `order`: + +- `declared` (default): the order the catalog declares its features, which is + part of the catalog's identity. +- `shuffled`: a permutation within each depth drawn once from the campaign's + `ordering.seed` when the plan compiles, frozen in the plan as the policy's + `nodeOrder`, and used by every stack in the campaign. The catalog and its + qualification are unchanged; the policy identity carries the order. +When limits are combined, the tightest remaining limit wins, and the result +names which one stopped a feature: `feature-repairs-exhausted`, +`depth-repairs-exhausted`, `total-repairs-exhausted`, or `repeated-findings` +when the same failures reach the configured observation count. The initial +failure counts as one; each completed repair with the same failures adds one. +Set `mode.unchangedFailureLimit` to a positive integer (default 3). To allow +five repairs even when all fail identically, set it to 6. A completed +repair counts even when its grade did not finish; its source is kept beside +the run and graded on resume before any further coding session. Sequential +mode requires `batch` selection and one `total` limit. + +Rejecting a repair restores its accepted source and recreates the isolated +grading database before restarting the app. A failed rollback preserves the +accepted source, grade evidence, and repair costs, and stops the attempt as a +harness failure. + +The plan, dashboard, and report show qualification status. Publish scores as +verified comparison data only after every selected level is qualified. + +`plans/reference-check.json` is the shipped zero-cost reference check. It uses +the reference adapter included in the controller image. `plans/ecommerce-progression.json` +keeps the full rigorous workload for reference validation; the short check does +not replace that workload. Both use hand-written reference apps and produce no +comparative model data. + +## Run a campaign + +Start the campaign. This command creates the run state and records the exact +test plan automatically: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm controller campaign run plans/paid-l1.json --out campaigns/campaign-001 +``` + +The campaign controls attempt counts and concurrency. `repetitions` sets the +default attempt count per stack. A stack can override it. `parallelism` limits +simultaneous attempts. Each live attempt receives isolated ports, database +names, locks, workspaces, and evidence paths. + +Coding, backend, browser, and broker containers share only their attempt's +private network namespace. Native firewall rules block host and other-attempt +connections. Concurrent native attempts and exact cleanup have passed the +bounded Docker checks. A clean appliance rehearsal remains a release gate. + +Set each plan's `parallelism` to the number of simultaneous attempts you want. +Admission automatically leases unused run indices and host ports across campaigns. +It reserves only each dispatched attempt's stack and releases the reservation after +verified cleanup. [Jobs](../docs/execution-jobs.md) select explicit capacity wait/fail +policy; requested parallelism never changes. No worker-pool setting is needed. The startup baseline +remains 4 CPUs and 8 GiB RAM. Preflight reports the requested campaign's +container caps and warns when their sum exceeds Docker's total allocation. +These caps do not reserve CPU or RAM and are not measured hardware minimums. + +Setup selects `STACK_BENCH_RUNNER_CAPACITY=dynamic`. Admission then checks +current host memory and CPU load instead of a fixed slot count. New claims +reserve startup headroom for one minute so concurrent launches cannot reuse the +same free-memory estimate. Campaigns and standalone qualification queue and retry +when resources are busy. The pressure check is an admission snapshot, not a +reservation against future spikes. Set a positive integer instead for a fixed +host quota measured for your workload. + +Each worker has a 2-CPU/4-GiB coding container, a 1-CPU/1-GiB backend, a +1-CPU/2-GiB browser, and a broker capped at 256 MiB when needed. Thus nine +workers have known caps totaling 36 CPUs and 65.25 GiB RAM. Broker CPU, +controller processes, the package cache, and Docker need additional resources. + +The shared package cache has caps of 1 CPU, 2 GiB RAM, and 128 processes. Allow +for its use alongside the controller and attempt containers. + +Docker's reported memory is total allocation, not free memory. Contention can +increase run time; heavier apps can exhaust memory. Validate the intended parallelism with the selected workload. A short fixture +test does not establish capacity for arbitrary model builds or timed grading. + +The 10-GiB disk check is a startup free-space check, not a per-worker reservation +or a storage quota. Concurrent installs, app builds, and retained evidence share +that storage. Keep space for their growth and for shared services. + +An attempt holds its worker from its first build through its final grade and +cleanup. Results remain provisional until their grading qualification is current. + +The remaining `campaign` snippets are controller subcommands. Run them after +the same Docker Compose `run --rm controller` prefix used above. + +Use durable state for normal control: + +```sh +campaign status +campaign stop +campaign inspect +campaign report +``` + +- `status` is the compact normal view. +- `stop` stops owned active work and retains its state and evidence. +- `inspect` adds score, cost, duration, cleanup, evidence, and feature progress. +- `report` rebuilds `report/report.json` and `report/report.html` from retained + evidence. + +Stop interrupts active attempts. A stopped sequential attempt remains invalid. +Running the same trial again starts only pending attempts; it does not restart +the interrupted attempt. Resume starts scheduled dependency work. + +Do not infer state from logs. Use logs only to diagnose a reported phase or +failure. Automatic retries are limited by the manifest +`attemptPolicy`. Additional repair grants and budget extensions require an +explicit operator action. + +## Resume and repair + +For a live provider wait, inspect the current execution: + +```sh +campaign continuation-status --attempt --json +campaign continue-provider --attempt --request-id +``` + +Use these commands through the controller, as with other campaign commands. +The second command authorizes one continuation of the current waiting session. +Fix the provider account first. Reusing a request ID for the same wait is +idempotent. An ID from an earlier wait cannot authorize a later wait. + +The wait keeps the app, database, candidate source, native session, and parallel +slot. Database clocks and background processes still run. Waiting consumes the +existing time allowance; `grant-time` can extend it separately. Continuation does +not add repairs, increase the cost limit, or change the provider, model, billing +mode, or task. All invocation receipts, wait events, and acceptance records stay +under the execution's `provider-waits` directory. +Reports read these events even when a killed process has no final agent result. +If no wait-end event exists, the last heartbeat gives a lower bound on wait time. + +The command works only while the original controller and execution remain live. +Cancellation, timeout, stale heartbeat, lost resources, or failed native-session +validation ends eligibility. A stopped historical execution cannot be restored +by this command. The current candidate is not graded during a provider wait. + +Add time to a running attempt without restarting its agent: + +```sh +campaign grant-time --attempt --grant-id --minutes 120 +``` + +This adds two hours to the existing limit. It does not change the frozen plan, +cost limit, or repair allowance. Reuse the same grant ID and minutes after an +uncertain response; a new ID requests more time. The request is pending until +the owning controller accepts it. The attempt page shows the effective limit +and the request status. Its **Add time** control uses the same operation. + +Only controllers built with time-grant support can accept live requests. +Do not replace a running controller to install this feature. A time grant does +not restart an expired attempt by itself. A stopped attempt can receive a grant +only at a verified completed-depth checkpoint, before the next build starts. +Interrupted coding and grading are not supported. Use the existing +`campaign resume --out ` command after the +grant. The dashboard combines these steps with **Add time and resume** only for +an eligible checkpoint. It shows the reason when continuation is not safe. +Source files alone do not restore an interrupted agent. + +The initial limit comes from `budgets.attemptTimeoutMinutes` in the frozen +plan. The Plans table shows it in hours and minutes. It includes coding, +grading, repairs, and host sleep, except for a verified planned depth pause. +Grant records retain the original limit and +each accepted extension; adding time alone does not invalidate efficacy data. + +If the controller stopped while an attempt remained live, reconcile ownership +before any resume: + +```sh +campaign reconcile --out +``` + +Reconciliation changes state only when private supervisor evidence proves that +the exact owned resources are clean. It does not restore an interrupted +database or agent session. Do not remove a worker claim or alter completion and +cost records to bypass continuation checks. + +Dependency campaigns can grant more repairs to selected exhausted features: + +```sh +campaign grant-repairs --attempt --grant-id --level --feature --repairs +``` + +The grant creates a linked continuation. It does not rewrite the completed +execution. Use `campaign resume --out ` to +run scheduled dependency work. + +## Continue to a higher level + +To keep the same live execution, select the full target before launch and use a +planned depth pause. The controller must remain running. This differs from the +source-seeded method below. + +### Planned depth pause + +Select the full target (for example, `levels: [1, 2, 3]`) and set +`mode.pauseAfterDepth: 2` with progressive dependency work. Each eligible +attempt waits after its L2 work, before the L3 request. Use enough parallelism +for the whole cohort: waiting attempts retain their processes and resource leases. + +```sh +campaign pause-status +campaign continue-depth +``` + +The release command waits for the cohort boundary: every attempt must be waiting +or terminal. Failed attempts stay in the cohort. Releasing the boundary does not +require 100% completion, grant repairs, restart earlier work, or change +eligibility. The existing progression rules determine which L3 features can start. + +This keeps the same process, accepted source, live database, progression history, +model configuration, and cumulative cost and repair budgets. Source and +progression changes during the hold cause an error. `depth-pause.json` records +each hold and `depth-release.json` records the cohort release. Working-time +allowance excludes the planned wait; total wall duration and paused duration +remain in the evidence. Cancellation still works. + +A controller loss is an interruption. `continue-depth` refuses a dead owner; it +cannot restore the database or agent session after controller shutdown. Keep the +full campaign directory for review; the partial research export omits the control +receipts. + +Database timers and external services keep running during a hold. A pause cannot +turn an already-started L2-only campaign into a predeclared L3 study; see the +[study method](../docs/study-method.md#measures-and-analysis) before comparing +staged and continuous attempts. + +### Source-seeded continuation + +Use a separate source-seeded campaign to continue a completed dependency campaign. +For example, prepare an L3 campaign from its passed L2 source without starting work: + +```sh +campaign extend --from --depth 2 --out --prepare-only +``` + +Preparation copies and verifies each source checkpoint and records its parent. +It makes no model calls and starts no attempts. After review, start the prepared +campaign with `campaign run --out `. +Without `--prepare-only`, `extend` prepares and starts the campaign immediately. + +Every matching parent attempt must be complete and pass the chosen depth. +The target must use progressive dependency work and include that depth plus a +higher depth. Stack, model, repetition, guidance, and repair condition must match. +Earlier levels are regraded without model work before any upgrade. If validation +fails, that attempt stops before higher-level work. + +This preserves source, not the agent session or database runtime. The new campaign +has its own time, cost, and repair budgets. Its reported cost excludes parent work; +reports identify the parent and label the result as a seeded continuation. Add +the parent cost when measuring the full path. Previously taught repairs remain in +the source, so this cannot turn a repaired app into an unaided first-build result. +Other target-definition changes require a separate comparison interpretation; this +is not evidence of an uninterrupted run under unchanged conditions. + +## Model-free trials and qualification + +`campaign trial` accepts only registered non-billable adapters and zero pricing. +It validates orchestration but does not produce comparative model data. + +Check qualification requirements without starting work: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm controller qualification status --track ecommerce --level --recipe +``` + +Run only evidence required by that exact status. Do not repeat reference, +mutation, or null work when its bound inputs have not changed. See the +[reference app guide](../reference-apps/README.md) and +[grader guide](../grader/README.md) for qualification rules. +Use the recipe from the compiled plan; sequential levels and dependency depths +can select different checks. Passing source tests or one reference trial does +not qualify the full scope. Pending qualification permits provisional campaigns, +but blocks verified comparison claims. + +## Dashboard + +Start the optional dashboard: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml --profile dashboard up -d dashboard +``` + +Open `http://127.0.0.1:7331`. The dashboard reads the same campaign state as the +CLI. Reading results does not require provider credentials. Run controls launch +the Compose controller and check provider configuration at launch. + +Run controls work directly in the local browser. There is no dashboard password +to retrieve. Same-origin and browser-token checks protect control requests. +The dashboard is for a trusted local machine; do not expose it on a shared network. +Model credentials are configured separately. + +See [dashboard/README.md](../dashboard/README.md). + +## Results and cleanup + +Results remain in the `stack-bench-state` Docker volume after the controller exits. +Verify and copy the complete campaign package before deleting the runner. + +For example, export the completed `campaign-001` to the host. Run from +`tools/stack-bench`; first create an empty local `results` directory if needed. +These commands copy evidence and remove only the temporary transfer container: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm controller campaign report campaigns/campaign-001 +docker create --name stack-bench-result-transfer --mount type=volume,source=stack-bench-state,target=/state,readonly stack-bench-controller:local --help +docker cp stack-bench-result-transfer:/state/results/campaigns/campaign-001 results/campaign-001 +docker rm stack-bench-result-transfer +``` + +Open `results/campaign-001/report/report.html` in a browser, or use the dashboard +to inspect checks, screenshots, logs, and cost evidence. The copied files remain +available when Docker is stopped. + +To prepare a smaller research pack, use +`campaign export campaigns/campaign-001 --out exports/campaign-001` in the +controller. The destination parent must already exist; the destination itself +must be new and outside the campaign directory. The export contains the report, +attempt/execution CSV tables, and indexed public artifacts with verified hashes. +It is a partial copy: source, transcripts, media, and external evidence are +omitted, so links to those files do not work offline. Review free text before +sharing. Keep the complete original campaign as the durable internal archive. + +A run removes only resources whose private ownership evidence still matches. +If cleanup cannot be proved, it preserves the evidence and quarantines the run. +Follow [RECOVERY.md](RECOVERY.md). Do not delete same-name resources or clear the +shared state root by guesswork. + +Workspace cleanup requires the owned build container to remain running until +the controller stops its application processes and restores directory permissions. +Normal run completion then removes the temporary work directory. Early aborts +and interruptions retain that directory for inspection, with controller access +restored when handback succeeds. Preserve needed files before an operator removes +the exact retained directory. The controller does not sweep retained work. +If that container exits, runs out of memory, or is removed before handback, cleanup +retains the private lease and reports the failure. Preserve the result package +and private recovery state. For a stopped container, diagnose the exit and restore +that exact container before retrying authenticated recovery. A container removed +before handback requires manual workspace ownership repair; automatic recovery +continues to refuse because it cannot prove the handback. No background sweep +repairs this condition. diff --git a/tools/stack-bench/appliance/RECOVERY.md b/tools/stack-bench/appliance/RECOVERY.md new file mode 100644 index 00000000000..92813fd60c6 --- /dev/null +++ b/tools/stack-bench/appliance/RECOVERY.md @@ -0,0 +1,91 @@ +# Interruption and recovery + +Recovery here means authenticated cleanup and state reconciliation. It does not +restore a database snapshot or restart an interrupted agent session. A planned +[depth pause](README.md#planned-depth-pause) retains live processes and +requires the original controller to stay running. After controller loss, +`continue-depth` refuses release and keeps the pause evidence unchanged. + +Stack Bench never guesses that a container, listener, lock, database, or data +directory is safe to delete. Normal teardown authenticates the run's private +lease, compares exact owned container and network IDs, and releases only locks +whose owner record still matches that lease. + +Paths below are inside the Docker state volume. Replace `` with +the exact `STACK_BENCH_STATE_ROOT` value written by setup in `operator.env`. +Run Compose commands from `tools/stack-bench`. The host does not need that +Linux directory. Compose mounts the state volume at the recorded path. + +Every appliance run keeps two different records: + +- `results/.../recovery.json` is public, contains no ownership token, and says + whether cleanup is `clean`, intentionally `retained`, or `quarantined`; +- `/controller-home/supervisor/.json` is private recovery authority for a standalone run. It + contains the lease token and must remain readable only by the appliance + operator. Normal cleanup deletes it. Refused cleanup deliberately preserves + it. + +Campaign supervisor records are under that campaign's `.private` directory. +For an interrupted campaign, first use the campaign recovery path: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm controller \ + campaign reconcile plans/campaign.json --out campaigns/campaign-001 +``` + +Use the campaign's original plan and output directory. Reconciliation checks +private child authority before it releases reservations or changes campaign +state. The direct commands below are for a specific retained supervisor or +lease path reported by the run. + +## If a run is interrupted + +1. Preserve the result directory and private supervisor-state file. +2. Read `recovery.json`. Do not publish an attempt whose status is + `quarantined`. +3. Do not start another run using any lock key listed in that artifact. +4. Retry authenticated cleanup from the controller: + +```sh +docker compose --env-file operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + recover /controller-home/supervisor/.json +``` + +On success the command changes `recovery.json` to `clean`, releases the exact +owned resources, and removes the private supervisor state. It is idempotent +when public lease evidence already proves that an earlier cleanup completed. + +If the parent process ended before it retained a supervisor file, recover from +the private runtime lease instead. Supply a durable output directory outside +the private runtime directory: + +```sh +docker compose --env-file operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + recover-lease /controller-home/runtime//backend-lease.json \ + --out /results/recovery/ +``` + +This path uses the same ownership token, container ID, network ID, and lock +checks. It refuses an output directory inside the runtime directory because a +successful recovery removes that directory. + +## If recovery refuses + +Refusal is the safety behavior. It means a live resource does not match the +lease or its identity could not be proven. The command leaves the private state, +lease, lock records, and public quarantine artifact intact. + +Compare the live container and network IDs with `recovery.json` and the +private lease before manual action. Never delete a same-name container, kill a +port's current listener, remove another lock, or recursively clear the shared +state root merely because its name resembles Stack Bench. Escalate with the +complete result directory and private state stored separately from public +artifacts. + +## Intentional retention + +`--retain-backend` is inspection mode, not successful cleanup. It writes +`status: "retained"` and preserves private recovery authority. No other run may +reuse the listed locks until the recovery command completes. diff --git a/tools/stack-bench/appliance/RELEASE.md b/tools/stack-bench/appliance/RELEASE.md new file mode 100644 index 00000000000..22d13f0c243 --- /dev/null +++ b/tools/stack-bench/appliance/RELEASE.md @@ -0,0 +1,125 @@ +# Release assembly and verification + +Stack Bench uses two deliberately different release states. + +- A `candidate` has exact image digests, checksummed files, and digest-bound + SPDX SBOMs. It is useful for inspecting and testing a proposed bundle, but it + is unsigned and cannot be called qualified. +- A `qualified` release adds a bundled public key, a detached Sigstore bundle + covering `release.json`, and registry signatures for every image. Verification + must use a public key obtained outside the release bundle. + +Schema v2 is the only accepted release format. + +## Build from the delivered branch + +Use a clean normal clone. The Docker build exports canonical Git content and +checks the checkout with the existing release-source and binary-source identity +code. It builds the controller, SDK, and native binaries without host Node or +Rust and without ignored binary files: + +```sh +docker build --platform linux/amd64 -t stack-bench-build:local tools/stack-bench/container +docker build --platform linux/amd64 -f tools/stack-bench/appliance/Controller.Dockerfile -t stack-bench-controller:local . +``` + +The controller contains `/opt/stack-bench/source-identity.json` and generated +`container/spacetimedb-binaries.json`. These bind the clean source revision and +native binary checksums. Setup resolves local image tags to immutable content +IDs before use. Rebuilding a tag does not change an already prepared run. + +The optional `container/build-linux-cli.sh` exports binaries through the same +Dockerfile's `binary-export` target. It is for maintainers who need loose files; +it is not a prerequisite for building the appliance. + +A branch build is a local candidate. It does not need registry publication, +Cosign, SBOM assembly, or a qualified release manifest to run provisional data. +The signed distribution path below remains a separate publication gate. + +## Build a candidate + +Publish the first-party images, resolve every first- and third-party image to an +exact single-platform `linux/amd64` manifest reference, then generate one SPDX +SBOM for each exact reference. Do not use a multi-architecture index digest: +Docker Scout correctly reports the selected child-manifest digest, so an index +digest cannot satisfy the one-image/one-SBOM identity contract. + +```sh +node dist/src/releases/release-bundle.js sbom registry.example/controller@sha256:DIGEST \ + --output bundle/sbom/controller.spdx.json +``` + +The command uses registry resolution, refuses mutable references and existing +output, and checks that Docker Scout's SPDX 2.3 document contains the requested +image digest. A successful tool exit without that digest binding is rejected. + +Create a strict release specification with `state: "candidate"`, +`signing: null`, and `files` entries containing only `path` and `role`. Place +every input below the bundle root, then materialize immutable size and SHA-256 +metadata: + +```sh +node dist/src/releases/release-bundle.js assemble release-spec.json \ + --root bundle --output bundle/release.json +node dist/src/releases/release-manifest.js verify bundle/release.json --root bundle +``` + +Candidate verification reports `candidate-file-integrity`. It validates all +declared files and each declared image-to-SBOM digest binding, including Convex when present. Candidate manifests +must use `signing: null` and cannot include a public signing key. + +## Sign and qualify + +Signing keys are external CI inputs. Never copy a private key, registry token, +or signing password into the source tree, image, bundle, Compose environment, +or command transcript. Sign each exact registry image with Cosign. The +authoritative image-signature evidence stays attached to the registry object +and is checked directly during verification; the release does not preserve a +redundant unverified export. Add the public half of the signing key as +`signing/cosign.pub` with the `public-key` role. + +Change the specification to `state: "qualified"` and declare: + +```json +{ + "signing": { + "scheme": "cosign-public-key-v1", + "publicKeyPath": "signing/cosign.pub", + "manifestBundlePath": "signing/release-manifest.sigstore.json" + } +} +``` + +Assemble `release.json` only after all other evidence exists, then sign that +exact file with a detached Cosign bundle: + +```sh +cosign sign-blob --yes --key "$COSIGN_KEY" \ + --bundle bundle/signing/release-manifest.sigstore.json bundle/release.json +``` + +The detached bundle is intentionally not checksummed by `release.json`: a file +cannot contain the hash of its own signature. Cosign authenticates it instead. + +Verify with the trusted public key copied to a path outside the downloaded +bundle: + +```sh +node dist/src/releases/release-manifest.js verify bundle/release.json --root bundle \ + --trusted-key /operator/trust/stack-bench-cosign.pub +``` + +Qualified verification refuses an absent or bundle-local trust key, requires +it to equal the public key bound by the signed manifest, verifies the detached +manifest signature, and runs `cosign verify` against every exact registry image +reference. A failed or unavailable Cosign invocation is a failed release; there +is no downgrade to candidate verification. The controller image includes +checksum-pinned Cosign 3.1.3 so this command is available in the delivered +appliance rather than depending on an untracked host installation. + +## Trust distribution + +The release bundle cannot establish trust in its own key. Publish the expected +public key and its SHA-256 fingerprint through a separately controlled channel. +The operator must compare that fingerprint before verification. Key rotation +requires a new release and an explicit trust-distribution update. diff --git a/tools/stack-bench/appliance/campaign.demo.json b/tools/stack-bench/appliance/campaign.demo.json new file mode 100644 index 00000000000..d98f7a79840 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.demo.json @@ -0,0 +1,150 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "demo", + "version": "1.0.0", + "state": "draft", + "title": "Model-free three-stack demo: selected L1 checks", + "track": "ecommerce", + "mode": { + "id": "sequential" + }, + "repair": { + "selection": "batch", + "budget": { + "total": 0 + } + }, + "levels": [ + 1 + ], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.transactional-integrity.unique-review.6b", + "ecommerce.spec.access-control.warehouse-area-boundary.7a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.stock-limit.3d" + ] + } + ] + }, + "stacks": [ + { + "id": "spacetime", + "adapterVersion": "1.4.0" + }, + { + "id": "postgres", + "adapterVersion": "1.6.0" + }, + { + "id": "mongodb", + "adapterVersion": "1.5.0" + } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "product-request", + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 1, + "parallelism": 3, + "ordering": { + "method": "balanced-rotation", + "seed": "demo-v1" + }, + "budgets": { + "attemptTimeoutMinutes": 20, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded", + "provider_failure" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-12T00:00:00Z", + "source": "reference fixture adapter makes no billable provider calls", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "checkCompletionRate", + "secondaryMetrics": [ + "totalCostUsd", + "totalTokens", + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json new file mode 100644 index 00000000000..48e1ae4e3b5 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json @@ -0,0 +1,95 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "ecommerce-progression-reference", + "version": "2.0.1", + "state": "draft", + "title": "Ecommerce progression reference pilot", + "track": "ecommerce", + "mode": { "id": "dependency", "workSelection": "progressive" }, + "repair": { "selection": "feature", "budget": { "total": 0 } }, + "levels": [1, 2, 3, 4, 5, 6], + "featureCatalog": "progression/ecommerce.json", + "selection": { + "levels": [ + { "level": 1, "recipe": "ecommerce.progression-catalog" }, + { "level": 2, "recipe": "ecommerce.progression-catalog" }, + { "level": 3, "recipe": "ecommerce.progression-catalog" }, + { "level": 4, "recipe": "ecommerce.progression-catalog" }, + { "level": 5, "recipe": "ecommerce.progression-catalog" }, + { "level": 6, "recipe": "ecommerce.progression-catalog" } + ] + }, + "stacks": [ + { "id": "mongodb", "adapterVersion": "1.5.0" }, + { "id": "postgres", "adapterVersion": "1.6.0" }, + { "id": "spacetime", "adapterVersion": "1.4.0" } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "reference-pilot", + "guidanceProfile": "neutral", + "repairPolicy": "scored-only" + } + ], + "repetitions": 1, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "ecommerce-progression-reference-1" + }, + "budgets": { + "attemptTimeoutMinutes": 180, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-25T00:00:00.000Z", + "source": "Reference fixtures make no provider calls.", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "finalScoreRate", + "secondaryMetrics": [ + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.example.json b/tools/stack-bench/appliance/campaign.example.json new file mode 100644 index 00000000000..29bed19f541 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.example.json @@ -0,0 +1,140 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "ecommerce-l1-reference-check", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce L1 model-free appliance check", + "track": "ecommerce", + "mode": { + "id": "sequential" + }, + "repair": { + "selection": "batch", + "budget": { + "total": 0 + } + }, + "levels": [ + 1 + ], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { + "id": "spacetime", + "adapterVersion": "1.4.0" + }, + { + "id": "postgres", + "adapterVersion": "1.6.0" + }, + { + "id": "mongodb", + "adapterVersion": "1.5.0" + } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "product-request", + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 1, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "replace-before-measurement" + }, + "budgets": { + "attemptTimeoutMinutes": 240, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 1, + "retryOn": [ + "provider_failure" + ], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-12T00:00:00.000Z", + "source": "reference fixture adapter makes no billable provider calls", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "firstBuildScoreRate", + "secondaryMetrics": [ + "finalScoreRate", + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.paid-l1-l3.json b/tools/stack-bench/appliance/campaign.paid-l1-l3.json new file mode 100644 index 00000000000..bb06079b529 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.paid-l1-l3.json @@ -0,0 +1,216 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "paid-l1-l3-pilot", + "version": "1.0.0", + "state": "draft", + "title": "Three-stack L1-L3 progression pilot", + "track": "ecommerce", + "mode": { + "id": "sequential" + }, + "repair": { + "selection": "batch", + "budget": { + "total": 6 + } + }, + "levels": [ + 1, + 2, + 3 + ], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + }, + { + "level": 2, + "recipe": "ecommerce.sequential-l2", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin", + "ecommerce.inventory-operations-features", + "ecommerce.operations-access-features", + "ecommerce.returns-pricing-features" + ], + "checks": [] + }, + { + "level": 3, + "recipe": "ecommerce.sequential-l3", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin", + "ecommerce.inventory-operations-features", + "ecommerce.operations-access-features", + "ecommerce.returns-pricing-features", + "ecommerce.l3.reservations-features", + "ecommerce.l3.scheduled-restocks-features", + "ecommerce.l3.order-delivery-features", + "ecommerce.l3.cart-expiration-features" + ], + "checks": [] + } + ] + }, + "stacks": [ + { + "id": "spacetime", + "adapterVersion": "1.4.0" + }, + { + "id": "postgres", + "adapterVersion": "1.6.0" + }, + { + "id": "mongodb", + "adapterVersion": "1.5.0" + } + ], + "agents": [ + { + "adapter": "claude-code", + "adapterVersion": "1.17.2", + "model": "claude-sonnet-5" + } + ], + "conditions": [ + { + "id": "product-request", + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + }, + { + "level": 2, + "requested": [], + "expected": [ + "ecommerce.inventory-operations-specifications", + "ecommerce.operations-access-specifications", + "ecommerce.returns-pricing-specifications", + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + }, + { + "level": 3, + "requested": [], + "expected": [ + "ecommerce.inventory-operations-specifications", + "ecommerce.l3.deferred-access-specifications", + "ecommerce.l3.deferred-durability-specifications", + "ecommerce.l3.deferred-integrity-specifications", + "ecommerce.l3.server-time-specifications", + "ecommerce.operations-access-specifications", + "ecommerce.returns-pricing-specifications", + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 1, + "parallelism": 3, + "ordering": { + "method": "balanced-rotation", + "seed": "paid-l1-l3-pilot-v1" + }, + "budgets": { + "attemptTimeoutMinutes": 120, + "maxCostUsdPerAttempt": 30 + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded", + "provider_failure" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-09-05T03:44:22.451Z", + "source": "https://platform.claude.com/docs/en/about-claude/pricing — verified current Sonnet 5 standard rates; subscription usage is normalized API-equivalent token cost, not an invoice charge", + "models": { + "claude-sonnet-5": { + "input": 2, + "output": 10, + "cacheWrite5m": 2.5, + "cacheWrite1h": 4, + "cacheRead": 0.2 + } + } + }, + "analysis": { + "primaryMetric": "checkCompletionRate", + "secondaryMetrics": [ + "totalCostUsd", + "totalTokens", + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.paid-l1.json b/tools/stack-bench/appliance/campaign.paid-l1.json new file mode 100644 index 00000000000..c1afb4e8165 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.paid-l1.json @@ -0,0 +1,140 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "paid-l1-demo", + "version": "1.0.0", + "state": "draft", + "title": "Three-stack L1 demo", + "track": "ecommerce", + "mode": { + "id": "sequential" + }, + "repair": { + "selection": "batch", + "budget": { + "total": 0 + } + }, + "levels": [ + 1 + ], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { + "id": "spacetime", + "adapterVersion": "1.4.0" + }, + { + "id": "postgres", + "adapterVersion": "1.6.0" + }, + { + "id": "mongodb", + "adapterVersion": "1.5.0" + } + ], + "agents": [ + { + "adapter": "claude-code", + "adapterVersion": "1.17.2", + "model": "claude-sonnet-5" + } + ], + "conditions": [ + { + "id": "product-request", + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 1, + "parallelism": 3, + "ordering": { + "method": "balanced-rotation", + "seed": "paid-l1-demo-v1" + }, + "budgets": { + "attemptTimeoutMinutes": 60, + "maxCostUsdPerAttempt": 10 + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded", + "provider_failure" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-09-05T03:44:22.451Z", + "source": "https://platform.claude.com/docs/en/about-claude/pricing — verified current Sonnet 5 standard rates; subscription usage is normalized API-equivalent token cost, not an invoice charge", + "models": { + "claude-sonnet-5": { + "input": 2, + "output": 10, + "cacheWrite5m": 2.5, + "cacheWrite1h": 4, + "cacheRead": 0.2 + } + } + }, + "analysis": { + "primaryMetric": "checkCompletionRate", + "secondaryMetrics": [ + "totalCostUsd", + "totalTokens", + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.product-brief-reference.json b/tools/stack-bench/appliance/campaign.product-brief-reference.json new file mode 100644 index 00000000000..a8133b1feaa --- /dev/null +++ b/tools/stack-bench/appliance/campaign.product-brief-reference.json @@ -0,0 +1,115 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "ecommerce-l1-product-brief-reference", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce L1 product brief and quality validation", + "track": "ecommerce", + "mode": { "id": "sequential" }, + "repair": { "selection": "batch", "budget": { "total": 0 } }, + "levels": [1], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { "id": "spacetime", "adapterVersion": "1.4.0" }, + { "id": "postgres", "adapterVersion": "1.6.0" }, + { "id": "mongodb", "adapterVersion": "1.5.0" } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "product-brief-quality", + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 2, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "product-brief-quality-reference-1" + }, + "budgets": { + "attemptTimeoutMinutes": 60, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 1, + "retryOn": ["harness_failure", "inconclusive"], + "excludeFromAnalysis": ["contaminated", "harness_failure", "inconclusive", "ungraded"] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-16T00:00:00.000Z", + "source": "reference fixture adapter makes no billable provider calls", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "firstBuildScoreRate", + "secondaryMetrics": [ + "finalScoreRate", + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/controller.ts b/tools/stack-bench/appliance/controller.ts new file mode 100644 index 00000000000..8a5d9ab985d --- /dev/null +++ b/tools/stack-bench/appliance/controller.ts @@ -0,0 +1,242 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { parsePreflightArgs } from '../commands/preflight-cli.js'; +import { parseBenchArguments } from '../commands/bench-arguments.js'; +import { AGENT_ADAPTER_REGISTRY } from '../src/agents/agent-adapters.js'; +import { stateVolumeCommand } from './state-volume.js'; + +const RUNTIME_ROOT = join(STACK_BENCH_ROOT, 'dist'); + +const COMMANDS = Object.freeze({ + 'demo': [join(RUNTIME_ROOT, 'appliance', 'demo.js')], + 'init-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'init'], + 'verify-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'verify'], + 'preflight': [join(RUNTIME_ROOT, 'commands', 'preflight.js')], + 'qualify-reference': [join(RUNTIME_ROOT, 'src', 'references', 'reference-live.js')], + 'qualify-null': [join(RUNTIME_ROOT, 'commands', 'null-control.js')], + 'qualification': [join(RUNTIME_ROOT, 'commands', 'qualification-cli.js')], + 'pack-budget': [join(RUNTIME_ROOT, 'commands', 'pack-budget.js')], + 'job': [join(RUNTIME_ROOT, 'commands', 'job-cli.js')], + 'campaign': [join(RUNTIME_ROOT, 'commands', 'campaign-cli.js')], + 'dashboard': [join(RUNTIME_ROOT, 'dashboard', 'dashboard-server.js')], + 'repair': [join(RUNTIME_ROOT, 'commands', 'repair-cli.js')], + 'run': [join(RUNTIME_ROOT, 'commands', 'bench.js')], + 'verify-release': [join(RUNTIME_ROOT, 'src', 'releases', 'release-manifest.js'), 'verify'], + 'recover': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover'], + 'recover-lease': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover-lease'], +} satisfies Record); + +const COMMANDS_REQUIRING_AGENT_AUTH = new Set(['preflight', 'run']); + +export function controllerCommandRequiresAgentAuth(command: string | undefined, + args: string[] = []): boolean { + if (command === 'job' && ['prepare', 'start', 'work', 'worker'].includes(args[0] ?? '')) return true; + if (command === 'run' && args.some(value => value === '--grade-from' || value.startsWith('--grade-from='))) { + return !parseBenchArguments([process.execPath, 'bench', ...args]).gradeFrom; + } + if (command === 'preflight' && args.length) { + const request = parsePreflightArgs([process.execPath, 'preflight', ...args]); + return AGENT_ADAPTER_REGISTRY.get(request.agentAdapter).costLimit !== 'non-billable'; + } + if (command && COMMANDS_REQUIRING_AGENT_AUTH.has(command)) return true; + return command === 'campaign' && ['run', 'resume', 'extend'].includes(args[0] ?? ''); +} + +export function controllerRuntimeEnvironment(source: NodeJS.ProcessEnv = process.env, + resolveImage = resolveContainerImage): NodeJS.ProcessEnv { + if (!source.STACK_BENCH_CONTROLLER_IMAGE) { + throw new Error('STACK_BENCH_CONTROLLER_IMAGE is required for runtime work'); + } + return { ...source, STACK_BENCH_CONTROLLER_IMAGE_ID: + resolveImage(source.STACK_BENCH_CONTROLLER_IMAGE).id }; +} + +export function controllerRuntimeCommand(args: string[], source: NodeJS.ProcessEnv = process.env) { + if (!source.STACK_BENCH_COMPOSE_FILE || !source.STACK_BENCH_STATE_ROOT + || !source.STACK_BENCH_CONTROLLER_IMAGE || !(source.STACK_BENCH_BUILD_IMAGE ?? source.STACK_BENCH_IMAGE)) { + throw new Error('controller launch requires the setup environment and appliance Compose file'); + } + const ownership = randomUUID(); + const containerName = `stack-bench-controller-${ownership}`; + const ownershipLabel = `io.spacetimedb.stack-bench.controller-owner=${ownership}`; + const env: NodeJS.ProcessEnv = { ...controllerChildEnvironment(source, { requireAgentAuth: false }), + STACK_BENCH_BUILD_IMAGE: source.STACK_BENCH_BUILD_IMAGE ?? source.STACK_BENCH_IMAGE }; + return { executable: 'docker', containerName, ownershipLabel, + args: ['compose', '-f', source.STACK_BENCH_COMPOSE_FILE, 'run', '--rm', '--no-deps', + '--name', containerName, '--label', ownershipLabel, 'controller', ...args], + env }; +} + +export interface ResolvedControllerCommand { + executable: string; + args: string[]; +} + +export function resolveControllerCommand(argv: string[]): ResolvedControllerCommand | null { + const [command, ...rest] = argv; + if (!command || command === '--help' || command === 'help') return null; + if (!Object.hasOwn(COMMANDS, command)) { + throw new Error(`unknown controller command ${JSON.stringify(command)}`); + } + return { executable: process.execPath, + args: [...COMMANDS[command as keyof typeof COMMANDS], ...rest] }; +} + +export function controllerChildEnvironment(source: NodeJS.ProcessEnv = process.env, + { requireAgentAuth = true }: { requireAgentAuth?: boolean } = {}): NodeJS.ProcessEnv { + const env = { ...source }; + const modes: Record = { + 'subscription-token': ['STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE', 'CLAUDE_CODE_OAUTH_TOKEN_FILE'], + 'api-key': ['STACK_BENCH_ANTHROPIC_API_KEY_FILE', 'ANTHROPIC_API_KEY_FILE'], + 'openrouter-api-key': ['STACK_BENCH_OPENROUTER_API_KEY_FILE', 'OPENROUTER_API_KEY_FILE'], + 'openai-api-key': ['STACK_BENCH_OPENAI_API_KEY_FILE', 'OPENAI_API_KEY_FILE'], + 'openai-account': ['STACK_BENCH_CODEX_AUTH_FILE', 'CODEX_AUTH_FILE'], + }; + // Named jobs select per attempt. Keep the legacy default available without + // clearing credentials belonging to other providers. + if (source.STACK_BENCH_CREDENTIAL_PROFILES_FILE) { + const selected = modes[source.STACK_BENCH_AGENT_AUTH ?? 'subscription-token']; + if (selected && source[selected[0]]?.trim()) env[selected[1]] = source[selected[0]]!.trim(); + return env; + } + for (const [, variable] of Object.values(modes)) { + delete env[variable]; + delete env[variable.replace(/_FILE$/, '')]; + } + delete env.STACK_BENCH_AGENT_API_KEY; + if (!requireAgentAuth) return env; + const mode = source.STACK_BENCH_AGENT_AUTH ?? 'subscription-token'; + const selected = Object.hasOwn(modes, mode) ? modes[mode] : undefined; + if (!selected) throw new Error(`STACK_BENCH_AGENT_AUTH must be ${Object.keys(modes).join(' or ')}`); + const [sourceName, variable] = selected; + const path = source[sourceName]?.trim(); + if (!path) throw new Error(`${mode} auth requires ${sourceName}`); + env[variable] = path; + return env; +} + +interface SignalChild { + kill(signal: NodeJS.Signals): unknown; +} + +interface SignalSource { + on(signal: NodeJS.Signals, listener: () => void): unknown; + off(signal: NodeJS.Signals, listener: () => void): unknown; +} + +export function forwardControllerSignals(child: SignalChild, + source: SignalSource = process): () => void { + const signals: NodeJS.Signals[] = ['SIGINT', 'SIGTERM']; + const listeners = new Map void>(signals.map(signal => + [signal, () => { child.kill(signal); }])); + for (const [signal, listener] of listeners) source.on(signal, listener); + return () => { + for (const [signal, listener] of listeners) source.off(signal, listener); + }; +} + +function help(): void { + process.stdout.write('Stack Bench controller\n' + + '\n' + + 'Docker setup\n' + + ' setup prepare the state volume and print operator.env\n' + + ' set-secret read one secret from stdin into the volume mounted at /state\n' + + '\n' + + 'A campaign compares stacks by building the same product on each. Point\n' + + 'commands at the durable plans/ and campaigns/ directories.\n' + + '\n' + + 'Run a campaign\n' + + ' job options list workload and agent choices\n' + + ' job prepare review selections without running models\n' + + ' job start --host start a reviewed run\n' + + ' job submit submit an idempotent execution job\n' + + ' job work --host claim and execute one submitted job\n' + + ' job worker --host --concurrency dispatch queued jobs automatically\n' + + ' job list|status |cancel inspect or cancel submitted work\n' + + ' preflight --backend --track --levels \n' + + ' verify the runner without creating an attempt\n' + + ' campaign validate compile a plan file and report what is wrong with it\n' + + ' campaign show print the compiled plan\n' + + ' campaign trial --out run the plan with a model-free agent\n' + + ' campaign run --out run the plan; run it again on the same to continue\n' + + ' campaign resume --out continue an interrupted dependency attempt from its saved state\n' + + ' campaign extend --from --depth --out continue a finished campaign deeper\n' + + ' campaign stop stop owned active work and retain its evidence\n' + + ' campaign pause-status inspect a planned between-depth hold\n' + + ' campaign continue-depth release the cohort at its planned depth boundary\n' + + ' campaign status [--full] what the campaign is doing now, from its saved state\n' + + ' campaign inspect every attempt, level, and check with its evidence\n' + + ' campaign report write the JSON and HTML report\n' + + ' campaign audit check a finished reference campaign against its promises\n' + + ' campaign grant-repairs --attempt --level --repairs add repair budget\n' + + ' campaign grant-time --attempt --grant-id --minutes add time\n' + + ' campaign reconcile --out clean up after an interruption and prove it\n' + + ' campaign modes list the campaign modes this controller knows\n' + + ' dashboard [--port N] serve the local dashboard\n' + + '\n' + + 'One attempt outside a campaign\n' + + ' run --backend --track --levels --out [...] build and grade one attempt\n' + + ' run --grade-from --grade-level --check --out replay saved dependency source without model calls\n' + + ' repair status --level can a failed level continue?\n' + + ' repair grant --level --repairs add one repair budget\n' + + '\n' + + 'Qualify the grader\n' + + ' qualify-reference --track --level grade the hand-built reference app, or its mutations\n' + + ' --mutation-workers N split the mutation run across 1 to 8 isolated workers\n' + + ' qualify-null --track --level prove an empty app scores nothing\n' + + ' qualification status --track --level which grading evidence is still missing\n' + + ' pack-budget recommend --track --level --recipe --evidence derive pack limits from reference evidence\n' + + '\n' + + 'Recover and verify\n' + + ' recover retry cleanup for an interrupted attempt, or keep its quarantine\n' + + ' recover-lease --out recover when the attempt state was not kept\n' + + ' verify-release verify a candidate or signed release\n' + + ' init-deps | verify-deps create or verify the release dependency volume\n'); +} + +interface ChildOutcome { + code: number | null; + signal: NodeJS.Signals | null; +} + +async function main(argv: string[]): Promise { + const command = argv[2]; + if (command === 'setup' || command === 'set-secret') { + stateVolumeCommand(command, argv.slice(3)); + return; + } + const resolved = resolveControllerCommand(argv.slice(2)); + if (!resolved) { help(); return; } + let env = controllerChildEnvironment(process.env, + { requireAgentAuth: controllerCommandRequiresAgentAuth(command, argv.slice(3)) }); + const runtime = ['preflight', 'run', 'qualify-reference', 'qualify-null', 'recover', 'recover-lease'] + .includes(command ?? '') || (command === 'campaign' + && ['run', 'trial', 'resume', 'extend', 'reconcile'].includes(argv[3] ?? '')) + || (command === 'job' && ['start', 'work', 'worker'].includes(argv[3] ?? '')); + if (runtime) env = controllerRuntimeEnvironment(env); + const child = spawn(resolved.executable, resolved.args, { stdio: 'inherit', env }); + const stopForwardingSignals = forwardControllerSignals(child); + let outcome: ChildOutcome; + try { + outcome = await new Promise((resolveExit, reject) => { + child.once('error', reject); + child.once('exit', (code, signal) => { resolveExit({ code, signal }); }); + }); + } finally { stopForwardingSignals(); } + if (outcome.signal) process.kill(process.pid, outcome.signal); + process.exitCode = outcome.code ?? 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main(process.argv).catch((error: unknown) => { + console.error(`stack-bench-controller: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/appliance/demo.compose.yaml b/tools/stack-bench/appliance/demo.compose.yaml new file mode 100644 index 00000000000..06267966444 --- /dev/null +++ b/tools/stack-bench/appliance/demo.compose.yaml @@ -0,0 +1,28 @@ +name: stack-bench-demo + +services: + build-image: + image: stack-bench-build:local + platform: linux/amd64 + build: + context: ../container + dockerfile: Dockerfile + entrypoint: ["/bin/true"] + demo: + image: stack-bench-controller:local + platform: linux/amd64 + build: + context: ../../.. + dockerfile: tools/stack-bench/appliance/Controller.Dockerfile + depends_on: + build-image: + condition: service_completed_successfully + init: true + command: ["demo"] + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - state:/state + +volumes: + state: + name: stack-bench-state diff --git a/tools/stack-bench/appliance/demo.ts b/tools/stack-bench/appliance/demo.ts new file mode 100644 index 00000000000..8ff0902403b --- /dev/null +++ b/tools/stack-bench/appliance/demo.ts @@ -0,0 +1,66 @@ +import { spawn } from 'node:child_process'; +import { chmodSync, existsSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { forwardControllerSignals } from './controller.js'; +import { prepareStateVolume } from './state-volume.js'; + +export function demoConfiguration(setup: string, source: NodeJS.ProcessEnv = process.env) { + const prepared: NodeJS.ProcessEnv = {}; + for (const line of setup.split('\n').filter(Boolean)) { + const separator = line.indexOf('='); + if (separator < 1) throw new Error('Invalid setup environment'); + prepared[line.slice(0, separator)] = line.slice(separator + 1); + } + const digest = prepared.STACK_BENCH_CONTROLLER_IMAGE?.match(/^(?:.*@)?sha256:([a-f0-9]{64})$/)?.[1]; + if (!digest) throw new Error('Demo requires a resolved controller image digest'); + prepared.STACK_BENCH_RELEASE_DEPS_VOLUME = `stack-bench-release-deps-${digest.slice(0, 12)}`; + const output = `campaigns/demo-${digest.slice(0, 12)}`; + const compose = ['compose', '-f', join(STACK_BENCH_ROOT, 'appliance/docker-compose.yaml')]; + return { env: { ...source, ...prepared }, output, + dashboard: [...compose, '--profile', 'dashboard', 'up', '-d', 'dashboard'], + campaign: [...compose, 'run', '--rm', '-T', '--name', `stack-bench-demo-${digest.slice(0, 12)}`, + 'controller', 'campaign', 'trial', 'plans/demo.json', '--out', output], + savedEnvironment: Object.entries(prepared) + .map(([key, value]) => `${key}=${value ?? ''}`).join('\n') + '\n', + }; +} + +async function docker(args: string[], env: NodeJS.ProcessEnv): Promise { + const child = spawn('docker', args, { env, stdio: 'inherit' }); + const stopForwarding = forwardControllerSignals(child); + try { + await new Promise((resolve, reject) => { + child.once('error', reject); + child.once('exit', (code, signal) => code === 0 ? resolve() + : reject(new Error(`Demo Docker command failed (${signal ?? code})`))); + }); + } finally { stopForwarding(); } +} + +export async function demoCommand(args: string[]): Promise { + if (args.length) throw new Error('demo accepts no arguments; set image references through the environment'); + const config = demoConfiguration(prepareStateVolume()); + const environmentPath = '/state/controller-home/demo.env'; + writeFileSync(environmentPath, config.savedEnvironment, { mode: 0o600 }); + chmodSync(environmentPath, 0o600); + await docker(config.dashboard, config.env); + console.log('Stack Bench dashboard: http://127.0.0.1:7331'); + const directory = join('/state/results', config.output); + if (existsSync(join(directory, 'state.json'))) { + const { state } = readCampaignState(directory); + console.log(`Existing demo campaign: ${state.status} (${config.output})`); + if (state.status !== 'completed') throw new Error(`Existing demo needs attention: ${state.status}`); + return; + } + await docker(config.campaign, config.env); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + demoCommand(process.argv.slice(2)).catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tools/stack-bench/appliance/dependency-volume.ts b/tools/stack-bench/appliance/dependency-volume.ts new file mode 100644 index 00000000000..e6f49e55a86 --- /dev/null +++ b/tools/stack-bench/appliance/dependency-volume.ts @@ -0,0 +1,165 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import { + chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, + renameSync, rmSync, writeFileSync, +} from 'node:fs'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +const MARKER = '.stack-bench-release-deps.json'; + +export interface DependencyManifestFile { + path: string; + size: number; + mode: number; + sha256: string; +} + +export interface DependencyManifest { + schemaVersion: 1; + files: DependencyManifestFile[]; +} + +interface DependencyVerification { + manifestSha256: string; + files: number; +} + +interface DependencyInitialization extends DependencyVerification { + initialized: boolean; +} + +function sha256Bytes(bytes: string | NodeJS.ArrayBufferView): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function normalizedRelative(root: string, path: string): string { + const value = relative(root, path).split(sep).join('/'); + if (!value || value.startsWith('../') || value === '..') throw new Error(`path escapes dependency root: ${path}`); + return value; +} + +function walk(root: string, current = root): string[] { + const files: string[] = []; + for (const entry of readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const path = join(current, entry.name); + if (entry.isSymbolicLink()) throw new Error(`dependency tree cannot contain symlinks: ${normalizedRelative(root, path)}`); + if (entry.isDirectory()) files.push(...walk(root, path)); + else if (entry.isFile()) files.push(path); + else throw new Error(`dependency tree contains unsupported entry: ${normalizedRelative(root, path)}`); + } + return files; +} + +export function createDependencyManifest(root: string): DependencyManifest { + const absolute = resolve(root); + if (!existsSync(absolute) || !lstatSync(absolute).isDirectory()) { + throw new Error(`dependency source is not a directory: ${absolute}`); + } + const files = walk(absolute).map(path => { + const bytes = readFileSync(path); + return { path: normalizedRelative(absolute, path), size: bytes.length, + mode: lstatSync(path).mode & 0o777, sha256: sha256Bytes(bytes) }; + }); + if (!files.length) throw new Error('dependency source is empty'); + return { schemaVersion: 1, files }; +} + +export function manifestSha256(manifest: DependencyManifest): string { + return sha256Bytes(`${JSON.stringify(manifest)}\n`); +} + +export function verifyDependencyTree(root: string, manifest: DependencyManifest, + { allowMarker = false }: { allowMarker?: boolean } = {}): DependencyVerification { + if (!manifest || manifest.schemaVersion !== 1 || !Array.isArray(manifest.files) || !manifest.files.length) { + throw new Error('dependency manifest is invalid'); + } + const absolute = resolve(root); + const actual = createDependencyManifest(absolute); + if (allowMarker) actual.files = actual.files.filter(file => file.path !== MARKER); + if (JSON.stringify(actual.files) !== JSON.stringify(manifest.files)) { + throw new Error(`dependency tree does not match manifest ${manifestSha256(manifest)}`); + } + return { manifestSha256: manifestSha256(manifest), files: manifest.files.length }; +} + +export function initializeDependencyVolume({ source, target, manifest }: + { source: string; target: string; manifest: DependencyManifest }): DependencyInitialization { + const sourceRoot = resolve(source); + const targetRoot = resolve(target); + const verified = verifyDependencyTree(sourceRoot, manifest); + mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); + const markerPath = join(targetRoot, MARKER); + const existing = readdirSync(targetRoot); + if (existing.length) { + if (!existsSync(markerPath)) throw new Error('dependency volume is non-empty but has no release marker'); + const marker = JSON.parse(readFileSync(markerPath, 'utf8')); + if (marker.schemaVersion !== 1 || marker.manifestSha256 !== verified.manifestSha256) { + throw new Error('dependency volume belongs to a different release'); + } + verifyDependencyTree(targetRoot, manifest, { allowMarker: true }); + return { ...verified, initialized: false }; + } + + const staging = join(targetRoot, `.staging-${process.pid}`); + mkdirSync(staging, { mode: 0o700 }); + try { + for (const file of manifest.files) { + const from = join(sourceRoot, ...file.path.split('/')); + const to = join(staging, ...file.path.split('/')); + mkdirSync(dirname(to), { recursive: true }); + copyFileSync(from, to); + chmodSync(to, file.mode); + } + for (const entry of readdirSync(staging)) renameSync(join(staging, entry), join(targetRoot, entry)); + rmSync(staging, { recursive: true, force: true }); + writeFileSync(markerPath, `${JSON.stringify({ schemaVersion: 1, + manifestSha256: verified.manifestSha256 })}\n`, { flag: 'wx', mode: 0o444 }); + verifyDependencyTree(targetRoot, manifest, { allowMarker: true }); + return { ...verified, initialized: true }; + } catch (error) { + rmSync(staging, { recursive: true, force: true }); + throw error; + } +} + +function main(argv: string[]): void { + const { values, positionals } = parseArgs({ args: argv.slice(2), allowPositionals: true, + options: { + source: { type: 'string', default: '/opt/stack-bench-embedded-deps' }, + target: { type: 'string', default: '/opt/stack-bench-release-deps' }, + manifest: { type: 'string', default: '/opt/stack-bench/dependency-manifest.json' }, + out: { type: 'string' }, + } }); + const [command] = positionals; + const source = values.source; + const target = values.target; + const manifestPath = values.manifest; + if (command === 'manifest') { + const output = values.out; + if (!output) throw new Error('manifest requires --out'); + writeFileSync(resolve(output), `${JSON.stringify(createDependencyManifest(source), null, 2)}\n`, { flag: 'wx' }); + return; + } + const manifest: DependencyManifest = JSON.parse(readFileSync(resolve(manifestPath), 'utf8')); + if (command === 'init') { + process.stdout.write(`${JSON.stringify(initializeDependencyVolume({ source, target, manifest }))}\n`); + return; + } + if (command === 'verify') { + process.stdout.write(`${JSON.stringify(verifyDependencyTree(target, manifest, { allowMarker: true }))}\n`); + return; + } + throw new Error('usage: dependency-volume manifest|init|verify [options]'); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { main(process.argv); } + catch (error) { + console.error(`dependency-volume: ${error instanceof Error ? error.message : String(error)}`); + process.exit(2); + } +} diff --git a/tools/stack-bench/appliance/docker-compose.yaml b/tools/stack-bench/appliance/docker-compose.yaml new file mode 100644 index 00000000000..e32295e82fe --- /dev/null +++ b/tools/stack-bench/appliance/docker-compose.yaml @@ -0,0 +1,164 @@ +name: stack-bench-appliance + +services: + deps-init: + image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + platform: linux/amd64 + command: ["init-deps"] + read_only: true + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + volumes: + - type: volume + source: release-deps + target: /opt/stack-bench-release-deps + tmpfs: + - /tmp:size=64m,mode=1777 + + controller: + image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + platform: linux/amd64 + init: true + network_mode: host + working_dir: ${STACK_BENCH_STATE_ROOT:?run controller setup}/results + read_only: true + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + depends_on: + deps-init: + condition: service_completed_successfully + npm-cache: + condition: service_healthy + environment: + STACK_BENCH_CONTROLLER_IMAGE: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + STACK_BENCH_BUILD_IMAGE: ${STACK_BENCH_BUILD_IMAGE:?set STACK_BENCH_BUILD_IMAGE} + STACK_BENCH_STATE_ROOT: ${STACK_BENCH_STATE_ROOT:?run controller setup} + STACK_BENCH_NPM_REGISTRY: http://127.0.0.1:4873/ + STACK_BENCH_IMAGE: ${STACK_BENCH_BUILD_IMAGE:?set STACK_BENCH_BUILD_IMAGE to the manifest digest reference} + STACK_BENCH_RELEASE_MANIFEST: ${STACK_BENCH_RELEASE_MANIFEST:-} + STACK_BENCH_APPLIANCE: "1" + STACK_BENCH_RUNNER_CAPACITY: ${STACK_BENCH_RUNNER_CAPACITY:-} + STACK_BENCH_COMPOSE_FILE: /opt/stack-bench/appliance/docker-compose.yaml + STACK_BENCH_WORK_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/work + STACK_BENCH_RESULTS_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/results + STACK_BENCH_SUPERVISOR_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/controller-home/supervisor + STACK_BENCH_RUNTIME_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/controller-home/runtime + STACK_BENCH_RESOURCE_LOCK_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/controller-home/resource-locks + STACK_BENCH_RELEASE_DEPS_VOLUME: ${STACK_BENCH_RELEASE_DEPS_VOLUME:-stack-bench-release-deps} + STACK_BENCH_LINUX_CLI: /opt/stack-bench-release-deps/spacetimedb-cli + STDB_PACKAGE: /opt/stack-bench-release-deps/bindings-typescript + SPACETIME_BIN: /opt/stack-bench-release-deps/spacetimedb-cli + STACK_BENCH_CREDENTIAL_PROFILES_FILE: ${STACK_BENCH_CREDENTIAL_PROFILES_FILE:-} + STACK_BENCH_AGENT_AUTH: ${STACK_BENCH_AGENT_AUTH:-subscription-token} + STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE: ${STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE:-${STACK_BENCH_STATE_ROOT:?run controller setup}/secrets/claude_subscription_token} + STACK_BENCH_ANTHROPIC_API_KEY_FILE: ${STACK_BENCH_ANTHROPIC_API_KEY_FILE:-} + STACK_BENCH_OPENROUTER_API_KEY_FILE: ${STACK_BENCH_OPENROUTER_API_KEY_FILE:-} + STACK_BENCH_OPENAI_API_KEY_FILE: ${STACK_BENCH_OPENAI_API_KEY_FILE:-} + STACK_BENCH_CODEX_AUTH_FILE: ${STACK_BENCH_CODEX_AUTH_FILE:-} + HOME: ${STACK_BENCH_STATE_ROOT:?run controller setup}/controller-home + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + - type: volume + source: state + target: ${STACK_BENCH_STATE_ROOT:?run controller setup} + - type: volume + source: release-deps + target: /opt/stack-bench-release-deps + read_only: true + tmpfs: + - /tmp:size=1g,mode=1777 + command: ["--help"] + logging: &bounded-logs + driver: json-file + options: + max-size: "10m" + max-file: "3" + + worker: + restart: on-failure:3 + extends: + service: controller + profiles: ["worker"] + # SIGTERM stops admission and drains claimed campaigns. Explicit job cancellation stops a run. + stop_grace_period: 24h + command: ["job", "worker", "--host", "${STACK_BENCH_HOST_ID:-}", "--concurrency", "${STACK_BENCH_JOB_CONCURRENCY:-}"] + + dashboard: + extends: + service: controller + profiles: ["dashboard"] + network_mode: bridge + ports: + - "127.0.0.1:7331:7331" + command: ["dashboard", "--host", "0.0.0.0", "--port", "7331", "--allow-container-bind"] + # Pull-through npm registry cache. Coding containers install through it, so a + # package reaches the public registry once per appliance, not once per run. + npm-cache: + image: verdaccio/verdaccio:6@sha256:09b403888c8f73ba9336d7fb3464622f64ea905a64bc46a87c847387c536d4dd + platform: linux/amd64 + container_name: stack-bench-npm-cache + cpus: 1 + mem_limit: 2g + pids_limit: 128 + logging: *bounded-logs + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + ports: ["127.0.0.1:4873:4873"] + configs: + - source: npm-cache-config + target: /verdaccio/conf/config.yaml + volumes: + - npmcache:/verdaccio/storage + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:4873/-/ping"] + interval: 5s + timeout: 5s + retries: 12 + +configs: + npm-cache-config: + content: | + storage: /verdaccio/storage/data + plugins: /verdaccio/plugins + web: + enable: false + auth: + htpasswd: + file: /verdaccio/storage/htpasswd + max_users: -1 + uplinks: + npmjs: + url: https://registry.npmjs.org/ + cache: true + timeout: 60s + maxage: 10m + max_fails: 4 + fail_timeout: 2m + packages: + '@*/*': + access: $$all + publish: nobody + unpublish: nobody + proxy: npmjs + '**': + access: $$all + publish: nobody + unpublish: nobody + proxy: npmjs + server: + keepAliveTimeout: 60 + log: + type: stdout + format: pretty + level: warn + +volumes: + release-deps: + name: ${STACK_BENCH_RELEASE_DEPS_VOLUME:-stack-bench-release-deps} + state: + external: true + name: stack-bench-state + npmcache: + name: stack-bench-appliance-npmcache diff --git a/tools/stack-bench/appliance/operator.env.example b/tools/stack-bench/appliance/operator.env.example new file mode 100644 index 00000000000..4bd5966b265 --- /dev/null +++ b/tools/stack-bench/appliance/operator.env.example @@ -0,0 +1,33 @@ +# Prefer the controller setup command. It prints this file with the Docker +# volume mountpoint and immutable local image IDs already resolved. +STACK_BENCH_STATE_ROOT=/var/lib/docker/volumes/stack-bench-state/_data + +# Subscription billing is the default. Generate a dedicated long-lived Claude +# setup token, write only the token to this mode-0600 file, and never commit it. +STACK_BENCH_AGENT_AUTH=subscription-token +STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE=${STACK_BENCH_STATE_ROOT}/secrets/claude_subscription_token + +# To bill through an API key instead, set the mode to api-key and provide an +# absolute path below STACK_BENCH_STATE_ROOT containing only that key. +# STACK_BENCH_ANTHROPIC_API_KEY_FILE=${STACK_BENCH_STATE_ROOT}/secrets/anthropic_api_key + +# For a codex agent, select openai-api-key or openai-account explicitly. +# Account mode uses an unexpired Codex login access token, with no automatic refresh. +# STACK_BENCH_AGENT_AUTH=openai-api-key +# STACK_BENCH_OPENAI_API_KEY_FILE=${STACK_BENCH_STATE_ROOT}/secrets/openai_api_key +# STACK_BENCH_AGENT_AUTH=openai-account +# STACK_BENCH_CODEX_AUTH_FILE=${STACK_BENCH_STATE_ROOT}/secrets/codex_auth + +# For the openrouter adapter, store the key with set-secret openrouter_api_key. +# Freeze model, providerRoute, and maxOutputTokens in each campaign agent selection. +# STACK_BENCH_AGENT_AUTH=openrouter-api-key +# STACK_BENCH_OPENROUTER_API_KEY_FILE=${STACK_BENCH_STATE_ROOT}/secrets/openrouter_api_key + +# Setup emits immutable local sha256: values. Distributed releases use +# registry references ending in @sha256:<64 hex chars>. +STACK_BENCH_CONTROLLER_IMAGE=registry.example/stack-bench-controller@sha256:replace-with-release-digest +STACK_BENCH_BUILD_IMAGE=registry.example/stack-bench-build@sha256:replace-with-release-digest + +# Optional for internal campaigns and required for a distributed release +# campaign. The file must be below the appliance state root. +STACK_BENCH_RELEASE_MANIFEST=${STACK_BENCH_STATE_ROOT}/release/release.json diff --git a/tools/stack-bench/appliance/state-volume.ts b/tools/stack-bench/appliance/state-volume.ts new file mode 100644 index 00000000000..7c309d20e4f --- /dev/null +++ b/tools/stack-bench/appliance/state-volume.ts @@ -0,0 +1,108 @@ +import { execFileSync } from 'node:child_process'; +import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { DATABASE_IMAGES } from '../src/stacks/database-containers.js'; +import { CONVEX_BACKEND_IMAGE } from '../src/stacks/backends/convex-lifecycle.js'; + +export const STATE_VOLUME = 'stack-bench-state'; +type Docker = (args: readonly string[]) => string; +const docker: Docker = args => execFileSync('docker', [...args], { + encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: args[0] === 'pull' ? 300_000 : 60_000, +}); + +/** The controller and Docker daemon see one native Linux path on every host OS. */ +export function prepareStateVolume(env: NodeJS.ProcessEnv = process.env, run: Docker = docker): string { + if (run(['version', '--format', '{{.Server.Os}}']).trim() !== 'linux') { + throw new Error('Stack Bench requires a Docker daemon running Linux containers'); + } + const inspect = (_command: string, args: readonly string[]) => run(args); + const controller = resolveContainerImage(env.STACK_BENCH_CONTROLLER_IMAGE + ?? 'stack-bench-controller:local', inspect).id; + const build = resolveContainerImage(env.STACK_BENCH_BUILD_IMAGE + ?? 'stack-bench-build:local', inspect).id; + for (const reference of [...Object.values(DATABASE_IMAGES), CONVEX_BACKEND_IMAGE]) { + try { resolveContainerImage(reference, inspect); } + catch { + run(['pull', '--platform', 'linux/amd64', reference]); + resolveContainerImage(reference, inspect); + } + } + run(['volume', 'create', STATE_VOLUME]); + const root = run(['volume', 'inspect', '--format', '{{.Mountpoint}}', STATE_VOLUME]).trim(); + if (!/^\/[A-Za-z0-9_./-]+$/.test(root) || root.split('/').includes('..')) { + throw new Error('Docker state volume has an invalid Linux mountpoint'); + } + run(['run', '--rm', '--platform', 'linux/amd64', '--network', 'none', '--mount', + `type=volume,source=${STATE_VOLUME},target=${root}`, '--entrypoint', 'node', controller, '-e', + 'const fs=require("node:fs"),p=require("node:path");' + + 'const root=process.argv[1];' + + 'for(const name of ["work","results/plans","results/run-presets","secrets","controller-home"])' + + 'fs.mkdirSync(p.join(root,name),{recursive:true,mode:0o700});' + + 'for(const [source,name] of [["campaign.example.json","reference-check.json"],["campaign.ecommerce-progression-reference.json","ecommerce-progression.json"],["campaign.demo.json","demo.json"]]) {' + + 'const target=p.join(root,"results/plans",name);if(!fs.existsSync(target))' + + 'fs.copyFileSync(p.join("/opt/stack-bench/appliance",source),target,fs.constants.COPYFILE_EXCL);}' + + 'const demo=p.join(root,"results/plans/paid-l1.json");if(!fs.existsSync(demo)){' + + 'const plan=JSON.parse(fs.readFileSync("/opt/stack-bench/appliance/campaign.paid-l1.json","utf8"));' + + 'plan.runtime.controllerImage=process.argv[2];plan.runtime.buildImage=process.argv[3];plan.state="frozen";' + + 'fs.writeFileSync(demo,JSON.stringify(plan,null,2)+"\\n",{flag:"wx",mode:0o600});}' + + 'const paid=JSON.parse(fs.readFileSync("/opt/stack-bench/appliance/campaign.paid-l1-l3.json","utf8"));' + + 'for(const [source,id,title] of [["campaign.paid-l1-l3.json","ecommerce-sequential","Ecommerce — sequential levels"],' + + '["campaign.ecommerce-progression-reference.json","ecommerce-progressive","Ecommerce — progressive features"],' + + '["campaign.ecommerce-progression-reference.json","ecommerce-progressive-l3","Ecommerce — four stacks, progressive L1–L3"],' + + '["campaign.ecommerce-progression-reference.json","ecommerce-single-build","Ecommerce — single build"]]){' + + 'const target=p.join(root,"results/run-presets",id+".json");if(fs.existsSync(target))continue;' + + 'const d=JSON.parse(fs.readFileSync(p.join("/opt/stack-bench/appliance",source),"utf8"));' + + 'd.id=id;d.title=title;d.state="frozen";d.agents=paid.agents;d.pricing=paid.pricing;' + + 'if(id==="ecommerce-single-build")d.mode.workSelection="all-at-once";' + + 'if(id==="ecommerce-progressive-l3"){d.levels=d.levels.filter(n=>n<=3);d.selection.levels=d.selection.levels.filter(s=>s.level<=3);d.stacks.push({id:"convex",adapterVersion:"1.0.0"});}' + + 'd.runtime.controllerImage=process.argv[2];d.runtime.buildImage=process.argv[3];' + + 'd.budgets=paid.budgets;d.repair.budget={total:0};d.parallelism=d.stacks.length;' + + 'if(id==="ecommerce-progressive"||id==="ecommerce-progressive-l3"){d.budgets={attemptTimeoutMinutes:240,maxCostUsdPerAttempt:50};d.mode.retainPriorContracts=true;d.mode.unchangedFailureLimit=7;}' + + 'd.conditions=["neutral","neutral-no-sdk","neutral-dev","neutral-dev-no-sdk"].map(g=>({...d.conditions[0],id:g,guidanceProfile:g}));' + + 'fs.writeFileSync(target,JSON.stringify(d,null,2)+"\\n",{flag:"wx",mode:0o600});}', + root, controller, build]); + return [ + `STACK_BENCH_STATE_ROOT=${root}`, + `STACK_BENCH_CONTROLLER_IMAGE=${controller}`, + `STACK_BENCH_BUILD_IMAGE=${build}`, + 'STACK_BENCH_RUNNER_CAPACITY=dynamic', + 'STACK_BENCH_AGENT_AUTH=subscription-token', + `STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE=${root}/secrets/claude_subscription_token`, + `STACK_BENCH_ANTHROPIC_API_KEY_FILE=${root}/secrets/anthropic_api_key`, + `STACK_BENCH_OPENROUTER_API_KEY_FILE=${root}/secrets/openrouter_api_key`, + `STACK_BENCH_OPENAI_API_KEY_FILE=${root}/secrets/openai_api_key`, + `STACK_BENCH_CODEX_AUTH_FILE=${root}/secrets/codex_auth`, + 'STACK_BENCH_RELEASE_MANIFEST=', + '', + ].join('\n'); +} + +export function writeStateSecret(name: string | undefined, input: string, + root = '/state'): void { + if (!['claude_subscription_token', 'anthropic_api_key', 'openai_api_key', 'openrouter_api_key', 'codex_auth'].includes(name ?? '')) { + throw new Error('secret name must be claude_subscription_token, anthropic_api_key, openai_api_key, openrouter_api_key, codex_auth'); + } + let value = input.trim(); + if (name === 'codex_auth') { + try { value = JSON.stringify(JSON.parse(value)); } + catch { throw new Error('codex_auth must be valid JSON from Codex account login'); } + } + if (!value || /[\r\n]/.test(value)) { + throw new Error('secret must be one non-empty line'); + } + mkdirSync(join(root, 'secrets'), { recursive: true, mode: 0o700 }); + chmodSync(join(root, 'secrets'), 0o700); + writeFileSync(join(root, 'secrets', name!), `${value}\n`, { mode: 0o600 }); + chmodSync(join(root, 'secrets', name!), 0o600); +} + +export function stateVolumeCommand(command: string, args: string[]): void { + if (command === 'setup') { + if (args.length) throw new Error('setup accepts no arguments; configure image references through the environment'); + process.stdout.write(prepareStateVolume()); + } else { + if (args.length !== 1) throw new Error('set-secret requires exactly one secret name'); + writeStateSecret(args[0], readFileSync(0, 'utf8')); + } +} diff --git a/tools/stack-bench/backends/convex.md b/tools/stack-bench/backends/convex.md new file mode 100644 index 00000000000..ce1b8a97c1b --- /dev/null +++ b/tools/stack-bench/backends/convex.md @@ -0,0 +1,20 @@ +# Backend: Convex + +Use a React client and native Convex TypeScript functions. Put the functions and +schema in `convex/`. The application mutations described below are exported from +`convex/api.ts`. Use the same native functions for the visible controls. + +## Deployment + +The local Convex deployment is already running. The environment supplies +`CONVEX_SELF_HOSTED_URL`, `CONVEX_SELF_HOSTED_ADMIN_KEY`, and `VITE_CONVEX_URL`. +Use these exact values at startup. Do not embed or save them in source or connect +to a hosted project. Keep the deployment key out of the web client and user sessions. +The native HTTP-action origin is `http://127.0.0.1:`. + +Create `/app/start.sh`. From a clean source checkout, it must install dependencies, +deploy the functions with `npx convex dev --once --typecheck disable`, build the web +application, and start it on `` without changing source files. +Deploy functions and initialize empty data even with +`APP_WARM_START=1`; reuse current dependencies when possible. Keep existing data +and accounts during subsequent starts, upgrades, and repairs. Leave the application running. diff --git a/tools/stack-bench/backends/minimal/convex.md b/tools/stack-bench/backends/minimal/convex.md new file mode 100644 index 00000000000..9c015518cb4 --- /dev/null +++ b/tools/stack-bench/backends/minimal/convex.md @@ -0,0 +1,22 @@ +# Convex + +Use Convex for application data and backend functions. Choose the client libraries, +architecture, and project structure. + +## Connection + +The local Convex service is already running. Read its URL from +`CONVEX_SELF_HOSTED_URL` and its deployment key from `CONVEX_SELF_HOSTED_ADMIN_KEY`. +The web client uses `VITE_CONVEX_URL`. These values are supplied in the process +environment and can change between launches. Do not embed or save them in source. +Keep the deployment key on the server; it is not an end-user credential. + +Use this deployment. Do not start another Convex server or connect to a hosted project. +The native HTTP-action origin is `http://127.0.0.1:`. +Serve the complete application on ``. + +Create `/app/start.sh`. From a clean source checkout, it must install dependencies, +deploy the Convex functions, build the web application, and start the complete application. +The script must not change source files. Deploy the functions and initialize empty application data even when +`APP_WARM_START=1`; that flag only permits reusing current dependencies. Preserve +existing application data and accounts. Leave the application running when work is complete. diff --git a/tools/stack-bench/backends/minimal/mongodb.md b/tools/stack-bench/backends/minimal/mongodb.md new file mode 100644 index 00000000000..a4a7e8d3c8a --- /dev/null +++ b/tools/stack-bench/backends/minimal/mongodb.md @@ -0,0 +1,21 @@ +# MongoDB + +Use MongoDB for the application data. Choose the libraries, architecture, and +project structure. + +## Connection + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| Web application | `http://localhost:` | + +The MongoDB service is already running as a single-node replica set. Use the exact `DATABASE_URL`. Do not +start another MongoDB server, connect to another instance, or create another +database. Serve the complete application on ``. +Read `DATABASE_URL` from the process environment at startup. It can change between +launches; do not embed it in source or override it with a saved value. +Create `/app/start.sh`. From a clean source checkout, it must install +dependencies, build the complete application, and start it on ``. +The script must not change source files. Leave the application running when the +work is complete. diff --git a/tools/stack-bench/backends/minimal/postgres.md b/tools/stack-bench/backends/minimal/postgres.md new file mode 100644 index 00000000000..1a078206cf4 --- /dev/null +++ b/tools/stack-bench/backends/minimal/postgres.md @@ -0,0 +1,21 @@ +# PostgreSQL + +Use PostgreSQL for the application data. Choose the libraries, architecture, +and project structure. + +## Connection + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| Web application | `http://localhost:` | + +The PostgreSQL service is already running. Use the exact `DATABASE_URL`. Do not +start another PostgreSQL server, connect to another instance, or create another +database. Serve the complete application on ``. +Read `DATABASE_URL` from the process environment at startup. It can change between +launches; do not embed it in source or override it with a saved value. +Create `/app/start.sh`. From a clean source checkout, it must install +dependencies, build the complete application, and start it on ``. +The script must not change source files. Leave the application running when the +work is complete. diff --git a/tools/stack-bench/backends/minimal/spacetime.md b/tools/stack-bench/backends/minimal/spacetime.md new file mode 100644 index 00000000000..508dbe353b5 --- /dev/null +++ b/tools/stack-bench/backends/minimal/spacetime.md @@ -0,0 +1,28 @@ +# SpacetimeDB + +Use SpacetimeDB for the application data. Put the TypeScript module in the +required directory below. Choose the schema, libraries, architecture, and the +rest of the project structure. + +## Connection + +Use the connection settings below. + +| Setting | Value | +|---|---| +| Server URI | `` | +| Module name | `` | +| SpacetimeDB CLI | `` | +| TypeScript SDK package | `` | +| Module source directory | `/app/backend/spacetimedb` | +| Web application | `http://localhost:` | + +Publish only the named module to the exact server URI. Local publish and +development commands must use `--yes`. Do not pipe confirmation input, publish +anonymously, or use the hosted service. Create `/app/start.sh`. From a clean +source checkout, it must install dependencies, build the complete application, +and start it on ``. The script must not change source files. Leave +the application running when the work is complete. + +The included TypeScript server and client skills provide SDK guidance. +CLI `--help` is available for command syntax. diff --git a/tools/stack-bench/backends/model-free-stub.md b/tools/stack-bench/backends/model-free-stub.md new file mode 100644 index 00000000000..7a83d0e4bc2 --- /dev/null +++ b/tools/stack-bench/backends/model-free-stub.md @@ -0,0 +1,4 @@ +# Model-free service + +Use the supplied service. Leave the app running on the assigned client port +when the work is complete. diff --git a/tools/stack-bench/backends/mongodb.md b/tools/stack-bench/backends/mongodb.md new file mode 100644 index 00000000000..48ae7319b5f --- /dev/null +++ b/tools/stack-bench/backends/mongodb.md @@ -0,0 +1,50 @@ +# Backend: MongoDB + +An Express API server with Socket.io for live updates, Mongoose over MongoDB, +and a React client. + +## Layout + +``` +/ + server/ + package.json express, socket.io, mongoose, dotenv, tsx + .env DATABASE_URL and PORT + src/models.ts Mongoose schemas and models + src/index.ts Express routes, Socket.io handlers + client/ + package.json react, react-dom, vite, socket.io-client + vite.config.ts server.port , proxy /api and /socket.io to + index.html + src/main.tsx + src/App.tsx +``` + +## Deploy + +```bash +cd server && npm install && npm run dev # on +cd client && npm install && npm run dev # on +``` + +Mongoose creates collections on first write; there is no migration step. + +The server prints to the terminal running `npm run dev`; it restarts on save, +so a code change is live without redeploying. + +Keep existing application data when you change the schema. Do not drop +collections during upgrades or repairs. + +## Configuration + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| API server port | `` | +| Client dev server | `` | + +Use this exact `DATABASE_URL`. Do not point at another MongoDB instance. +Read `DATABASE_URL` from the process environment at startup. It can change between +launches; do not embed it in source or override it with a saved value. + +The supplied database is a single-node replica set. diff --git a/tools/stack-bench/backends/postgres.md b/tools/stack-bench/backends/postgres.md new file mode 100644 index 00000000000..b605b9aeaf7 --- /dev/null +++ b/tools/stack-bench/backends/postgres.md @@ -0,0 +1,50 @@ +# Backend: PostgreSQL + +An Express API server with Socket.io for live updates, Drizzle ORM over +PostgreSQL, and a React client. + +## Layout + +``` +/ + server/ + package.json express, socket.io, drizzle-orm, pg, dotenv, tsx + .env DATABASE_URL and PORT + drizzle.config.ts + src/schema.ts Drizzle table definitions + src/index.ts Express routes, Socket.io handlers + client/ + package.json react, react-dom, vite, socket.io-client + vite.config.ts server.port , proxy /api and /socket.io to + index.html + src/main.tsx + src/App.tsx +``` + +## Deploy + +```bash +cd server && npm install && npx drizzle-kit push && npm run dev # on +cd client && npm install && npm run dev # on +``` + +Re-run `npx drizzle-kit push` after any schema change. + +The server prints to the terminal running `npm run dev`; it restarts on save, +so a code change is live without redeploying. + +Keep existing application data when you change the schema. Do not drop or +recreate tables during upgrades or repairs. + +## Configuration + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| API server port | `` | +| Client dev server | `` | + +Use this exact `DATABASE_URL`. Do not point at another PostgreSQL instance and do +not create databases outside it. +Read `DATABASE_URL` from the process environment at startup. It can change between +launches; do not embed it in source or override it with a saved value. diff --git a/tools/stack-bench/backends/spacetime.md b/tools/stack-bench/backends/spacetime.md new file mode 100644 index 00000000000..250587f88fe --- /dev/null +++ b/tools/stack-bench/backends/spacetime.md @@ -0,0 +1,80 @@ +# Backend: SpacetimeDB + +The database runs your server logic. There is no separate API server and no ORM: +tables and reducers are a WASM module you publish, and the client subscribes to +tables and calls reducers over a live connection. + +## Layout + +``` +/ + backend/spacetimedb/ + package.json { "type": "module", dependencies: { "spacetimedb": "" }, + devDependencies: { "typescript": "~5.6.2" } } ← required; the build runs tsc from node_modules + tsconfig.json + src/schema.ts tables and indexes + src/index.ts reducers and lifecycle hooks + client/ + package.json react, react-dom, vite, and "spacetimedb": "" + vite.config.ts server.port must be + index.html + src/config.ts MODULE_NAME and SPACETIMEDB_URI + src/main.tsx React entry + src/App.tsx + src/module_bindings/ generated; never edit by hand +``` + +## Deploy + +Publish the module, then regenerate the client bindings from it: + +```bash + publish --module-path backend/spacetimedb -s --yes + generate --lang typescript --out-dir client/src/module_bindings --module-path backend/spacetimedb +``` + +**While iterating, run development mode instead of republishing by hand.** It +watches the module and automatically rebuilds, publishes, and regenerates the +client bindings on every save: + +```bash + dev --module-path backend/spacetimedb -s --yes +``` + +Leave it running in the background while you work. The manual commands below +are for one-off publishes and for the first deploy. + +Republish after any server change, and regenerate after any schema change. + +Keep existing application data when you change the schema. + +Always use `--yes` for local publish and development commands. It selects the +CLI's non-interactive authentication flow for the target server. Do not pipe +`y` into the command and do not publish anonymously. Use the same local +identity for every publish to the named module. + +Then start the client: + +```bash +cd client && npm install && npm run dev +``` + +` logs -s ` shows module output, including reducer errors. + +To inspect stored data while debugging: + +```bash + sql "SELECT * FROM item LIMIT 5" -s +``` + +## Configuration + +| Setting | Value | +|---|---| +| Server URI | `` | +| Module name | `` | +| Client dev server | `` | + +The SDK reference for writing modules and clients is in the skill documents +included with these instructions. Follow them for API specifics: import paths, +type builders, accessors and context typing. diff --git a/tools/stack-bench/backends/workflows/spacetime-dev.md b/tools/stack-bench/backends/workflows/spacetime-dev.md new file mode 100644 index 00000000000..7c281dd2dac --- /dev/null +++ b/tools/stack-bench/backends/workflows/spacetime-dev.md @@ -0,0 +1,44 @@ +--- +name: spacetime-dev +description: Use the SpacetimeDB development watcher while implementing an application. +--- + +# Development workflow + +Use `spacetime dev` while implementing and repairing the application. Keep one +watcher running for the assigned database. It builds module changes, publishes +them, and generates client bindings. Use the supplied CLI, server URI, database +name, and module directory. See the CLI skill for command syntax. + +Use the supplied server URL directly; do not register a server nickname or +change the CLI login. Use a project configuration with both publish and generate +targets. For example, in `/app/spacetime.json`, replacing the server, database, +and client directory with the supplied settings and your actual paths: + +```json +{ + "server": "http://SERVER:PORT", + "database": "DATABASE", + "module-path": "backend/spacetimedb", + "generate": [ + { "language": "typescript", "out-dir": "frontend/src/module_bindings" } + ] +} +``` + +From `/app`, run the supplied CLI with `dev --yes --delete-data=never +--server-only`. Start the web client separately. With these configured targets, +omit `--module-path`, `--project-path`, and `--module-bindings-path` flags. +Paths in this example are relative to the project directory. +Do not run competing publish commands or watchers. If bindings generation is skipped, +correct the generate target before continuing. + +Wait for the initial publish and bindings to succeed before opening the app. +After a module edit, check that the watcher published it successfully before +checking app behavior. Fix watcher errors; do not assume that the live module is current. +Restart the watcher if it exited or its configuration changed. + +Keep `/app/start.sh` able to build and start the complete application from a +clean source checkout without this development session. Stop the watcher before +checking that startup path. One-shot commands remain appropriate for that script +and for diagnosing a watcher failure. diff --git a/tools/stack-bench/backends/workflows/spacetime-managed-dev.md b/tools/stack-bench/backends/workflows/spacetime-managed-dev.md new file mode 100644 index 00000000000..9b4b4c39960 --- /dev/null +++ b/tools/stack-bench/backends/workflows/spacetime-managed-dev.md @@ -0,0 +1,24 @@ +--- +name: spacetime-managed-dev +description: Use the supplied command to manage the SpacetimeDB development watcher. +--- + +# Development workflow + +Development watcher support is available at `/deps/spacetime-dev`. It has not +started yet. Create the module and `/app/spacetime.json` first. Configure the +supplied server URL and database name, your `module-path`, and TypeScript +`generate` targets with their `out-dir` paths. Keep these paths inside `/app`. +This helper supports one database per application. See the CLI skill for configuration syntax. + +Run `/deps/spacetime-dev start`. It starts one watcher with data deletion disabled. +Repeated calls report the existing watcher. Run `/deps/spacetime-dev status` to +check startup, and read the reported log for build or publish errors. +"Starting" does not mean the initial publish has completed. "Running" confirms +the initial publish and bindings; later edits can still fail, so check the log. +Start the frontend separately. Do not start another watcher or competing publisher. + +Use `/deps/spacetime-dev stop` before changing configuration or testing a clean +startup, then `start` again when needed. An exited watcher is not restarted +automatically. Keep `/app/start.sh` able to build and start the complete application +without this development session. Container cleanup stops the watcher. diff --git a/tools/stack-bench/commands/agent.ts b/tools/stack-bench/commands/agent.ts new file mode 100644 index 00000000000..97fb78e9ef2 --- /dev/null +++ b/tools/stack-bench/commands/agent.ts @@ -0,0 +1,1004 @@ +#!/usr/bin/env node + +import { randomUUID } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import type { ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, realpathSync, + openSync, readSync, closeSync, readdirSync } from 'node:fs'; +import { join, dirname, resolve, relative, isAbsolute, sep } from 'node:path'; +import { homedir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { loadTrack, levelPrompt, appendix, suitesFor, dbName, moduleName, portsFor, + DEFAULT_TRACK, TRACK_MANIFEST_FILE } from '../src/composition/tracks.js'; +import type { Track, TrackDefinition } from '../src/composition/tracks.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { parseGuidanceMode, resolveDefaultGuidanceForStack, type GuidanceMode, + type ResolvedGuidanceDocument, type ResolvedSkills } + from '../src/campaigns/condition-compiler.js'; +import type { RecipeBinding, RecipeRequest } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { agentVisibleContractText, assertAgentVisibleText } + from '../src/composition/agent-visible-contract.js'; +import { DEFAULT_SPACETIME_SERVER_URI, leaseFromEnv } from '../src/runtime/backend-lease.js'; +import { CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_BUG_REPORT_FILE, + CODING_CONTAINER_RELEASE_DEPS_ROOT, CODING_CONTAINER_SPACETIME_CLI, + CODING_CONTAINER_SPACETIME_PACKAGE } + from '../src/runtime/coding-container-policy.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { hashDirectory, sessionProvenance, sha256 } from '../src/evidence/provenance.js'; +import type { StackRunPorts } from '../src/stacks/stack-adapter-contract.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { attemptDatabaseUrl } from '../src/stacks/hosted-database-identity.js'; +import { requireLeasedDatabase, requireLeasedSpacetime } + from '../src/stacks/backend-reset-guard.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { dockerMountArguments } from '../src/runtime/container-mount.js'; +import { normalizePromptText, readAgentSkillDocuments, selectAgentSkills } from '../src/agents/agent-materials.js'; +import { codingSessionFailure, DEFAULT_THROTTLE_MAX_WAIT_MS, providerSessionFailure, + runCodingSessionWithRetries, PROVIDER_CONTINUATION_MESSAGE } from '../src/agents/coding-session-retry.js'; +import { captureNativeContinuation } from '../src/agents/provider-native-continuation.js'; +import { campaignProviderContinuationContext, persistCampaignProviderInvocation, + waitForCampaignProviderContinuation } from '../src/campaigns/campaign-provider-continuation.js'; +import type { CodingSessionRetryResult } from '../src/agents/coding-session-retry.js'; +import { AGENT_PROCESS_TIMEOUT_MS } from '../src/agents/coding-session-timeouts.js'; +import { assertNewOrEmptyDirectory } from '../src/runtime/path-safety.js'; +import { resolveContainerAuth } from '../container/container-auth.js'; +import { CODING_PROVIDERS, parseCodingProvider } from '../container/coding-providers.js'; +import { validateProviderRoute, validateProviderOutputLimit } from '../src/agents/agent-adapter-contract.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import type { PricingAuthority } from '../src/evidence/pricing-authority.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const CONTROL_COMMAND_TIMEOUT_MS = 120_000; +const DEFAULT_CODING_INTERRUPTION_RETRIES = 2; + +type UnknownRecord = Record; +interface PromptMaterials { + skillsText?: string; + requirementText?: string; + contractText?: string; + startingCatalog?: string; +} + +type RecipeTaskRequest = Parameters[1] & { + recipe?: Exclude; +}; + +type AgentMode = 'build' | 'upgrade' | 'fix' | 'resume'; + +interface AgentArgs { + provider: keyof typeof CODING_PROVIDERS; + providerRoute?: string; + maxOutputTokens?: number; + mode: AgentMode; + backend: string; + app: string; + level: number; + runIndex: number; + model: string; + guidance: GuidanceMode; + productionQuality?: boolean; + track: string; + pricing: Readonly | null; + guidanceDocument?: ResolvedGuidanceDocument; + credentialAliases?: Readonly>; + recipe?: string; + recipeTask?: RecipeTaskRequest; + thinking?: string; + maxBudgetUsd?: number; + skills?: string[]; + skillIdentity?: ResolvedSkills; + apiKey?: string; + printPrompt?: boolean; +} + +interface ThinkingVolume { + blocks: number; + signatureBytes: number; + bytesPerBlock: number; +} + +interface SessionUsage { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; +} + +const isRecord = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const stringValue = (value: unknown): string | null => typeof value === 'string' ? value : null; + +function stringArray(value: string, option: string): string[] { + const parsed: unknown = JSON.parse(value); + if (!Array.isArray(parsed) || parsed.some(item => typeof item !== 'string')) { + throw new Error(`${option} must be an array of strings`); + } + return parsed; +} + +function sessionUsage(value: unknown): SessionUsage { + return isRecord(value) ? { + input_tokens: typeof value.input_tokens === 'number' ? value.input_tokens : undefined, + output_tokens: typeof value.output_tokens === 'number' ? value.output_tokens : undefined, + cache_creation_input_tokens: typeof value.cache_creation_input_tokens === 'number' + ? value.cache_creation_input_tokens : undefined, + cache_read_input_tokens: typeof value.cache_read_input_tokens === 'number' + ? value.cache_read_input_tokens : undefined, + } : {}; +} + +// Use only the benchmark-owned SpacetimeDB host. +const STDB_URI = process.env.STACK_BENCH_STDB_URI ?? DEFAULT_SPACETIME_SERVER_URI; + +// Test the CLI and SDK from this checkout. +const LOCAL_CLI = join(REPO, 'target', 'release', 'spacetimedb-cli.exe'); +const STDB_BIN = process.env.SPACETIME_BIN ?? (existsSync(LOCAL_CLI) ? LOCAL_CLI : 'spacetime'); +const LOCAL_PKG = process.env.STDB_PACKAGE ?? join(REPO, 'crates', 'bindings-typescript'); + +const fwd = (path: string): string => path.split('\\').join('/'); + +// Keep the provider's default thinking budget unless an experiment selects one +// explicitly. The run records observed reasoning volume so default changes are +// visible in the evidence. +const THINKING_TOKENS = process.env.STACK_BENCH_THINKING ?? null; + +const EFFORT = process.env.STACK_BENCH_EFFORT ?? 'high'; + +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; + +// Containers reach host services through this address. App ports remain local. +export function hostServiceAddress(env: NodeJS.ProcessEnv = process.env): string { + return env.STACK_BENCH_HOST_ALIAS + ?? (env.STACK_BENCH_APPLIANCE === '1' ? '127.0.0.1' : 'host.docker.internal'); +} + +const HOST_ADDR = hostServiceAddress(); +const hostUrl = (url: string): string => url.replace(/127\.0\.0\.1|localhost/g, HOST_ADDR); + +const C_BIN = CODING_CONTAINER_SPACETIME_CLI; + +// The container requires the Linux CLI from this checkout. +const LINUX_CLI = process.env.STACK_BENCH_LINUX_CLI + ?? join(ROOT, 'container', 'bin', 'spacetimedb-cli'); + +// The provider CLI keeps one JSONL transcript per session under its project +// directory for the application path. +function transcriptFile(appDir: string, sessionId: string): string | null { + const store = join(homedir(), '.claude', 'projects'); + if (!existsSync(store)) return null; + const want = resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase(); + const dir = readdirSync(store).find(d => { + const n = d.toLowerCase(); + return n === want || n === want.replace(/^-+/, ''); + }); + const file = dir && join(store, dir, `${sessionId}.jsonl`); + return file && existsSync(file) ? file : null; +} + +// The model ids the provider actually served. The requested name is an alias +// that can resolve to different snapshots over time; the transcript records +// what answered each request. +function transcriptModels(appDir: string, sessionIds: readonly (string | null | undefined)[]): string[] { + const models = new Set(); + for (const sessionId of new Set(sessionIds.filter((id): id is string => Boolean(id)))) { + try { + const file = transcriptFile(appDir, sessionId); + if (!file) continue; + for (const line of readFileSync(file, 'utf8').split('\n')) { + if (!line.includes('"model"')) continue; + let record: unknown; + try { record = JSON.parse(line); } catch { continue; } + if (!isRecord(record) || !isRecord(record.message)) continue; + const model = stringValue(record.message.model); + if (model) models.add(model); + } + } catch { /* an unreadable transcript leaves the list shorter, never wrong */ } + } + return [...models].sort(); +} + +// The transcript exposes reasoning blocks and signature bytes, not reasoning tokens. +function thinkingVolume(appDir: string, sessionId: string | null | undefined): ThinkingVolume | null { + if (!sessionId) return null; + try { + const file = transcriptFile(appDir, sessionId); + if (!file) return null; + + let blocks = 0, bytes = 0; + for (const line of readFileSync(file, 'utf8').split('\n')) { + if (!line.includes('"thinking"')) continue; // cheap filter before parsing + let record: unknown; + try { record = JSON.parse(line); } catch { continue; } + if (!isRecord(record) || !isRecord(record.message) + || !Array.isArray(record.message.content)) continue; + for (const content of record.message.content) { + if (!isRecord(content) || content.type !== 'thinking') continue; + blocks++; + bytes += stringValue(content.signature)?.length ?? 0; + } + } + return { blocks, signatureBytes: bytes, + bytesPerBlock: blocks ? Math.round(bytes / blocks) : 0 }; + } catch { return null; } +} + +function combinedThinkingVolume(appDir: string, sessionIds: readonly (string | null | undefined)[]): ThinkingVolume | null { + const volumes: ThinkingVolume[] = [...new Set(sessionIds.filter((id): id is string => Boolean(id)))] + .map(id => thinkingVolume(appDir, id)).filter((item): item is ThinkingVolume => item !== null); + if (!volumes.length) return null; + const blocks = volumes.reduce((sum, item) => sum + item.blocks, 0); + const signatureBytes = volumes.reduce((sum, item) => sum + item.signatureBytes, 0); + return { blocks, signatureBytes, + bytesPerBlock: blocks ? Math.round(signatureBytes / blocks) : 0 }; +} + + +// Record the Linux CLI executed by the container. The host and container +// binaries can change independently and must not share an identity. +function linuxSpacetimeVersion(image: string): { commit: string | null; binarySha256: string | null; raw: string } { + try { + const releaseVolume = process.env.STACK_BENCH_RELEASE_DEPS_VOLUME?.trim() || null; + const mountArgs = releaseVolume + ? dockerMountArguments({ kind: 'volume', source: releaseVolume, + target: CODING_CONTAINER_RELEASE_DEPS_ROOT, readOnly: true }) + : ['-v', `${LINUX_CLI}:${CODING_CONTAINER_SPACETIME_CLI}:ro`]; + const entrypoint = releaseVolume + ? `${CODING_CONTAINER_RELEASE_DEPS_ROOT}/spacetimedb-cli` + : CODING_CONTAINER_SPACETIME_CLI; + const out = execFileSync('docker', + ['run', '--rm', ...mountArgs, '--entrypoint', entrypoint, image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }); + const commit = out.match(/Commit:\s*([0-9a-f]+)/i)?.[1] ?? null; + return { commit, binarySha256: sha256(readFileSync(LINUX_CLI)), + raw: out.trim().split(/\r?\n/).slice(0, 2).join(' ') }; + } catch { return { commit: null, binarySha256: null, raw: 'unknown' }; } +} + +function bindingsIdentity(pkgDir: string): { package: string; sourceSha256: string | null; sourceFiles: number } { + try { + const p = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')); + const source = hashDirectory(pkgDir, { exclude: name => + /(^|\/)(node_modules|dist|target)(\/|$)/.test(name) }); + return { package: `${p.name}@${p.version}`, sourceSha256: source.sha256, + sourceFiles: source.files.length }; + } catch { return { package: 'unknown', sourceSha256: null, sourceFiles: 0 }; } +} + +// The CLI version inside the build image. Read by running it, not by trusting +// the tag: the image is pinned by ARG and a tag can be moved. +function imageCliVersion(image: string, executable: string): string { + try { + return execFileSync('docker', ['run', '--rm', '--entrypoint', executable, image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + } catch { return 'unknown'; } +} + +function imageNodeVersion(image: string): string { + try { + return execFileSync('docker', ['run', '--rm', '--entrypoint', 'node', image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + } catch { return 'unknown'; } +} + +function containerImage(name: string): { reference: string; imageId: string | null | undefined } { + try { + const out = execFileSync('docker', ['inspect', '-f', '{{.Config.Image}} {{.Image}}', name], + { encoding: 'utf8', stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + const [reference, imageId] = out.split(/\s+/, 2); + return { reference: reference ?? '', imageId }; + } catch { return { reference: 'unknown', imageId: null }; } +} + +// Record ambient provider configuration that can change model behaviour while +// replacing credential values with presence markers. +function ambientEnv(): Record { + const seen: Record = {}; + for (const [k, v] of Object.entries(process.env)) { + if (!/^(CLAUDE|ANTHROPIC|OPENAI|OPENROUTER|CODEX|MAX_THINKING|DISABLE_AUTOUPDATER|FORCE_PROMPT)/.test(k)) continue; + // Never record a credential, only that one was present. + seen[k] = /KEY|TOKEN|SECRET|AUTH/i.test(k) ? '' : v; + } + return seen; +} + +export function parseAgentArgs(argv: readonly string[]): AgentArgs { + const strings = ['provider', 'provider-route', 'max-output-tokens', 'mode', 'track', 'backend', 'level', 'app', 'run-index', 'model', + 'pricing-json', 'guidance', 'guidance-document-json', 'credential-aliases-json', + 'recipe', 'recipe-task-json', 'thinking', 'max-budget-usd', 'skills', 'skills-json', + 'skill-identity-json', 'api-key'] as const; + const { values: rawValues } = parseNodeArgs({ args: [...argv.slice(2)], options: Object.fromEntries([ + ...strings.map(name => [name, { type: 'string' as const }]), + ['print-prompt', { type: 'boolean' as const }], + ['production-quality', { type: 'boolean' as const }], + ['no-production-quality', { type: 'boolean' as const }], + ]), strict: true, allowPositionals: false }); + const values = rawValues as Partial> + & { 'print-prompt'?: boolean; 'production-quality'?: boolean; 'no-production-quality'?: boolean }; + if (values['production-quality'] && values['no-production-quality']) throw new Error('choose only one production-quality flag'); + const mode = values.mode; + if (mode !== 'build' && mode !== 'upgrade' && mode !== 'fix' && mode !== 'resume') { + throw new Error('--mode must be build, upgrade, fix, or resume'); + } + const backend = values.backend; + const app = values.app; + if (!backend || !app) { + throw new Error('usage: node dist/commands/agent.js --mode build|upgrade|fix|resume ' + + '--backend --app [--level ]'); + } + const level = values.level === undefined ? 1 : Number(values.level); + if (!Number.isSafeInteger(level) || level < 1) { + throw new Error('--level must be a positive integer'); + } + const runIndex = values['run-index'] === undefined ? 0 : Number(values['run-index']); + if (!Number.isSafeInteger(runIndex) || runIndex < 0) { + throw new Error('--run-index must be a non-negative integer'); + } + const provider = parseCodingProvider(values.provider ?? 'anthropic'); + const codingProvider = CODING_PROVIDERS[provider]; + if (codingProvider.requiresBudget && !values.model) throw new Error(`--model is required for ${provider}`); + if (codingProvider.executable !== 'claude' && values.thinking) { + throw new Error(`${provider} uses STACK_BENCH_EFFORT, not --thinking`); + } + const providerRoute = validateProviderRoute(provider, values['provider-route']); + const maxOutputTokens = validateProviderOutputLimit(provider, + values['max-output-tokens'] === undefined ? undefined : Number(values['max-output-tokens'])); + const model = values.model ?? 'claude-sonnet-5'; + const maxBudgetUsd = values['max-budget-usd'] === undefined + ? undefined : Number(values['max-budget-usd']); + if (maxBudgetUsd !== undefined && (!Number.isFinite(maxBudgetUsd) || maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (values.skills !== undefined && values['skills-json'] !== undefined) { + throw new Error('--skills and --skills-json cannot be used together'); + } + const skills = values.skills?.split(',').map(skill => skill.trim()).filter(Boolean) + ?? (values['skills-json'] === undefined ? undefined + : stringArray(values['skills-json'], '--skills-json')); + let pricing = values['pricing-json'] === undefined + ? undefined : validatePricingAuthority(JSON.parse(values['pricing-json']), { at: '--pricing-json' }); + if (pricing === undefined && maxBudgetUsd !== undefined) { + const rates = CODING_PROVIDERS[provider].rates(model); + if (!rates) throw new Error(`no default pricing is recorded for model ${model}`); + pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } + return { provider, ...(providerRoute ? { providerRoute } : {}), mode, backend, app, level, runIndex, model, + ...(maxOutputTokens ? { maxOutputTokens } : {}), + guidance: parseGuidanceMode(values.guidance ?? 'prescribed'), + productionQuality: mode !== 'resume' && !values['no-production-quality'], + track: values.track ?? DEFAULT_TRACK, pricing: pricing ?? null, + ...(values['guidance-document-json'] ? { + guidanceDocument: JSON.parse(values['guidance-document-json']) as ResolvedGuidanceDocument, + } : {}), + ...(values['credential-aliases-json'] ? { + credentialAliases: JSON.parse(values['credential-aliases-json']) as Record, + } : {}), + ...(values.recipe ? { recipe: values.recipe } : {}), + ...(values['recipe-task-json'] ? { + recipeTask: JSON.parse(values['recipe-task-json']) as RecipeTaskRequest, + } : {}), + ...(values.thinking ? { thinking: values.thinking } : {}), + ...(maxBudgetUsd !== undefined ? { maxBudgetUsd } : {}), + ...(skills ? { skills } : {}), + ...(values['skill-identity-json'] ? { + skillIdentity: validateSkillIdentity(JSON.parse(values['skill-identity-json'])), + } : {}), + ...(values['api-key'] ? { apiKey: values['api-key'] } : {}), + ...(values['print-prompt'] ? { printPrompt: true } : {}) }; +} + +const dbUrl = (backend: string, runIndex: number, dbPort: number | null, track: Track): string | null => { + const adapter = STACK_ADAPTER_REGISTRY.get(backend); + if (adapter.id === 'spacetime' || adapter.id === 'convex' || adapter.id === 'stub') return null; + if (!dbPort) throw new Error(`${backend} has no assigned database port`); + if (process.env.STACK_BENCH_APPLIANCE === '1' && process.env.STACK_BENCH_LEASE) { + const { lease } = leaseFromEnv(process.env, { backend, active: true }); + if (lease.resources.network) return attemptDatabaseUrl({ backend, + database: lease.resources.database!, ownershipToken: lease.ownershipToken }); + } + return adapter.agent.connectionUrl({ dbPort, database: dbName(track, runIndex), hostUrl }); +}; + +// Create the leased database before the app connects. A build clears its schema. +// A reset between suites preserves the schema required by the running app. +type DatabasePreparationLease = BackendLease; + +type DatabaseCommandOptions = Pick; + +type DatabaseCommandExecutor = (command: string, args: readonly string[], + options: DatabaseCommandOptions) => string; + +const databaseCommandExecutor: DatabaseCommandExecutor = (command, args, options) => + String(execFileSync(command, args, { ...options, encoding: 'utf8' })); + +interface DatabasePreparationOptions { + exec?: DatabaseCommandExecutor; + stdbBin?: string; + lease?: DatabasePreparationLease; +} + +export function ensureDatabase(backend: string, runIndex: number, dbPort: number | null, + track: Pick, wipe = false, + { exec = databaseCommandExecutor, stdbBin = STDB_BIN, lease: suppliedLease }: DatabasePreparationOptions = {}) { + const lease = suppliedLease ?? leaseFromEnv(process.env, { backend, active: true }).lease; + if (lease.runIndex !== runIndex || lease.track !== track.name) { + throw new Error(`backend lease ${lease.runId} belongs to ${lease.track}/run${lease.runIndex}, ` + + `not ${track.name}/run${runIndex}`); + } + const expectedName = dbName(track, runIndex); + const name = lease.resources.database ?? expectedName; + const input = { name, expectedName, wipe, exec, cli: stdbBin, + expectedServerUri: STDB_URI, expectedModule: moduleName(track, runIndex), dbPort }; + const adapter = STACK_ADAPTER_REGISTRY.get(backend); + if (adapter.id === 'postgres' || adapter.id === 'mongodb') { + return adapter.database.prepare({ ...input, lease: requireLeasedDatabase(lease) }); + } + if (adapter.id === 'spacetime') { + return adapter.database.prepare({ ...input, lease: requireLeasedSpacetime(lease) }); + } + if (adapter.id === 'convex') { + if (!lease.resources.serverUri || !lease.resources.container?.owned) { + throw new Error('Convex application setup requires its active owned deployment'); + } + // The lifecycle owns the native deployment; start.sh deploys its functions. + return; + } + return adapter.database.prepare({ name }); +} + +// Prescribed guidance chooses an implementation stack. Neutral guidance gives +// only stack access facts and the selected API references. +export function readBackendGuidanceDocument( + document: ResolvedGuidanceDocument | undefined, + fallbackRelativePath: string, +): string { + if (typeof fallbackRelativePath !== 'string' || !fallbackRelativePath) { + throw new Error('backend guidance fallback path is required'); + } + if (document !== undefined) { + const fields = new Set(['path', 'sha256', 'bytes', 'applicationInterface']); + if (!document || typeof document !== 'object' || Array.isArray(document) + || Object.keys(document).some(field => !fields.has(field)) + || typeof document.path !== 'string' || !document.path || isAbsolute(document.path) + || document.path.includes('\\') + || !/^[a-f0-9]{64}$/.test(document.sha256) + || !Number.isSafeInteger(document.bytes) || document.bytes < 0 + || !['http', 'reducer', 'convex'].includes(document.applicationInterface)) { + throw new Error('campaign guidance document identity is invalid'); + } + } + const root = realpathSync(ROOT); + const candidate = resolve(root, document?.path ?? fallbackRelativePath); + const candidateRel = relative(root, candidate); + if (candidateRel === '..' || candidateRel.startsWith(`..${sep}`) || isAbsolute(candidateRel)) { + throw new Error('campaign guidance document escapes the Stack Bench root'); + } + const selectedPath = realpathSync(candidate); + const resolvedRel = relative(root, selectedPath); + if (resolvedRel === '..' || resolvedRel.startsWith(`..${sep}`) || isAbsolute(resolvedRel)) { + throw new Error('campaign guidance document resolves outside the Stack Bench root'); + } + const bytes = Buffer.from(normalizePromptText(readFileSync(selectedPath, 'utf8')), 'utf8'); + if (document && (sha256(bytes) !== document.sha256 || bytes.length !== document.bytes)) { + throw new Error(`campaign guidance document changed after compilation: ${document.path}`); + } + return bytes.toString('utf8'); +} + +function validateSkillIdentity(value: unknown): ResolvedSkills { + const fields = new Set(['ids', 'sha256', 'bytes']); + if (!isRecord(value) || Object.keys(value).some(field => !fields.has(field)) + || !Array.isArray(value.ids) || new Set(value.ids).size !== value.ids.length + || value.ids.some(id => typeof id !== 'string' || !/^[a-z][a-z0-9-]*$/.test(id)) + || typeof value.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(value.sha256) + || !Number.isSafeInteger(value.bytes) || Number(value.bytes) < 0) { + throw new Error('campaign skill identity is invalid'); + } + return { ids: value.ids as string[], sha256: value.sha256, bytes: Number(value.bytes) }; +} + +function backendDoc(args: AgentArgs, p: StackRunPorts, track: Track): string { + const defaultGuidance = resolveDefaultGuidanceForStack(args.guidance, args.backend); + let defaultPath = defaultGuidance?.documents[args.backend]?.path; + if (!defaultPath && args.guidance === 'neutral') { + throw new Error(`neutral guidance has no document for ${args.backend}`); + } + defaultPath ??= join('backends', `${args.backend}.md`); + const raw = readBackendGuidanceDocument(args.guidanceDocument, defaultPath); + return raw + .replaceAll('', String(p.vite)) + .replaceAll('', String(p.express ?? '')) + .replaceAll('', track.title) + .replaceAll('', moduleName(track, args.runIndex)) + .replaceAll('', p.dbPort ? dbUrl(args.backend, args.runIndex, p.dbPort, track) ?? '' : '') + .replaceAll('', hostUrl(STDB_URI)) + .replaceAll('', C_BIN) + .replaceAll('', `file:${CODING_CONTAINER_SPACETIME_PACKAGE}`); +} + +// Fail before a paid session when the selected container cannot run this checkout. +function containerBlocker(backend: string): string | null { + try { + execFileSync('docker', ['image', 'inspect', IMAGE], + { stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS }); + } catch (error: unknown) { + const detail = error instanceof Error ? error.message.split('\n')[0] : String(error).split('\n')[0]; + return `cannot verify isolation image ${IMAGE}: ${detail} — ` + + `build it with docker build -t ${IMAGE} ${fwd(join(ROOT, 'container'))}`; + } + if (!STACK_ADAPTER_REGISTRY.get(backend).agent.linuxCliRequired) return null; + if (!existsSync(LINUX_CLI)) { + return `no Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)} — ` + + 'bash tools/stack-bench/container/build-linux-cli.sh'; + } + // A file at this path must be a Linux executable, not the Windows build. + const magic = Buffer.alloc(4); + try { + const fd = openSync(LINUX_CLI, 'r'); + try { readSync(fd, magic, 0, 4, 0); } finally { closeSync(fd); } + } catch { + return `cannot read the Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)}`; + } + if (magic.toString('binary') !== '\x7fELF') { + return `${fwd(LINUX_CLI)} is not a Linux binary; rebuild it with ` + + 'container/build-linux-cli.sh'; + } + return null; +} + +function decideIsolation(args: AgentArgs): { container: true; reason: null } { + const blocker = containerBlocker(args.backend); + if (!blocker) return { container: true, reason: null }; + console.error(`agent.js: isolated build unavailable: ${blocker}`); + console.error(' benchmark coding sessions require the isolation container'); + process.exit(2); +} + +// Pin every round to the build's recorded container topology. +function resolveIsolation(args: AgentArgs): { container: true; reason: null } { + const marker = resolve(args.app, '..', '.stack-bench-isolation'); + const backendMarker = resolve(args.app, '..', '.stack-bench-backend'); + + if (args.mode === 'build') { + const decided = decideIsolation(args); + if (!args.printPrompt) { + mkdirSync(dirname(marker), { recursive: true }); + writeFileSync(marker, 'container'); + } + return decided; + } + + if (existsSync(marker)) { + const pinned = readFileSync(marker, 'utf8').trim(); + if (pinned !== 'container') { + console.error(`agent.js: unsupported isolation marker ${JSON.stringify(pinned)}; expected "container"`); + process.exit(2); + } + const blocker = containerBlocker(args.backend); + if (blocker) { + console.error(`agent.js: this run's build ran in a container, but ${blocker}`); + console.error(' refusing to run this round in a different environment'); + process.exit(2); + } + return { container: true, reason: null }; + } + + // A backend marker without an isolation marker is ambiguous prior state. + if (existsSync(backendMarker)) { + console.error('agent.js: app has prior benchmark state but no isolation marker'); + console.error(' refusing to guess where earlier rounds ran; start a clean run'); + process.exit(2); + } + const decided = decideIsolation(args); + if (!args.printPrompt) { + mkdirSync(dirname(marker), { recursive: true }); + writeFileSync(marker, 'container'); + } + return decided; +} + +export function buildPrompt(args: AgentArgs, p: StackRunPorts, track: Track, + materials: PromptMaterials = {}): string { + const prompt = (lines: string[]): string => assertAgentVisibleText(lines.join('\n')); + const applicationInterface = args.guidanceDocument?.applicationInterface + ?? resolveDefaultGuidanceForStack(args.guidance, args.backend) + ?.documents[args.backend]?.applicationInterface; + if (applicationInterface !== 'http' && applicationInterface !== 'reducer' && applicationInterface !== 'convex') { + throw new Error(`stack ${args.backend} has no application interface`); + } + const common = [ + `Build the app in ${CODING_CONTAINER_APP_ROOT}.`, + // Published container ports require the app to bind to all interfaces. + '', + 'The web application must listen on 0.0.0.0, not localhost, so it is reachable ' + + 'outside its process.', + 'The environment can run /app/start.sh again with APP_WARM_START=1. ' + + 'When dependencies are current, reuse them instead of installing them again.', + 'Startup must work with an empty database by creating the supplied starting data and accounts. ' + + 'On an existing database, preserve current quantities, prices, and user data. ' + + 'This applies after upgrades and repairs too.', + '', + 'Chromium is installed at /usr/bin/chromium (CHROME_BIN). ' + + 'Use that executable with --no-sandbox in this isolated container; no browser download is needed. ' + + 'Puppeteer Core is installed at /opt/browser-tools/node_modules/puppeteer-core. ' + + 'In a .cjs script, use const puppeteer = require("/opt/browser-tools/node_modules/puppeteer-core"); ' + + 'then await puppeteer.launch({ executablePath: process.env.CHROME_BIN, args: ["--no-sandbox"] }).', + '', + '## Stack', + '', + agentVisibleContractText(backendDoc(args, p, track), args.credentialAliases, + applicationInterface), + ]; + const skills = materials.skillsText ?? readAgentSkillDocuments(ROOT, args.skills ?? []); + if (skills) common.push('', '## Selected API reference', '', skills); + + if (args.mode === 'resume') { + return prompt([ + 'Restore the existing application to a runnable state.', + '', + 'This is a saved application from an earlier completed run. Install its', + 'dependencies and start its existing database module, server, and web client', + 'as needed. Do not implement features or fix application behavior. Do not', + 'change source files. The saved source must remain byte-for-byte identical.', + '', + 'Output RESUME_COMPLETE when the existing app is running.', + '', + ...common, + ]); + } + + if (args.productionQuality) common.unshift('Build a production-quality application suitable for real users, not a prototype or demo.', ''); + const startingCatalog = materials.startingCatalog + ? ['', '## Starting catalog', '', args.mode === 'build' + ? 'Use exactly this starting data:' + : 'This is the original catalog baseline. Preserve its entity names and relationships. Do not reset current quantities, prices, or user data.', + '', '```json', materials.startingCatalog, '```'] : []; + + if (args.mode === 'fix') { + return prompt([ + 'Fix the reported application bugs.', + '', + `Read ${CODING_CONTAINER_BUG_REPORT_FILE} in the app directory. Each entry says what was expected`, + 'and what actually happened. Fix the app so the behaviour matches, redeploy,', + 'and make sure the dev server is running.', + '', + 'Change only what is needed. Do not alter behaviour that is already correct.', + '', + 'Output FIX_COMPLETE when done.', + '', + ...common, + '', + agentVisibleContractText(materials.requirementText ?? levelPrompt(track, args.level), + args.credentialAliases, applicationInterface), + ...startingCatalog, + '', + '## Application interface', + '', + agentVisibleContractText(materials.contractText ?? appendix(track, args.level), + args.credentialAliases, applicationInterface), + ]); + } + + const verb = args.mode === 'upgrade' + ? [ + 'Add the features below to the existing app.', + '', + 'Keep completed features working. Add only the current features below.', + ] + : [`Build the application described below and leave it running.`]; + + return prompt([ + ...verb, + '', + `After the web application is running, reply with ${args.mode === 'upgrade' + ? 'UPGRADE_COMPLETE' : 'DEPLOY_COMPLETE'}.`, + '', + ...common, + '', + agentVisibleContractText(materials.requirementText ?? levelPrompt(track, args.level), + args.credentialAliases, applicationInterface), + ...startingCatalog, + '', + '## Application interface', + '', + agentVisibleContractText(materials.contractText ?? appendix(track, args.level), + args.credentialAliases, applicationInterface), + ]); +} + +export function agentScenarioPaths(track: Track, level: number, + recipeBinding: RecipeBinding | null = null): string[] { + const execution = recipeBinding?.execution; + if (execution) return execution.map(entry => resolve(track.dir, entry.source ?? '')); + return suitesFor(track, level).map(suite => suite.spec); +} + +export function agentRecipeRequest(explicitRecipe: string | null = null, + recipeTask: RecipeTaskRequest | null = null): RecipeRequest | null { + const bound = recipeTask?.recipe; + if (!bound) return explicitRecipe; + if (explicitRecipe && explicitRecipe !== bound.id) { + throw new Error(`agent recipe ${explicitRecipe} does not match bound task ${bound.id}`); + } + return bound; +} + +// The coding container must not contain the controller or grading inputs. + +export function refreshCodingInvocationCredentials({ provider, apiKey, keyFile, + expectedMode, env = process.env }: { provider: keyof typeof CODING_PROVIDERS; + apiKey?: string; keyFile?: string; expectedMode: string | null; env?: NodeJS.ProcessEnv }) { + const credential = keyFile ? readFileSync(keyFile, 'utf8').trim() + : apiKey ?? env[CODING_PROVIDERS[provider].apiKeyEnvironment] ?? ''; + if (keyFile && !credential) throw new Error('selected API key file is empty'); + const auth = resolveContainerAuth({ provider, apiKey: credential, env, + credentialsPath: CODING_PROVIDERS[provider].credentialPath }); + if (expectedMode !== null && expectedMode !== auth.mode) { + throw new Error('provider billing mode changed during the coding action'); + } + return { mode: auth.mode, env: { ...env, STACK_BENCH_AGENT_API_KEY: credential } }; +} + +async function main() { + const args = parseAgentArgs(process.argv); + const track = loadTrack(args.track); + const p = portsFor(track, args.backend, args.runIndex); + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const defaultGuidance = resolveDefaultGuidanceForStack(args.guidance, args.backend); + args.credentialAliases ??= defaultGuidance?.credentialAliases ?? {}; + const profileSkills = defaultGuidance?.skills[args.backend]?.ids; + const defaultSkills = profileSkills ?? [...adapter.agent.defaultSkills]; + const selectedSkills = selectAgentSkills(defaultSkills, + args.skillIdentity?.ids ?? args.skills ?? null); + const skillsText = readAgentSkillDocuments(ROOT, selectedSkills); + if (args.skillIdentity && (sha256(skillsText) !== args.skillIdentity.sha256 + || Buffer.byteLength(skillsText) !== args.skillIdentity.bytes)) { + throw new Error('campaign skill material changed after compilation'); + } + const recipeBinding = resolveRecipeRelease(track, args.level, + agentRecipeRequest(args.recipe ?? null, args.recipeTask ?? null)); + if (args.recipeTask && !recipeBinding) { + throw new Error(`L${args.level} has no recipe release for the requested task`); + } + const selectedTask = recipeBinding + ? (args.recipeTask + ? resolveBoundRecipeTaskRequest(recipeBinding, args.recipeTask) + : createBoundRecipeTaskRequest(recipeBinding)) + : null; + const requirementText = selectedTask?.task.requirementText ?? levelPrompt(track, args.level); + const contractText = selectedTask?.task.contractText ?? appendix(track, args.level); + const startingCatalog = recipeBinding ? JSON.stringify({ + warehouses: recipeBinding.plan.fixture.warehouses, + items: recipeBinding.plan.fixture.items, + }, null, 2) : undefined; + + // Print the exact prompt without starting a session or changing the app. + if (args.printPrompt) { + process.stdout.write(buildPrompt(args, p, track, + { skillsText, requirementText, contractText, startingCatalog })); + return; + } + if (args.mode === 'build') { + assertNewOrEmptyDirectory(args.app, 'build application directory'); + } + resolveIsolation(args); + const imageIdentity = resolveContainerImage(IMAGE); + // Build wipes all backend state. Later rounds preserve it. + ensureDatabase(args.backend, args.runIndex, p.dbPort, track, args.mode === 'build'); + // Never erase a caller-supplied application tree. + mkdirSync(args.app, { recursive: true }); + writeFileSync(resolve(args.app, '..', '.stack-bench-backend'), args.backend); + + const prompt = buildPrompt(args, p, track, + { skillsText, requirementText, contractText, startingCatalog }); + const bugReportPath = join(args.app, CODING_CONTAINER_BUG_REPORT_FILE); + const bugReportText = args.mode === 'fix' && existsSync(bugReportPath) + ? readFileSync(bugReportPath, 'utf8') : null; + const provenance = sessionProvenance({ prompt, skillsText, contractText, bugReportText, + scenarioPaths: agentScenarioPaths(track, args.level, recipeBinding), + trackDir: track.dir, trackManifestPath: join(track.dir, TRACK_MANIFEST_FILE) }); + const started = Date.now(); + const retryLimitRaw = process.env.STACK_BENCH_CODING_INTERRUPTION_RETRIES + ?? String(DEFAULT_CODING_INTERRUPTION_RETRIES); + const retryLimit = Number(retryLimitRaw); + if (!Number.isInteger(retryLimit) || retryLimit < 0 || retryLimit > 3) { + throw new Error('STACK_BENCH_CODING_INTERRUPTION_RETRIES must be an integer from 0 to 3'); + } + // The throttle wait must fit inside the adapter deadline. + const throttleWaitRaw = process.env.STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES + ?? String(DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000); + const throttleMaxWaitMinutes = Number(throttleWaitRaw); + if (!Number.isInteger(throttleMaxWaitMinutes) || throttleMaxWaitMinutes < 0 + || throttleMaxWaitMinutes > DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000) { + throw new Error('STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES must be an integer from 0 to ' + + `${DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000}`); + } + // Concurrent campaign slots must not wake and retry as one burst. This + // stable offset keeps retries reproducible while spreading them over 45s. + const throttleJitterMs = parseInt(sha256(Buffer.from( + `${args.backend}:${args.runIndex}:${args.level}:${args.mode}`)).slice(0, 8), 16) % 45_001; + const selectedKeyFile = process.env.STACK_BENCH_AGENT_API_KEY_FILE + ?? process.env.STACK_BENCH_API_KEY_FILE + ?? process.env[`${CODING_PROVIDERS[args.provider].apiKeyEnvironment}_FILE`]; + let selectedAuthMode: string | null = null; + const invocationEnvironment = (baseEnv: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv => { + const refreshed = refreshCodingInvocationCredentials({ provider: args.provider, apiKey: args.apiKey, + keyFile: selectedKeyFile, expectedMode: selectedAuthMode, env: baseEnv }); + selectedAuthMode = refreshed.mode; + return refreshed.env; + }; + const actionId = randomUUID(); + const continuationContext = campaignProviderContinuationContext(); + const continuationIdentity = { actionId, mode: args.mode, level: args.level, model: args.model, + provider: args.provider, providerRoute: args.providerRoute, imageId: imageIdentity.id, provenance, pricing: args.pricing, + maxOutputTokens: args.maxOutputTokens, + guidance: args.guidance, skillIdentity: args.skillIdentity, + ...(args.productionQuality ? { productionQuality: true } : {}), + continuationMessage: PROVIDER_CONTINUATION_MESSAGE }; + let coding: CodingSessionRetryResult; + try { + // Send prompts through stdin to avoid the Windows command-line limit. + const cliEnv = { ...process.env, + // Absent unless deliberately overridden — see THINKING_TOKENS above. + ...(args.provider === 'anthropic' && (args.thinking ?? THINKING_TOKENS) + ? { MAX_THINKING_TOKENS: String(args.thinking ?? THINKING_TOKENS) } + : {}), + // Keep the CLI fixed across the campaign. + ...(args.provider === 'anthropic' ? { DISABLE_AUTOUPDATER: '1', + // Pin cache lifetime so run order cannot change cost. + FORCE_PROMPT_CACHING_5M: '1' } : {}) }; + + coding = runCodingSessionWithRetries({ prompt, model: args.model, retryLimit, + maxBudgetUsd: args.maxBudgetUsd, + throttleMaxWaitMs: throttleMaxWaitMinutes * 60_000, + throttleJitterMs, + onInvocation: record => persistCampaignProviderInvocation({ + evidence: { ...continuationIdentity, ...record, billingMode: selectedAuthMode } }), + waitForProvider: continuationContext ? request => { + const nativeOptions = { appDir: args.app, provider: args.provider, + sessionId: request.sessionId, model: args.model, imageId: imageIdentity.id, env: process.env }; + const nativeIdentity = captureNativeContinuation(nativeOptions); + return waitForCampaignProviderContinuation({ + evidence: { ...continuationIdentity, ...request, nativeIdentity, billingMode: selectedAuthMode }, + validate: phase => { + const env = phase === 'continue' ? invocationEnvironment() : process.env; + if (captureNativeContinuation({ ...nativeOptions, env }) !== nativeIdentity) { + throw new Error('native provider session or runtime changed during provider wait'); + } + }, + }); + } : undefined, + invoke: ({ input, maxBudgetUsd, resumeSession, recoverStoppedContainer }) => + execFileSync(process.execPath, [ + compiledEntrypoint('container', 'run-build.js'), + '--provider', args.provider, + ...(args.providerRoute ? ['--provider-route', args.providerRoute] : []), + ...(args.maxOutputTokens ? ['--max-output-tokens', String(args.maxOutputTokens)] : []), + '--app', args.app, + '--backend', args.backend, + '--image', imageIdentity.id, + '--effort', EFFORT, + '--model', args.model, + ...(args.pricing ? ['--pricing-json', JSON.stringify(args.pricing)] : []), + '--completion-marker', args.mode === 'fix' ? 'FIX_COMPLETE' + : args.mode === 'upgrade' ? 'UPGRADE_COMPLETE' + : args.mode === 'resume' ? 'RESUME_COMPLETE' : 'DEPLOY_COMPLETE', + ...(maxBudgetUsd != null ? ['--max-budget-usd', String(maxBudgetUsd)] : []), + '--ports', [p.vite, p.express].filter(Boolean).join(','), + ...(resumeSession ? ['--resume-session', resumeSession] : []), + ...(recoverStoppedContainer ? ['--recover-stopped-container'] : []), + ], { input, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, + env: invocationEnvironment(cliEnv), + timeout: AGENT_PROCESS_TIMEOUT_MS }), + }); + } catch (err: unknown) { + coding = { raw: '', spawnError: codingSessionFailure(isRecord(err) ? err : {}), sessionResults: [], + interruptions: [], result: { total_cost_usd: 0, num_turns: 0, + usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, + cache_read_input_tokens: 0 }, stack_bench_cost_receipts: [] }, + throttle: { waits: 0, waitedMs: 0, maxWaitMs: throttleMaxWaitMinutes * 60_000, + jitterMs: throttleJitterMs } }; + } + + const { raw, spawnError, sessionResults, interruptions, result, throttle } = coding; + const noOutput = !result.session_id && !raw.trim(); + const failed = Boolean(spawnError || noOutput); + const providerFailure = providerSessionFailure(result); + const usage = sessionUsage(result.usage); + const input = usage.input_tokens ?? 0; + const output = usage.output_tokens ?? 0; + const cacheWrite = usage.cache_creation_input_tokens ?? 0; + const cacheRead = usage.cache_read_input_tokens ?? 0; + const turns = result.num_turns ?? 0; + + // Preserve the cost inputs needed to explain stack differences. + const setupMetadata = adapter.agent.setupMetadata({ + imageId: imageIdentity.id, + localPackage: LOCAL_PKG, + env: process.env, + helpers: { linuxSpacetimeVersion, bindingsIdentity, containerImage }, + }); + const out = { + ...(args.productionQuality ? { productionQuality: true } : {}), + appDir: args.app, + mode: args.mode, + level: args.level, + track: args.track, + backend: args.backend, + model: args.model, + guidance: args.guidance, + setup: { + provider: args.provider, + ...(args.providerRoute ? { providerRoute: args.providerRoute } : {}), + ...(args.maxOutputTokens ? { maxOutputTokens: args.maxOutputTokens } : {}), + thinkingTokens: args.provider !== 'anthropic' ? null : (args.thinking ?? THINKING_TOKENS) ? Number(args.thinking ?? THINKING_TOKENS) : 'cli default', + permissionMode: args.provider === 'anthropic' ? 'acceptEdits' : 'container-isolated', + effort: EFFORT, + skills: selectedSkills, + cacheTier: args.provider === 'anthropic' ? '5m' : 'provider-managed', + autoUpdater: 'disabled', + codingInterruptionRetries: { limit: retryLimit, + used: interruptions.filter(item => item.kind !== 'provider-throttled').length }, + providerThrottle: { maxWaitMinutes: throttleMaxWaitMinutes, + waits: throttle?.waits ?? 0, waitedMs: throttle?.waitedMs ?? 0, + jitterMs: throttle?.jitterMs ?? throttleJitterMs }, + cliVersion: imageCliVersion(imageIdentity.id, CODING_PROVIDERS[args.provider].executable), + isolation: { mode: 'container', image: imageIdentity.reference, + imageId: imageIdentity.id, hostAlias: HOST_ADDR }, + auth: selectedAuthMode ?? 'not-selected', + ...(isRecord(setupMetadata) ? setupMetadata : {}), + env: ambientEnv(), + node: { orchestrator: process.version, codingContainer: imageNodeVersion(imageIdentity.id) }, + platform: process.platform, + resources: result.stack_bench_resources ?? null, + }, + costUsd: Number((result.total_cost_usd ?? 0).toFixed(6)), + costReceipts: result.stack_bench_cost_receipts ?? [], + ...(result.stack_bench_unaccounted_invocations ? { unaccountedInvocations: result.stack_bench_unaccounted_invocations } : {}), + tokens: input + output + cacheWrite + cacheRead, + outputTokens: output, + usage: { input, output, cacheWrite, cacheRead }, + provenance, + turns, + promptBytes: Buffer.byteLength(prompt), + tokensPerTurn: turns ? Math.round((input + output + cacheWrite + cacheRead) / turns) : null, + thinking: args.provider === 'anthropic' + ? combinedThinkingVolume(args.app, sessionResults.map(item => item.session_id)) : null, + durationMs: Date.now() - started, + sessionId: result.session_id ?? null, + ok: !failed && result.is_error === false, + providerMetadata: { failureCode: failed + ? String(spawnError ?? '').startsWith('provider stayed throttled') + ? 'provider-throttle-exhausted' + : providerFailure?.code ?? (noOutput ? 'coding-session-no-output' : 'coding-session-failed') + : result.is_error === true ? 'provider-session-error' : null, + diagnostic: spawnError, + failure: failed ? { + providerStatus: result.api_error_status ?? null, + waitedMs: throttle?.waitedMs ?? 0, + waits: throttle?.waits ?? 0, + ...(result.stack_bench_provider_failure?.budget ? { budget: result.stack_bench_provider_failure.budget } : {}), + } : null, + interruptions, invocations: sessionResults.length, + providerWaits: coding.providerWaits ?? [], + terminalRecovery: isRecord(result) ? result.terminal_recovery ?? null : null, + credentialBroker: result.stack_bench_credential_broker ?? null, + sessionIds: [...new Set(sessionResults.map(item => item.session_id).filter(Boolean))], + models: args.provider === 'anthropic' + ? transcriptModels(args.app, sessionResults.map(item => item.session_id)) : [] }, + }; + console.log(JSON.stringify(out)); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(err => { console.error(err); process.exit(1); }); +} diff --git a/tools/stack-bench/commands/bench-arguments.ts b/tools/stack-bench/commands/bench-arguments.ts new file mode 100644 index 00000000000..d1e442a215e --- /dev/null +++ b/tools/stack-bench/commands/bench-arguments.ts @@ -0,0 +1,368 @@ +import { dirname, resolve } from 'node:path'; +import { parseArgs } from 'node:util'; +import { readArtifact } from '../src/evidence/artifacts.js'; +import { DEFAULT_TRACK, RUN_INDEX_CAP } from '../src/composition/tracks.js'; +import { campaignProgressionOwner, validateCompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptPlan, CampaignSelection } + from '../src/campaigns/campaign-compiler.js'; +import { readCampaignAdmission } + from '../src/campaigns/campaign-admission.js'; +import { compileProgressionInput, dependencyRuntimeDefinition, progressionLevels, + validateFeatureCatalogInput, validateProgressionInput } + from '../src/progression/progression-definition.js'; +import type { CompiledDependencyPolicyDefinition, CompiledProgressionDefinition, + ProgressionInput } from '../src/progression/progression-definition.js'; +import { validatePricingAuthority } from '../src/evidence/pricing-authority.js'; +import type { PricingAuthority } from '../src/evidence/pricing-authority.js'; +import { parseGuidanceMode } from '../src/campaigns/condition-compiler.js'; +import type { GuidanceMode } from '../src/campaigns/condition-compiler.js'; +import { validateCampaignExtensionSeed } from '../src/campaigns/campaign-scheduler.js'; +import type { CampaignExtensionSeed } from '../src/campaigns/campaign-scheduler.js'; +import { repairBudgetLimit } from '../src/progression/repair-plan.js'; + +type StudyCondition = CampaignAttemptPlan['condition']; +type UnknownRecord = Record; + +export interface BenchArguments { + backend?: string; + track: string; + levels: string; + levelsProvided: boolean; + levelList: number[]; + model: string | null; + providerRoute?: string; + maxOutputTokens?: number; + agentAdapter: string; + pricing?: PricingAuthority | null; + repairs: number; + maxStalledRepairs: number; + maxBudgetUsd?: number; + runIndex: number; + out?: string; + app?: string; + url?: string; + media: boolean; + retainBackend?: boolean; + guidance: GuidanceMode; + productionQuality?: boolean; + guidanceDocument?: unknown; + condition?: StudyCondition; + /** Aliases the grader expects instead of the condition's; `{}` grades with the fixture credentials. */ + gradingCredentialAliases?: Record; + selectionRequest?: CampaignSelection; + taskMode?: string; + retainPriorContracts?: boolean; + pauseAfterDepth?: number; + packIds: string[]; + checkKeys: string[]; + featureIds: string[]; + requestedSpecifications: string[]; + expectedSpecifications: string[]; + observedSpecifications: string[]; + skills?: string[]; + apiKey?: string; + apiKeyFile?: string; + mutations?: string; + mutationShardIndex?: number; + mutationShardCount?: number; + mutationResumeFrom?: string; + mutationCheckpointOut?: string; + mutationBaselineBundle?: string; + expectedMutationCalibration?: unknown; + mutationMaxRuntimeMinutes: number; + referenceMutationOnly?: boolean; + seedFrom?: string; + seedThrough?: number; + progressionSeed?: CampaignExtensionSeed; + parentAttemptId?: string; + repairFrom?: string; + gradeFrom?: string; + gradeLevel?: number; + repairLevel?: number; + recipe?: string; + campaignFile?: string; + campaignAttemptId?: string; + campaignAdmissionId?: string; + progressionResumeFrom?: string; + experimentIdentity?: { id: string; version: string; sha256: string; state: string }; + runMode?: CampaignAttemptPlan['mode']; + featureCatalog?: ProgressionInput; + dependencyPolicy?: ProgressionInput; + progression?: ProgressionInput; + progressionOwner?: UnknownRecord; +} + +interface BenchCliOptions extends Partial { + pack?: string[]; + check?: string[]; + pricingJson?: unknown; + featureModule?: string[]; + requestSpec?: string[]; + expectSpec?: string[]; + observeSpec?: string[]; + expectedMutationCalibrationJson?: unknown; + progressionSeedJson?: unknown; +} + +function parseCli(argv: readonly string[]): BenchCliOptions { + const strings = ['backend', 'track', 'levels', 'campaign-file', 'campaign-attempt-id', + 'campaign-admission-id', 'progression-resume-from', 'recipe', 'model', 'provider-route', 'max-output-tokens', 'pricing-json', + 'repairs', 'max-stalled-repairs', 'max-budget-usd', 'run-index', 'out', 'app', 'url', + 'agent-adapter', 'guidance', 'task-mode', 'skills', 'mutations', + 'mutation-shard-index', 'mutation-shard-count', 'mutation-resume-from', + 'mutation-checkpoint-out', 'mutation-baseline-bundle', + 'expected-mutation-calibration-json', 'mutation-max-runtime-minutes', 'seed-from', + 'seed-through', 'progression-seed-json', + 'parent-attempt-id', 'repair-from', 'repair-level', 'grade-from', 'grade-level'] as const; + const multiple = ['pack', 'check', 'feature-module', 'request-spec', 'expect-spec', + 'observe-spec'] as const; + const options = Object.fromEntries([ + ...strings.map(name => [name, { type: 'string' as const }]), + ...multiple.map(name => [name, { type: 'string' as const, multiple: true }]), + ...['no-media', 'retain-backend', 'reference-mutation-only', 'production-quality', 'no-production-quality'].map(name => + [name, { type: 'boolean' as const }]), + ]); + const { values } = parseArgs({ args: [...argv.slice(2)], options, strict: true, + allowPositionals: false }); + const parsed: Record = {}; + for (const [key, value] of Object.entries(values)) { + parsed[key.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase())] = value; + } + for (const key of ['pack', 'check', 'featureModule', 'requestSpec', 'expectSpec', + 'observeSpec']) { + const value = parsed[key] as string[] | undefined; + if (value) parsed[key] = value.flatMap(item => item.split(',').filter(Boolean)); + } + for (const key of ['repairs', 'maxStalledRepairs', 'maxBudgetUsd', 'maxOutputTokens', 'mutationShardIndex', + 'mutationShardCount', 'mutationMaxRuntimeMinutes', 'repairLevel', 'gradeLevel', 'seedThrough']) { + if (typeof parsed[key] === 'string') parsed[key] = Number(parsed[key]); + } + if (typeof parsed.runIndex === 'string') parsed.runIndex = Number(parsed.runIndex); + for (const key of ['campaignFile', 'progressionResumeFrom', 'mutations', + 'mutationResumeFrom', 'mutationCheckpointOut', 'mutationBaselineBundle', 'repairFrom', 'gradeFrom']) { + if (typeof parsed[key] === 'string') parsed[key] = resolve(parsed[key]); + } + for (const key of ['pricingJson', 'expectedMutationCalibrationJson', 'progressionSeedJson']) { + if (typeof parsed[key] === 'string') parsed[key] = JSON.parse(parsed[key]); + } + if (typeof parsed.guidance === 'string') parsed.guidance = parseGuidanceMode(parsed.guidance); + if (typeof parsed.skills === 'string') parsed.skills = parsed.skills.split(',').filter(Boolean); + if (parsed.noMedia === true) parsed.media = false; + delete parsed.noMedia; + if (parsed.productionQuality && parsed.noProductionQuality) throw new Error('choose only one production-quality flag'); + if (parsed.noProductionQuality) parsed.productionQuality = false; + delete parsed.noProductionQuality; + return parsed as BenchCliOptions; +} + +export function parseBenchArguments(argv: readonly string[]): BenchArguments { + const args: BenchArguments = { model: null, agentAdapter: 'claude-code', + repairs: 10, runIndex: 0, levels: '1', levelsProvided: false, media: true, + levelList: [], maxStalledRepairs: 3, guidance: 'prescribed', productionQuality: true, track: DEFAULT_TRACK, + packIds: [], checkKeys: [], featureIds: [], requestedSpecifications: [], + expectedSpecifications: [], observedSpecifications: [], + mutationMaxRuntimeMinutes: 60 }; + const { pack, check, pricingJson, featureModule, requestSpec, expectSpec, observeSpec, + expectedMutationCalibrationJson, progressionSeedJson, ...options } = parseCli(argv); + Object.assign(args, options); + if (args.gradeLevel !== undefined && (!args.gradeFrom + || !Number.isSafeInteger(args.gradeLevel) || args.gradeLevel < 1)) { + throw new Error('--grade-level requires --grade-from and a positive integer depth'); + } + for (const flag of ['--grade-from', '--grade-level']) { + if (argv.slice(2).filter(value => value.split('=', 1)[0] === flag).length > 1) { + throw new Error(`${flag} must be supplied only once`); + } + } + if (args.gradeFrom) { + const allowed = new Set(['--grade-from', '--grade-level', '--out', '--no-media', '--check']); + const forbidden = argv.slice(2).find(value => value.startsWith('--') + && !allowed.has(value.split('=', 1)[0]!)); + if (forbidden) throw new Error(`--grade-from cannot be combined with ${forbidden}`); + if (!args.out) throw new Error('--grade-from requires a separate --out directory'); + args.out = resolve(args.out); + args.repairs = 0; + } + if (pack) args.packIds = pack; + if (check) args.checkKeys = check; + if (pricingJson !== undefined) { + args.pricing = validatePricingAuthority(pricingJson, { at: '--pricing-json' }); + } + if (featureModule) args.featureIds = featureModule; + if (requestSpec) args.requestedSpecifications = requestSpec; + if (expectSpec) args.expectedSpecifications = expectSpec; + if (observeSpec) args.observedSpecifications = observeSpec; + args.levelsProvided = options.levels !== undefined; + if (expectedMutationCalibrationJson !== undefined) { + args.expectedMutationCalibration = expectedMutationCalibrationJson; + } + if (progressionSeedJson !== undefined) { + args.progressionSeed = validateCampaignExtensionSeed(progressionSeedJson); + } + if ((args.mutationResumeFrom || args.mutationCheckpointOut || args.mutationBaselineBundle) + && !args.mutations) { + throw new Error('mutation control options require --mutations'); + } + if (args.expectedMutationCalibration && !args.mutations) { + throw new Error('--expected-mutation-calibration-json requires --mutations'); + } + if (!Number.isFinite(args.mutationMaxRuntimeMinutes) || args.mutationMaxRuntimeMinutes < 1 + || args.mutationMaxRuntimeMinutes > 120) { + throw new Error('--mutation-max-runtime-minutes must be from 1 through 120'); + } + if (args.referenceMutationOnly && (!args.mutations || args.agentAdapter !== 'reference-fixture' + || args.repairs !== 0 || !args.app || args.campaignFile)) { + throw new Error('--reference-mutation-only requires a mutation-bound reference fixture run'); + } + if (args.mutationBaselineBundle && !args.referenceMutationOnly) { + throw new Error('--mutation-baseline-bundle is an internal reference mutation option'); + } + if (args.repairFrom && (args.repairLevel === undefined + || !Number.isSafeInteger(args.repairLevel) || args.repairLevel < 1)) { + throw new Error('--repair-from requires --repair-level with a positive integer'); + } + if (args.campaignFile && !args.campaignAttemptId) { + throw new Error('--campaign-file requires --campaign-attempt-id'); + } + if (!args.campaignFile && (args.campaignAttemptId || args.campaignAdmissionId)) { + throw new Error('campaign binding requires --campaign-file'); + } + if (args.progressionResumeFrom && !args.campaignFile) { + throw new Error('--progression-resume-from requires a compiled campaign'); + } + if (args.seedThrough !== undefined && (!args.seedFrom || !args.campaignFile)) { + throw new Error('--seed-through requires --seed-from and a compiled campaign'); + } + if (args.seedThrough !== undefined && !args.progressionSeed) { + throw new Error('--seed-through requires --progression-seed-json'); + } + if (args.progressionSeed !== undefined && args.seedThrough === undefined) { + throw new Error('--progression-seed-json requires --seed-through'); + } + if (args.progressionSeed && args.progressionSeed.fromDepth !== args.seedThrough) { + throw new Error('--progression-seed-json does not match --seed-through'); + } + if (args.campaignFile) { + const allowed = new Set(['--campaign-file', '--campaign-attempt-id', + '--campaign-admission-id', '--progression-resume-from', '--run-index', '--out', + '--max-budget-usd', '--seed-from', '--seed-through', '--progression-seed-json']); + const override = argv.slice(2).find(value => value.startsWith('--') + && !allowed.has(value.split('=', 1)[0]!)); + if (override) throw new Error(`campaign attempts cannot override ${override}`); + bindCampaign(args); + } + if (!args.backend && !args.repairFrom && !args.gradeFrom) { + throw new Error('--backend is required unless --repair-from, --grade-from, or --campaign-file is supplied'); + } + if (args.progression) { + if (args.levelsProvided) throw new Error('--levels cannot be combined with progression input'); + args.progression = validateProgressionInput(args.progression); + args.levelList = progressionLevels(args.progression); + args.levels = `${args.levelList[0]}-${args.levelList.at(-1)}`; + const seedThrough = args.seedThrough; + if (seedThrough !== undefined && (!args.levelList.includes(seedThrough) + || !args.levelList.some(level => level > seedThrough))) { + throw new Error('--seed-through must precede another planned dependency depth'); + } + if (seedThrough !== undefined + && args.dependencyPolicy?.definition.workSelection !== 'progressive') { + throw new Error('--seed-through requires progressive dependency work selection'); + } + } else { + const [fromText, toText] = args.levels.split('-'); + const from = Number(fromText); + const to = toText === undefined ? from : Number(toText); + if (!Number.isSafeInteger(from) || from < 1 || !Number.isSafeInteger(to) || to < from) { + throw new Error('--levels must be one positive level or an ascending range'); + } + args.levelList = Array.from({ length: (to ?? from) - from + 1 }, (_, index) => from + index); + if (args.seedThrough !== undefined) { + throw new Error('--seed-through requires dependency mode'); + } + } + if (args.recipe && args.levelList.length !== 1) { + throw new Error('--recipe requires exactly one requested level'); + } + if (!Number.isSafeInteger(args.repairs) || args.repairs < 0) { + throw new Error('--repairs must be a non-negative safe integer'); + } + if (!Number.isInteger(args.maxStalledRepairs) || args.maxStalledRepairs < 0 + || args.maxStalledRepairs > 20) { + throw new Error('--max-stalled-repairs must be an integer from 0 through 20'); + } + if (args.maxBudgetUsd !== undefined + && (!Number.isFinite(args.maxBudgetUsd) || args.maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (!Number.isSafeInteger(args.runIndex) || args.runIndex < 0 || args.runIndex > RUN_INDEX_CAP) { + throw new Error(`--run-index must be an integer from 0 through ${RUN_INDEX_CAP}`); + } + if ((args.mutationShardIndex === undefined) !== (args.mutationShardCount === undefined)) { + throw new Error('--mutation-shard-index and --mutation-shard-count must be supplied together'); + } + return args; +} + +function bindCampaign(args: BenchArguments): void { + if (!args.campaignFile) throw new Error('campaign file is required'); + const artifact = readArtifact(args.campaignFile, { expectedKind: 'campaign_plan' }); + const plan = validateCompiledCampaignPlan(artifact.payload); + const attempt = plan.attempts.find(item => item.id === args.campaignAttemptId); + if (!attempt) throw new Error('--campaign-attempt-id is not in the compiled campaign plan'); + const plannedBudget = plan.definition.budgets.maxCostUsdPerAttempt; + if (args.maxBudgetUsd !== undefined + && (plannedBudget === null || args.maxBudgetUsd > plannedBudget)) { + throw new Error('--max-budget-usd exceeds the compiled campaign budget'); + } + args.backend = attempt.stack; + args.track = plan.definition.track; + args.model = attempt.model; + if (args.providerRoute !== undefined && args.providerRoute !== attempt.providerRoute) { + throw new Error('--provider-route differs from the compiled campaign'); + } + args.providerRoute = attempt.providerRoute; + if (args.maxOutputTokens !== undefined && args.maxOutputTokens !== attempt.maxOutputTokens) { + throw new Error('--max-output-tokens differs from the compiled campaign'); + } + args.maxOutputTokens = attempt.maxOutputTokens; + args.agentAdapter = attempt.agentAdapter; + args.pricing = validatePricingAuthority(attempt.pricing, { at: 'compiled campaign pricing' }); + args.guidance = parseGuidanceMode(attempt.guidance); + args.condition = structuredClone(attempt.condition); + args.productionQuality = attempt.condition.productionQuality === true; + args.skills = structuredClone(attempt.skills); + args.selectionRequest = structuredClone(plan.definition.selection); + args.guidanceDocument = structuredClone( + attempt.condition.guidance.documents[attempt.stack]); + args.packIds = structuredClone(plan.definition.selection.packs ?? []); + args.checkKeys = structuredClone(plan.definition.selection.checks ?? []); + args.repairs = attempt.mode.id === 'dependency' + ? 0 : repairBudgetLimit(plan.definition.repair); + args.maxBudgetUsd ??= plannedBudget ?? undefined; + args.parentAttemptId = attempt.id; + args.media = false; + args.levels = `${Math.min(...attempt.levels)}-${Math.max(...attempt.levels)}`; + args.experimentIdentity = { + id: plan.id, version: plan.version, sha256: plan.contentSha256, state: plan.state, + }; + if (args.campaignAdmissionId) { + const admission = readCampaignAdmission(dirname(args.campaignFile), + args.campaignAdmissionId, plan); + if (!admission.ok) throw new Error('campaign admission did not pass'); + } + args.runMode = structuredClone(attempt.mode); + if (plan.featureCatalog) { + args.featureCatalog = validateFeatureCatalogInput(plan.featureCatalog); + } + if (attempt.mode.id === 'dependency') { + args.pauseAfterDepth = attempt.mode.pauseAfterDepth; + args.retainPriorContracts = attempt.mode.retainPriorContracts === true; + if (!plan.dependencyPolicy || !args.featureCatalog) { + throw new Error('dependency campaign requires a feature catalog and dependency policy'); + } + args.dependencyPolicy = plan.dependencyPolicy; + args.progression = compileProgressionInput(dependencyRuntimeDefinition( + args.featureCatalog, args.dependencyPolicy)); + args.progressionOwner = { ...campaignProgressionOwner(plan, attempt) }; + } +} diff --git a/tools/stack-bench/commands/bench.ts b/tools/stack-bench/commands/bench.ts new file mode 100644 index 00000000000..130b147fa13 --- /dev/null +++ b/tools/stack-bench/commands/bench.ts @@ -0,0 +1,3156 @@ +#!/usr/bin/env node + +import { campaignProviderContinuationContext } from '../src/campaigns/campaign-provider-continuation.js'; +import { execFileSync } from 'node:child_process'; +import type { ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, mkdtempSync, existsSync, copyFileSync, cpSync, rmSync, readdirSync, realpathSync, lstatSync } from 'node:fs'; +import { join, dirname, resolve, relative, sep, isAbsolute } from 'node:path'; +import { tmpdir } from 'node:os'; +import { pathToFileURL } from 'node:url'; +import { loadTrack, resultsName, portsFor, workDirFor, + moduleName, dbName, suitesFor } from '../src/composition/tracks.js'; +import { parseBenchArguments } from './bench-arguments.js'; +import type { BenchArguments } from './bench-arguments.js'; +import { packageRegistry, packageRegistryEnvironment } from '../src/runtime/package-registry.js'; +import { runBounded } from '../src/runtime/bounded-process.js'; +import { formatRepairProgress } from '../src/evidence/scoring.js'; +import { ARTIFACT_FILE, emptyArtifactIdentities, readArtifact, readArtifactPayload, + writeArtifact, writeRunJson, currentEngineIdentity } from '../src/evidence/artifacts.js'; +import { aggregateRunOutcome, classifyBundle, ladderMayAdvance, ladderMayContinue, + mutationControlEligible, runExitCode, runOutcomeKind } from '../src/evidence/outcomes.js'; +import { summarizeSessions } from '../src/evidence/session-metrics.js'; +import { hashDirectory, sha256 } from '../src/evidence/provenance.js'; +import { createBackendLease, newRunId, publicBackendLease, readBackendLease, + claimBackendResourcesWhenAvailable, backendResourceLockKeys, resourceLockScope, loopbackHttpUri } from '../src/runtime/backend-lease.js'; +import { borrowCampaignReservation } + from '../src/campaigns/campaign-admission.js'; +import { captureApplicationDiagnostics } from '../src/runtime/backend-control.js'; +import type { RuntimeControlSpec } from '../src/runtime/backend-control.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createAgentVisibleTaskRequest, createBoundRecipeTaskRequest } + from '../src/composition/recipe-selection.js'; +import { criterionEvidence, evidencePassed } from '../src/evidence/check-evidence.js'; +import { leasedDatabaseEnvironment, STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { agentRecipeIdentity, agentRequestArgv, validateProviderRoute, validateProviderOutputLimit } from '../src/agents/agent-adapter-contract.js'; +import { agentSessionFailure, validateAgentResult } + from '../src/agents/agent-result-contract.js'; +import { AGENT_ADAPTER_REGISTRY, agentAdapterIdentity } from '../src/agents/agent-adapters.js'; +import { archiveTranscripts } from '../src/agents/transcript-archive.js'; +import { runPreflight } from '../src/runtime/preflight.js'; +import { checkpointSchema, recordRunCheckpoint } from '../src/evidence/run-checkpoints.js'; +import type { RunCheckpoint } from '../src/evidence/run-checkpoints.js'; +import { runCostEvidence } from '../src/evidence/cost-proof.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { SUPERVISOR_STATE_VERSION, writeRecoveryArtifact } from '../src/runtime/recovery.js'; +import { applyAgentCredential } from '../src/agents/agent-credentials.js'; +import { assertPlainAppSourceTree, hashAppSource, resetAppToSource, seedAppSource, snapshotAppSource } from '../src/runtime/source-snapshot.js'; +import { finalPackageEvidenceRequired, preserveFinalPackageEvidence, preserveLevelCheckpoint, + sourceBoundFirstBuildOutcome } from '../src/runtime/source-checkpoint.js'; +import { materializationAppFailure, materializeAcceptedSource, restoreRepairSource } + from '../src/runtime/source-materialization.js'; +import { compareRepairBaseline, createRepairGrant } from '../src/runtime/repair-grant.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { contractInterfaceNames } from '../src/composition/agent-visible-contract.js'; +import { clearPrivateGradingEvidence, privateGradingDirectory, levelGradeIsUsable, repairEvidenceDecision, + repairHistoryEntry, repairProgressState, repairRegressionDecision, + restorePrivateGradingEvidence } + from '../src/evidence/repair-evidence.js'; +import { mutationControlArgv, mutationControlTimeoutMs, pristineMutationBaselinePath } + from '../src/evidence/mutation-control.js'; +import type { MutationControlArgs } from '../src/evidence/mutation-control.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { dependencyLevelRepairRecords, dependencyRepairBudget, dependencyRepairRecords, dependencyRepairStopReason } + from '../src/progression/dependency-mode.js'; +import { resolveProgressionRecipeAction, resolveProgressionRecipeLevelSelection, + resolveProgressionRepairTarget, validateProgressionCampaignLevelScope } + from '../src/progression/progression-recipe-selection.js'; +import { createLiveProgressionExecution, clearTimeContinuationBoundary, + commitTimeContinuationBoundary } + from '../src/progression/live-progression.js'; +import type { CampaignSelection } from '../src/campaigns/campaign-compiler.js'; +import { gradingRunTimeoutMs, selectedGradingSourceCount } + from '../src/runtime/grading-timeout.js'; +import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import type { BoundRecipeTaskRequestResult } from '../src/composition/recipe-selection.js'; +import type { RecipeBinding } from '../src/composition/recipe-release.js'; +import type { RepairGrantResolution, RepairOutcome } from '../src/runtime/repair-grant.js'; +import type { AgentAdapter, AgentMode, AgentRequest } + from '../src/agents/agent-adapter-contract.js'; +import type { ValidatedAgentResult } from '../src/agents/agent-result-contract.js'; +import type { Track } from '../src/composition/tracks.js'; +import type { RunOutcome } from '../src/evidence/outcomes.js'; +import type { GradeBundlePayload, BenchmarkRunRecord, RunLevelRecord, + RunContinuation, RunRepairCandidate, RunSessionRecord, RunTotals } + from '../src/evidence/benchmark-run.js'; +import { depthPauseDue, readDepthPauseContext, waitAtDepthBoundary } from '../src/campaigns/campaign-depth-pause.js'; +import { addCostUsd, finalizeRunTotals, runSessionRecord } + from '../src/evidence/benchmark-run.js'; +import { formatLevelSummary } from '../src/evidence/evidence-presentation.js'; +import type { ProgressionAction } from '../src/progression/progression-engine.js'; +import type { ProgressionRepairRegression, ProgressionState } + from '../src/progression/progression-state.js'; +import type { ProgressionRecipeAction, ProgressionRecipeSelections } + from '../src/progression/progression-recipe-selection.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import { runningContainerIdentity } from '../src/runtime/container-identity.js'; +const COMMAND_TIMEOUT_MS = 20 * 60_000; + +type UnknownRecord = Record; +type ContaminationAudit = { kind: 'contaminated' | 'harness_failure'; evidence: string[]; + verdict: string }; +type LeakAuditEntry = { hits: Array<{ kind: string; path: string }> }; +type CommandFailure = Error & { stdout?: string | Buffer; stderr?: string | Buffer; + status?: number | null; signal?: NodeJS.Signals | null }; +type GradeOptions = { observation?: 'scored' | 'observed'; out?: string | null; + sourceSha256?: string | null; applicationFailure?: RunOutcome | null; + recipeTask?: GradeRecipeTask }; +type MutationControlResult = UnknownRecord & { ok: boolean; artifact?: string; + skipped?: boolean; processError?: string | null; outcome: RunOutcome | null }; +type RecipeTask = (BoundRecipeTaskRequestResult | ProgressionRecipeSelections['grader']) & { agentRequest?: UnknownRecord; + progressionAction?: ProgressionAction }; +type BenchArgs = BenchArguments & { + recipeTasks: Map; + recipeBindings: Map; + repairGrant?: RepairGrantResolution; + mutationImageId?: string; + spentBudgetUsd?: number; +}; +type ProgressionWorkRecipeAction = ProgressionRecipeSelections & { + action: Exclude; +}; +type FirstBuildRecord = { + score: number | null; + max: number | null; + regression: NonNullable['regression'] | null; + contractPass: boolean | null; + outcome: RunOutcome; + source: { sha256: string; files: number } | null; + missed: string[]; + observations?: UnknownRecord; +}; +type RepairStatus = 'not-needed' | 'corrected' | 'budget-exhausted' | 'incomplete' | 'ungraded'; +type ProgressionFailure = { kind?: string; reason?: string }; + +const object = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const errorMessage = (error: unknown): string => + redactCredentials(error instanceof Error ? error.message : String(error)); + +function commandFailure(error: unknown): CommandFailure { + if (error instanceof Error) return error; + throw error; +} + +function parseLeakAudit(value: string): LeakAuditEntry[] { + const parsed: unknown = JSON.parse(value); + if (!Array.isArray(parsed)) throw new Error('contamination audit output must be an array'); + return parsed.map((entry, index) => { + if (!object(entry) || !Array.isArray(entry.hits)) { + throw new Error(`contamination audit output[${index}] is invalid`); + } + const hits = entry.hits.map((hit, hitIndex) => { + if (!object(hit) || typeof hit.kind !== 'string' || typeof hit.path !== 'string') { + throw new Error(`contamination audit output[${index}].hits[${hitIndex}] is invalid`); + } + return { kind: hit.kind, path: hit.path }; + }); + return { hits }; + }); +} + +function stringArray(value: unknown, at: string): string[] { + if (!Array.isArray(value) || value.some(entry => typeof entry !== 'string')) { + throw new Error(`${at} must be an array of strings`); + } + return [...value]; +} + +function campaignSelection(value: unknown, at: string): CampaignSelection { + if (!object(value)) throw new Error(`${at} must be an object`); + const optionalStrings = (field: 'packs' | 'checks'): string[] | undefined => { + const entry = value[field]; + if (entry === undefined) return undefined; + return stringArray(entry, `${at}.${field}`); + }; + let levels: CampaignSelection['levels']; + if (value.levels !== undefined) { + if (!Array.isArray(value.levels)) throw new Error(`${at}.levels must be an array`); + levels = value.levels.map((entry, index) => { + if (!object(entry)) throw new Error(`${at}.levels[${index}] is invalid`); + const level = entry.level; + const recipe = entry.recipe; + if (typeof level !== 'number' || !Number.isSafeInteger(level) || typeof recipe !== 'string') { + throw new Error(`${at}.levels[${index}] is invalid`); + } + return { level, recipe, + ...(entry.features === undefined ? {} : { features: stringArray(entry.features, + `${at}.levels[${index}].features`) }), + ...(entry.checks === undefined ? {} : { checks: stringArray(entry.checks, + `${at}.levels[${index}].checks`) }) }; + }); + } + return { ...(optionalStrings('packs') === undefined ? {} : { packs: optionalStrings('packs') }), + ...(optionalStrings('checks') === undefined ? {} : { checks: optionalStrings('checks') }), + ...(levels === undefined ? {} : { levels }) }; +} + +function isProgressionWorkRecipeAction(value: ProgressionRecipeAction): + value is ProgressionWorkRecipeAction { + return value.action.type !== 'terminal'; +} + +function repairCheckKeys(value: ProgressionRecipeAction | null): string[] { + if (!value || !isProgressionWorkRecipeAction(value) || value.action.type !== 'repair') return []; + if (!object(value.action.prompt) || !Array.isArray(value.action.prompt.nodeIds) + || !object(value.action.grading) || !Array.isArray(value.action.grading.checks)) { + throw new Error('dependency repair action has invalid prompt or grading selections'); + } + const promptNodeIds = new Set(value.action.prompt.nodeIds.map(nodeId => { + if (typeof nodeId !== 'string' || !nodeId) { + throw new Error('dependency repair action has an invalid prompt node'); + } + return nodeId; + })); + const checks = value.action.grading.checks.flatMap(check => { + if (!object(check) || typeof check.id !== 'string' || !check.id + || typeof check.nodeId !== 'string' || !check.nodeId) { + throw new Error('dependency repair action has an invalid grading check'); + } + return promptNodeIds.has(check.nodeId) ? [check.id] : []; + }); + if (checks.length === 0) throw new Error('dependency repair action selects no repair checks'); + return checks; +} + +function repairReportArgs(value: ProgressionRecipeAction | null): string[] { + const checks = repairCheckKeys(value); + if (!value || !isProgressionWorkRecipeAction(value) || checks.length === 0) return []; + const interfaces = contractInterfaceNames(value.agent.task.contractText); + return ['--checks-json', JSON.stringify(checks), + '--controls-json', JSON.stringify(interfaces)]; +} + +function repairOwnerNodeIds(value: ProgressionRecipeAction | null): string[] { + if (!value || !isProgressionWorkRecipeAction(value) || value.action.type !== 'repair') return []; + return [...value.action.repair.nodeIds].sort(); +} + +function savedRepairRegression(state: ProgressionState | null, + selected: ProgressionRecipeAction | null): ProgressionRepairRegression | null { + const saved = state?.attempts.at(-1)?.repairRegression; + if (!saved) return null; + const owners = repairOwnerNodeIds(selected); + return JSON.stringify([...saved.ownerNodeIds].sort()) === JSON.stringify(owners) + ? structuredClone(saved) : null; +} + +function requireProgressionState(state: ProgressionState | null): ProgressionState { + if (!state) throw new Error('live dependency progression has no active state'); + return state; +} + +function requireContinuation(run: BenchmarkRunRecord): RunContinuation { + if (!run.continuation) throw new Error('repair continuation has no continuation record'); + return run.continuation; +} + +function requireRunTotals(run: BenchmarkRunRecord): RunTotals { + if (!run.totals) throw new Error('benchmark run totals are not available'); + return run.totals; +} + +function repairOutcome(outcome: RunOutcome): RepairOutcome { + return { kind: outcome.kind, appFailures: [...(outcome.appFailures ?? [])], + inconclusive: [...(outcome.inconclusive ?? [])], + harnessFailures: [...(outcome.harnessFailures ?? [])] }; +} + +function progressionFailure(outcome: RunOutcome): ProgressionFailure { + return { kind: outcome.kind, ...(outcome.reason === null || outcome.reason === undefined + ? {} : { reason: outcome.reason }) }; +} + +function featureCheckKeys(selected: ProgressionWorkRecipeAction, + state: ProgressionState): string[] { + if (!object(selected.action.prompt) || !Array.isArray(selected.action.prompt.nodeIds)) { + throw new Error('feature action has no prompt nodes'); + } + const selectedNodes = new Set(selected.action.prompt.nodeIds); + return state.definition.nodes + .filter(node => selectedNodes.has(node.id)) + .flatMap(node => node.gradingChecks + .filter(check => check.role === 'feature') + .map(check => check.id)); +} + +function bundlePassedChecks(bundle: GradeBundlePayload | null): Set { + return new Set(Object.values(bundle?.suites ?? {}).flatMap(suite => + (suite?.features ?? []).flatMap(feature => + (feature.criteria ?? []).flatMap(criterion => + typeof criterion.stableKey === 'string' + && evidencePassed(criterionEvidence(criterion)) ? [criterion.stableKey] : [])))); +} + +function featureCandidateAccepted(selected: ProgressionWorkRecipeAction, + state: ProgressionState, candidate: GradeBundlePayload | null): boolean { + if (!levelGradeIsUsable(classifyBundle(candidate))) return false; + const passed = bundlePassedChecks(candidate); + if (!featureCheckKeys(selected, state).every(check => passed.has(check))) return false; + return state.definition.nodes.every(node => node.gradingChecks.every(check => + state.nodes[node.id]?.checks[check.id] !== 'pass' || passed.has(check.id))); +} + +function featureActionNeedsCoding(selected: ProgressionWorkRecipeAction, + state: ProgressionState): boolean { + if (selected.action.type === 'repair') return true; + if (!object(selected.action.prompt) || !Array.isArray(selected.action.prompt.nodeIds)) { + throw new Error('feature action has no prompt nodes'); + } + return selected.action.prompt.nodeIds.some(nodeId => { + const node = state.nodes[String(nodeId)]; + return node?.status === 'active' + && Object.values(node.checks).every(outcome => outcome === null); + }); +} + +export function synchronizeProgressionSummary(run: Pick, + state: ProgressionState): void { + run.validation.ladder.completedLevels = [...new Set(state.attempts + .filter(attempt => attempt.outcome === 'conclusive').map(attempt => attempt.level))]; + for (const level of run.levels) { + const latest = state.attempts.findLast(attempt => attempt.level === level.level); + if (latest?.outcome === 'inconclusive') { + level.graded = false; + level.score = null; + level.max = null; + level.selection = latest.selectionSha256 ? { sha256: latest.selectionSha256 } : null; + if (level.outcome?.reason !== latest.reason + || ['passed', 'app_failure'].includes(level.outcome.kind)) { + const kind = latest.category === 'interrupted' ? 'ungraded' + : latest.category === 'inconclusive_evidence' ? 'inconclusive' + : runOutcomeKind(latest.category ?? 'harness_failure'); + level.outcome = { kind, phase: 'progression', reason: latest.reason ?? null, + appFailures: [], inconclusive: [], + harnessFailures: kind === 'harness_failure' ? [latest.reason ?? 'progression failed'] : [] }; + } + } + if (level.repair) { + const used = state.attempts.filter(attempt => attempt.level === level.level && attempt.repair).length; + level.repairs = used - (level.priorRepairs ?? 0); + level.repair.used = used; + level.repair.limit = Math.max(level.repair.limit, used); + level.repair.nodeRepairs = dependencyLevelRepairRecords(state, level.level); + if (level.cumulativeRepairs !== undefined) level.cumulativeRepairs = used; + if (latest?.outcome === 'inconclusive') level.repair.status = 'ungraded'; + else if (level.outcome.kind === 'passed') { + level.repair.status = used ? 'corrected' : 'not-needed'; + level.repair.stopReason = used ? 'passed' : 'not-needed'; + level.stalled = false; + } + } + } +} + +export function mergeFeatureLevelRecord(previous: RunLevelRecord | null, + current: RunLevelRecord): RunLevelRecord { + if (!previous) return current; + const buildSessions: RunSessionRecord[] = [ + ...(previous.buildSessions ?? []), + ...(current.buildSessions ?? []), + ]; + const repairSessions = [...(previous.repairSessions ?? []), ...(current.repairSessions ?? [])]; + const sessionTotals = summarizeSessions([...buildSessions, + ...(current.resumeSession ?? previous.resumeSession ? [current.resumeSession ?? previous.resumeSession!] : []), + ...repairSessions]); + const repairs = (previous.repairs ?? 0) + (current.repairs ?? 0); + const repair = current.repair ? { + ...current.repair, + limit: Math.max(previous.repair?.limit ?? 0, + (previous.repairs ?? 0) + current.repair.limit), + used: repairs, + } : previous.repair; + const merged: RunLevelRecord = { + ...previous, + ...current, + buildSessions, + buildCostUsd: addCostUsd(previous.buildCostUsd, current.buildCostUsd), + repairSessions, + repairCostUsd: addCostUsd(previous.repairCostUsd, current.repairCostUsd), + repairHistory: [...(previous.repairHistory ?? []), ...(current.repairHistory ?? [])], + repairs, + repair, + sessionTotals, + tokens: sessionTotals.tokens, + usage: sessionTotals.usage, + turns: sessionTotals.turns, + promptBytes: sessionTotals.promptBytes, + tokensPerTurn: sessionTotals.turns + ? Math.round(sessionTotals.tokens / sessionTotals.turns) : null, + thinking: sessionTotals.thinking, + costUsd: addCostUsd(previous.costUsd, current.costUsd), + durationSec: (previous.durationSec ?? 0) + (current.durationSec ?? 0), + }; + return merged; +} + +function mutationControlArgs(args: BenchArgs): MutationControlArgs { + if (!args.out || !args.mutations || !args.backend || !args.parentAttemptId) { + throw new Error('mutation control has incomplete run identity'); + } + return { levelList: args.levelList, out: args.out, recipe: args.recipe, + recipeTasks: args.recipeTasks, mutations: args.mutations, backend: args.backend, + track: args.track, runIndex: args.runIndex, parentAttemptId: args.parentAttemptId, + mutationShardIndex: args.mutationShardIndex, mutationShardCount: args.mutationShardCount, + mutationResumeFrom: args.mutationResumeFrom, mutationCheckpointOut: args.mutationCheckpointOut, + mutationBaselineBundle: args.mutationBaselineBundle, + expectedMutationCalibration: args.expectedMutationCalibration, + mutationMaxRuntimeMinutes: args.mutationMaxRuntimeMinutes, + mutationImageId: args.mutationImageId }; +} + +function mutationOutcome(value: unknown): RunOutcome | null { + if (value === null || value === undefined) return null; + if (!object(value)) { + throw new Error('mutation control artifact outcome is invalid'); + } + return { kind: runOutcomeKind(value.kind), + ...(typeof value.phase === 'string' ? { phase: value.phase } : {}), + ...(typeof value.reason === 'string' ? { reason: value.reason } : {}) }; +} + +function recipeRequestIdentity(value: unknown): { recipeSha256: string; selectionSha256: string; + taskPacks: unknown; taskSha256: string } { + if (!object(value) || !object(value.recipe) || !object(value.selection) || !object(value.task) + || typeof value.recipe.contentSha256 !== 'string' || typeof value.selection.sha256 !== 'string' + || typeof value.task.sha256 !== 'string') { + throw new Error('recipe task request has no complete identity'); + } + return { recipeSha256: value.recipe.contentSha256, selectionSha256: value.selection.sha256, + taskPacks: value.selection.taskPacks, taskSha256: value.task.sha256 }; +} + +function snapshotSource(appDir: string, to: string): void { + snapshotAppSource(appDir, to); +} + +// Match the database and registry endpoints supplied to the coding container. +// A matching port on another host is not owned by this run. +export function runAuditNetworkContext(track: Parameters[0], + args: { backend: string; runIndex: number }, lease: BackendLease): { ownEndpoints: string[]; isolatedLoopback: boolean } { + const ports = portsFor(track, args.backend, args.runIndex); + const databaseUrl = leasedDatabaseEnvironment(STACK_ADAPTER_REGISTRY.get(args.backend), { + database: lease.resources.database, networkMode: lease.resources.buildContainer?.networkMode, + lease, + }).DATABASE_URL; + const urls = [ports.vite, ports.express].filter((port): port is number => port !== null) + .map(port => `http://127.0.0.1:${port}`); + urls.push(...[databaseUrl, lease.resources.serverUri, packageRegistryEnvironment(packageRegistry(), + lease.resources.buildContainer?.networkMode, lease.resources.network).NPM_CONFIG_REGISTRY].filter((url): url is string => !!url)); + // Only endpoint authority leaves this process. Database credentials stay private. + // A loopback endpoint is also owned at each of the attempt's own network addresses, + // which is how a coding agent reaches its own application by bridge address. + const ownAddresses = lease.resources.network?.ownAddresses ?? []; + const ownEndpoints = [...new Set(urls.flatMap(value => { + const url = new URL(value); + const port = url.port || (url.protocol === 'https:' ? '443' : '80'); + const hosts = /^(?:127\.0\.0\.1|localhost|0\.0\.0\.0)$/.test(url.hostname) + ? [url.hostname, ...ownAddresses] : [url.hostname]; + return hosts.map(host => `${host}:${port}`); + }))]; + // The authenticated lease records the inspected coding container's namespace. + // A transcript's cwd or a Docker bridge alone does not prove isolation. + const network = lease.resources.network; + const build = lease.resources.buildContainer; + const isolatedLoopback = !!(network?.namespaceContainerId + && lease.resources.container?.owned && lease.resources.container.id === network.namespaceContainerId + && network.firewallSha256 && network.firewallInstalledAt + && build?.owned && build.networkMode === `container:${network.namespaceContainerId}`); + return { ownEndpoints, isolatedLoopback }; +} + +// Check contamination after every coding session. File-tool permissions do not +// govern shell reads, so the transcript audit remains a separate hard gate. +function auditContamination(appDir: string, network: ReturnType, + expectTranscripts: boolean): ContaminationAudit | null { + // A non-billable adapter runs no provider session and leaves no transcript; + // there is nothing to audit and nothing that could have been read. + if (!expectTranscripts) return null; + const args = [join(ROOT, 'dist', 'commands', 'leak-audit.js'), '--app', appDir, '--json', + '--own-endpoints', network.ownEndpoints.join(','), ...(network.isolatedLoopback ? ['--isolated-loopback'] : [])]; + let firstFailure: unknown = null; + for (let attempt = 1; attempt <= 2; attempt++) { + try { + const audit = sh('node', args, { stdio: 'pipe' }); + const entries = parseLeakAudit(audit); + if (entries.length === 0) { + return { kind: 'harness_failure', + evidence: ['no session transcript was found to audit'], + verdict: 'SCORES NOT USABLE — nothing verified this build stayed inside its directory.' }; + } + const escapes = entries.flatMap(entry => entry.hits); + const serious = escapes.filter(h => /GRADER|CONTRACT|BENCHMARK NOTES|PROMPTS|NETWORK/.test(h.kind)); + if (firstFailure) { + console.error(` warning: contamination audit passed on retry after: ${auditFailureSummary(firstFailure)}`); + } + if (!serious.length) return null; + return { kind: 'contaminated', + evidence: [...new Set(serious.map(h => `${h.kind}: ${h.path.split('/').slice(-2).join('/')}`))].slice(0, 8), + verdict: 'SCORES NOT USABLE — the audit detected restricted file or network access attempts; see the evidence categories.' }; + } catch (error) { + firstFailure ??= error; + if (attempt === 2) { + // An audit that could not run is not a pass. Keep the process details so + // the failure can be repaired without another paid reproduction. + return { kind: 'harness_failure', + evidence: [`audit did not run after retry: ${auditFailureSummary(error)}`], + verdict: 'SCORES NOT USABLE — nothing verified this build stayed inside its directory.' }; + } + } + } + return null; +} + +export function auditFailureSummary(error: unknown): string { + const failure = object(error) ? error : {}; + const message = errorMessage(error).split(/\r?\n/)[0] ?? ''; + const stderrLines = String(failure.stderr ?? '').trim().split(/\r?\n/).filter(Boolean); + const stderr = stderrLines.find(line => /(?:error|eacces|permission denied|failed)/i.test(line)) + ?? stderrLines[0]; + const details = [ + Number.isInteger(failure.status) ? `exit ${String(failure.status)}` : null, + failure.signal ? `signal ${String(failure.signal)}` : null, + stderr ? `stderr: ${stderr}` : null, + ].filter(Boolean); + return details.length ? `${message} (${details.join('; ')})` : message; +} + +const sh = (cmd: string, args: readonly string[], + opts: Omit = {}): string => + execFileSync(cmd, [...args], { + encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: COMMAND_TIMEOUT_MS, ...opts, + }); + +let activeAgentCancellation: AbortController | null = null; +// Set once a run owns resources. The top-level rejection handler invokes this +// directly; relying only on process 'exit' made cleanup best-effort precisely +// when an awaited build rejected unexpectedly. +let emergencyTeardown: (() => void) | null = null; + +export function parseAgentProcessResult(stdout: string, stderr: string, processError: unknown, + request: AgentRequest): ValidatedAgentResult { + const resultLine = stdout.trim().split('\n').pop(); + let result: ValidatedAgentResult; + try { + if (!resultLine) throw new Error('agent returned no result line'); + result = validateAgentResult(JSON.parse(resultLine), request); + } catch (resultError) { + const stdoutTail = stdout.trim().slice(-2000) || ''; + const stderrTail = stderr.trim().slice(-4000) || ''; + const processDetail = processError ? `agent process failed: ${errorMessage(processError)}\n` : ''; + throw new Error(`${processDetail}agent returned an invalid result: ${errorMessage(resultError)}\n` + + `agent stdout tail:\n${stdoutTail}\nagent stderr tail:\n${stderrTail}`); + } + if (processError && result.ok) { + throw new Error(`agent process failed after reporting success: ${errorMessage(processError)}`); + } + return result; +} + +export async function runAgent( + args: BenchArgs, + adapter: AgentAdapter, + mode: AgentMode, + level: number, + appDir: string, +): Promise { + if (!args.backend || !args.model) throw new Error('agent run requires backend and model'); + const remainingBudget = args.maxBudgetUsd == null ? null + : addCostUsd(args.maxBudgetUsd, -(args.spentBudgetUsd ?? 0)); + if (remainingBudget !== null && remainingBudget <= 0) { + throw new Error(`Cost cap reached: attempt cost cap of ${args.maxBudgetUsd} was exhausted before ${mode} L${level}`); + } + if (remainingBudget !== null && adapter.costLimit === 'unsupported') { + throw new Error(`agent adapter ${adapter.id} cannot enforce --max-budget-usd`); + } + const recipeTask = args.recipeTasks?.get(level)?.agentRequest + ?? args.recipeTasks?.get(level)?.request ?? null; + const request: AgentRequest = { mode, level, app: appDir, backend: args.backend, track: args.track, + runIndex: args.runIndex, model: args.model, guidance: args.guidance, skills: args.skills, + productionQuality: args.productionQuality, + ...(adapter.usesStackSkills + ? { skillIdentity: args.condition?.guidance.skills[args.backend] } : {}), + recipe: agentRecipeIdentity(args.recipe, recipeTask), + guidanceDocument: args.guidanceDocument, + credentialAliases: args.condition?.guidance?.credentialAliases ?? {}, + recipeTask, pricing: args.pricing, providerRoute: args.providerRoute, maxOutputTokens: args.maxOutputTokens, + maxBudgetUsd: remainingBudget, adapterCostLimit: adapter.costLimit }; + const argv = agentRequestArgv(adapter, request); + if (args.apiKey && !adapter.apiKeyEnvironmentVariable) { + throw new Error(`agent adapter ${adapter.id} does not accept an API key`); + } + const env = { ...process.env }; + if (args.apiKeyFile) env.STACK_BENCH_AGENT_API_KEY_FILE = args.apiKeyFile; + if (args.apiKey) { + const credentialName = adapter.apiKeyEnvironmentVariable; + if (!credentialName) throw new Error(`agent adapter ${adapter.id} does not accept an API key`); + env[credentialName] = args.apiKey; + } + const supervised = campaignProviderContinuationContext(env) !== null; + const capture = mkdtempSync(join(dirname(appDir), '.agent-output-')); + const cancellation = new AbortController(); + activeAgentCancellation = cancellation; + try { + const processResult = await runBounded(process.execPath, argv, { + env, stdio: 'ignore', timeoutMs: supervised ? null : adapter.deadlineMs, + signal: cancellation.signal, + logs: { stdout: join(capture, 'stdout'), stderr: join(capture, 'stderr'), maxBytes: 64 * 1024 * 1024 }, + }); + const processError = processResult.error ?? (processResult.timedOut + ? new Error(`agent deadline exceeded after ${adapter.deadlineMs} ms`) + : processResult.cancelled ? new Error('agent process cancelled') + : !processResult.ok ? new Error(`agent exited ${processResult.code ?? processResult.signal}`) : null); + if (Object.values(processResult.logs ?? {}).some(log => log.truncated)) { + throw new Error('agent output exceeded the 64 MiB capture limit'); + } + const result = parseAgentProcessResult(readFileSync(join(capture, 'stdout'), 'utf8'), + readFileSync(join(capture, 'stderr'), 'utf8'), processError, request); + args.spentBudgetUsd = addCostUsd(args.spentBudgetUsd, result.costUsd); + return result; + } finally { + if (activeAgentCancellation === cancellation) activeAgentCancellation = null; + rmSync(capture, { recursive: true, force: true }); + } +} + +interface GradeCheck { + stableKey: string; + executionId?: string; + source?: string; + packId?: string; +} + +interface GradeRecipeTask { + request: UnknownRecord; + selection: { checks: readonly GradeCheck[] } + | { scoredChecks: readonly GradeCheck[]; observedChecks?: readonly GradeCheck[] }; +} + +/** The aliases grading expects: the run's own when set, else the condition's. */ +function gradingCredentialAliases(args: GradeArguments): Record | undefined { + return args.gradingCredentialAliases ?? args.condition?.guidance?.credentialAliases; +} + +function checksForGrade(task: GradeRecipeTask | undefined, observation: GradeOptions['observation']): + readonly GradeCheck[] { + if (!task) return []; + if ('scoredChecks' in task.selection) { + return observation === 'observed' + ? task.selection.observedChecks ?? [] : task.selection.scoredChecks; + } + return task.selection.checks; +} + +type GradeArguments = Pick & { + agentAdapter?: BenchArgs['agentAdapter']; + recipeTasks?: ReadonlyMap; + progression?: { identity: { policy?: string } }; + condition?: { guidance?: { credentialAliases?: Record } }; + gradingCredentialAliases?: Record; +}; + +export function gradeArgv( + args: GradeArguments, + appDir: string, + url: string, + label: string, + level: number, + track: Track, + parentAttemptId: string, + options: GradeOptions = {}, +): string[] { + const { observation = 'scored', out = null, sourceSha256 = null, + applicationFailure = null } = options; + if (!args.backend) throw new Error('grading requires a backend'); + const restartSpec = restartSpecFor(args, appDir, track); + const task = options.recipeTask ?? args.recipeTasks?.get(level); + return [compiledEntrypoint('commands', 'run-suite.js'), '--app', appDir, '--url', url, + '--backend', args.backend, '--label', label, '--level', String(level), + '--track', args.track, + '--run-index', String(args.runIndex), + '--parent-attempt-id', parentAttemptId, + '--observation', observation, + '--out', privateGradingDirectory(appDir, out), + ...(sourceSha256 ? ['--source-sha256', sourceSha256] : []), + ...(args.recipe ? ['--recipe', args.recipe] : []), + ...(task ? ['--recipe-task-json', JSON.stringify(task.request)] : []), + ...(gradingCredentialAliases(args) + ? ['--credential-aliases-json', JSON.stringify(gradingCredentialAliases(args))] : []), + ...(!applicationFailure && args.agentAdapter !== 'reference-fixture' + && STACK_ADAPTER_REGISTRY.get(args.backend).runPolicy.resetEnabled + ? ['--retry-inconclusive'] : []), + ...(applicationFailure + ? ['--application-failure-json', JSON.stringify(applicationFailure)] : []), + ...(observation === 'scored' && args.recipeTasks && !args.progression + ? ['--regression-checks-json', JSON.stringify([...args.recipeTasks.entries()] + .filter(([priorLevel]) => priorLevel < level) + .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored') + .map(check => check.stableKey)))] : []), + ...(args.media && observation === 'scored' ? [] : ['--no-media']), + ...(!STACK_ADAPTER_REGISTRY.get(args.backend).runPolicy.resetEnabled + ? ['--no-reset'] + : ['--restart-spec', JSON.stringify(restartSpec)])]; +} + +export function archiveCandidateGrade(appDir: string, outputDir: string, label: string): void { + const gradingDirectory = privateGradingDirectory(appDir); + if (!existsSync(gradingDirectory)) return; + cpSync(gradingDirectory, privateGradingDirectory(appDir, join(outputDir, 'candidate-grades', label)), { + recursive: true, + filter: source => !/[\\/]media([\\/]|$)/.test(source), + }); +} + +function grade( + args: BenchArgs, + appDir: string, + url: string, + label: string, + level: number, + track: Track, + parentAttemptId: string, + options: GradeOptions = {}, +): GradeBundlePayload | null { + const out = privateGradingDirectory(appDir, options.out); + const source = hashAppSource(appDir); + const argv = gradeArgv(args, appDir, url, label, level, track, parentAttemptId, { + ...options, sourceSha256: options.sourceSha256 ?? source.sha256, + }); + const bundle = join(out, ARTIFACT_FILE.gradeBundle); + rmSync(bundle, { force: true }); + const task = options.recipeTask ?? args.recipeTasks?.get(level); + const currentChecks = checksForGrade(task, options.observation); + const regressionChecks = options.observation === 'observed' || args.progression + ? [] + : [...(args.recipeTasks?.entries() ?? [])] + .filter(([priorLevel]) => priorLevel < level) + .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored')); + const sourceCount = task + ? selectedGradingSourceCount(currentChecks, regressionChecks) + : suitesFor(track, level).length; + try { + sh('node', argv, { stdio: 'inherit', timeout: gradingRunTimeoutMs(sourceCount, + args.recipeBindings.get(level)?.plan.packs ?? [], [...currentChecks, ...regressionChecks]) }); + } catch { /* a current bundle may still explain a scored failure */ } + return existsSync(bundle) + ? readArtifactPayload(bundle, { expectedKind: 'grade_bundle' }) : null; +} + +function restartSpecFor(args: Pick, + appDir: string, track: Track): RuntimeControlSpec { + if (!args.backend) throw new Error('restart specification requires a backend'); + const port = portsFor(track, args.backend, args.runIndex).vite ?? null; + if (port == null) throw new Error(`stack ${args.backend} has no application port`); + return { backend: args.backend, app: appDir, port: Number(port), probe: '' }; +} + +function runMutationControl( + args: BenchArgs, + appDir: string, + url: string, + track: Track, + imageId: string | null, +): MutationControlResult { + if (!args.out || !args.mutations) throw new Error('mutation control requires output and manifest paths'); + const output = join(args.out, ARTIFACT_FILE.mutationControl); + if (!args.mutationResumeFrom || resolve(args.mutationResumeFrom) !== resolve(output)) { + rmSync(output, { force: true }); + } + if (imageId) args.mutationImageId = imageId; + else delete args.mutationImageId; + const argv = mutationControlArgv(mutationControlArgs(args), appDir, url, track); + let processError = null; + try { sh(process.execPath, argv, { + stdio: 'inherit', timeout: mutationControlTimeoutMs(args.mutationMaxRuntimeMinutes), + }); } + catch (error) { processError = errorMessage(error).split('\n')[0] ?? null; } + if (!existsSync(output)) { + return { ok: false, artifact: output, processError, + outcome: { kind: 'harness_failure', phase: 'mutation-control', + reason: processError ?? 'mutation runner produced no artifact' } }; + } + const artifact = readArtifactPayload(output, { expectedKind: 'mutation_control' }); + return { ok: artifact.ok === true && !processError, artifact: output, + processError, summary: artifact.summary ?? null, outcome: mutationOutcome(artifact.outcome), + results: artifact.results ?? [] }; +} + +function validateMutationInput(args: BenchArgs): void { + if (!args.mutations) return; + if (!args.app) throw new Error('--mutations requires an explicit pristine --app'); + const manifest = JSON.parse(readFileSync(args.mutations, 'utf8')); + if (!/^[a-f0-9]{64}$/.test(manifest.fixtureSha256 ?? '')) { + throw new Error('mutation manifest has no valid fixtureSha256'); + } + const fixture = hashDirectory(args.app); + if (fixture.sha256 !== manifest.fixtureSha256) { + throw new Error(`mutation manifest targets fixture ${manifest.fixtureSha256}, not ${fixture.sha256}`); + } +} + +export function inspectGradeSource(directory: string, + options: { level?: number; checkKeys?: string[] } = {}) { + const root = realpathSync(directory); + const runPath = join(root, ARTIFACT_FILE.run); + const parent = readArtifact(runPath, { expectedKind: 'benchmark_run' }); + const run = parent.payload; + if (!object(run.mode) + || !['sequential', 'dependency'].includes(String(run.mode.id)) || run.contaminated) { + throw new Error('--grade-from requires an uncontaminated sequential or dependency run'); + } + const dependency = run.mode.id === 'dependency'; + if (!parent.timestamps.completedAt) { + const recoveryPath = join(root, ARTIFACT_FILE.recovery); + if (!dependency || !existsSync(recoveryPath)) { + throw new Error('unfinished --grade-from requires a recovered dependency run and a completed candidate grade'); + } + const recovery = readArtifact<{ runId: string; backend: string; status: string; + cleanup: { succeeded: boolean; retained: boolean } }>(recoveryPath, { expectedKind: 'recovery' }); + if (recovery.attempt.parentId !== parent.id || recovery.payload.runId !== parent.id + || recovery.payload.backend !== run.backend || recovery.payload.status !== 'clean' + || recovery.payload.cleanup?.succeeded !== true || recovery.payload.cleanup.retained !== false) { + throw new Error('unfinished --grade-from recovery does not prove cleanup of its parent'); + } + } + if (dependency && (!Number.isSafeInteger(options.level) || options.level! < 1)) { + throw new Error('dependency --grade-from requires a positive --grade-level'); + } + if (dependency && !options.checkKeys?.length) throw new Error('dependency --grade-from requires explicit --check keys'); + if (!dependency && (options.level !== undefined || run.levels.length !== 1)) { + throw new Error('sequential --grade-from requires a single level and does not accept --grade-level'); + } + const levels = dependency ? run.levels.filter(item => item.level === options.level) : run.levels; + if (levels.length !== 1 || (dependency && run.mode.workSelection === 'feature')) { + throw new Error('saved replay requires exactly one unambiguous first-build candidate at the selected depth'); + } + const level = levels[0]!; + if (!parent.identities.agentAdapter?.id || !Number.isSafeInteger(run.backendLease?.runIndex) + || run.backendLease.runIndex < 0 || !run.runtime?.buildImage) { + throw new Error('saved run lacks its agent, runtime image, or run index'); + } + const serverUri = ['spacetime', 'convex'].includes(run.backend) + ? loopbackHttpUri(run.backendLease.resources?.serverUri).origin : null; + const checkpoint = level.checkpoint; + const condition = run.condition as BenchArguments['condition']; + let declared = condition?.requested?.levels?.find(item => item.level === level.level); + if ((!dependency && !checkpoint) || !declared || declared.selection.schemaVersion !== 3 + || !declared.task.contractSha256 || !declared.task.requirementSha256 + || !Array.isArray(declared.selection.requested.features) + || !Array.isArray(declared.selection.scoredChecks)) { + throw new Error('saved run lacks a source checkpoint or a bound modular grading scope'); + } + const child = (name: string): string => { + const path = realpathSync(resolve(root, name)); + const rel = relative(root, path); + if (!rel || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) throw new Error('saved checkpoint path escapes its execution'); + return path; + }; + const sourceDirectory = dependency ? `first-build-l${level.level}` : checkpoint!.directory; + const sourcePath = child(sourceDirectory); + assertPlainAppSourceTree(sourcePath); + const source = hashAppSource(sourcePath); + let candidate = null; + if (dependency) { + const gradePath = `first-build-l${level.level}-grading/bundle.json`; + const gradeFile = child(gradePath); + const grade = readArtifact(gradeFile, { expectedKind: 'grade_bundle' }); + const expected = level.firstBuild?.source; + if (!object(expected) || source.sha256 !== expected.sha256 || source.files.length !== expected.files + || grade.attempt.parentId !== parent.id || grade.payload.source?.sha256 !== source.sha256 + || grade.payload.backend !== run.backend || grade.payload.track !== run.track + || grade.payload.level !== level.level || grade.payload.observation !== 'scored' + || canonicalDefinitionJson(grade.identities.engine) !== canonicalDefinitionJson(parent.identities.engine) + || grade.identities.recipe?.id !== declared.recipe.id + || grade.identities.recipe?.sha256 !== declared.recipe.contentSha256 + || grade.payload.selection?.schemaVersion !== 3 + || grade.payload.selection.sha256 !== level.selection?.sha256 + // Inconclusive progression summaries retain only the selection hash. + || (Object.keys(level.selection).length !== 1 + && canonicalDefinitionJson(grade.payload.selection) !== canonicalDefinitionJson(level.selection)) + || !Array.isArray(grade.payload.selection.requested?.features) + || !Array.isArray(grade.payload.selection.scoredChecks)) { + throw new Error('saved first-build source or grade does not match its parent run and candidate scope'); + } + declared = { ...declared, selection: grade.payload.selection }; + candidate = { kind: 'first-build', directory: sourceDirectory, level: level.level, + grade: { path: gradePath, sha256: sha256(readFileSync(gradeFile)), id: grade.id, + identities: grade.identities }, outcome: level.firstBuild?.outcome ?? null }; + } else { + const saved = readArtifact<{ source: { directory: string; sha256: string; files: number }; + backend: string; track: string; level: number; selectionSha256: string }>(child(checkpoint!.artifact), + { expectedKind: 'source_checkpoint' }); + if (saved.attempt.parentId !== parent.id || saved.payload.backend !== run.backend + || saved.payload.track !== run.track || saved.payload.level !== level.level + || saved.payload.source.directory !== checkpoint!.directory + || saved.payload.source.sha256 !== checkpoint!.sha256 || saved.payload.source.files !== checkpoint!.files + || saved.payload.selectionSha256 !== declared.selection.sha256 + || source.sha256 !== checkpoint!.sha256 || source.files.length !== checkpoint!.files) { + throw new Error('saved source or checkpoint does not match its parent run'); + } + } + if (options.checkKeys?.some(key => !declared.selection.scoredChecks!.some(check => check.stableKey === key))) { + throw new Error('regrade checks must belong to the original scored scope'); + } + const aliases = condition?.guidance.credentialAliases; + if (!aliases || Object.values(aliases).some(value => typeof value !== 'string')) { + throw new Error('saved run lacks valid grading credential aliases'); + } + return { root, parent, declared, dependency, candidate, serverUri, sourcePath, source: { sha256: source.sha256, files: source.files.length }, + runSha256: sha256(readFileSync(runPath)), aliases }; +} + +export function pendingRunSnapshot(run: BenchmarkRunRecord, pending: RunLevelRecord | null, + costComplete: boolean): BenchmarkRunRecord { + const snapshot = { ...run, levels: [...run.levels] }; + if (pending) { + const index = snapshot.levels.findIndex(level => level.level === pending.level); + const record = mergeFeatureLevelRecord(index < 0 ? null : snapshot.levels[index]!, pending); + if (index < 0) snapshot.levels.push(record); + else snapshot.levels[index] = record; + if (!['harness_failure', 'provider_failure'].includes(run.outcome?.kind ?? '')) snapshot.outcome = { kind: 'ungraded', phase: 'interrupted-level', + reason: 'level has not completed', appFailures: [], inconclusive: [], harnessFailures: [] }; + } + finalizeRunTotals(snapshot, Date.parse(run.startedAt), { costComplete }); + return snapshot; +} + +async function main() { + let pendingLevel: RunLevelRecord | null = null; + let sessionInFlight = false; + let runCostComplete = true; + const persistRun = (path: string, value: unknown) => { + if (pendingLevel || sessionInFlight) { + return writeRunJson(path, pendingRunSnapshot(value as BenchmarkRunRecord, + pendingLevel, runCostComplete && !sessionInFlight)); + } + return writeRunJson(path, value); + }; + const args: BenchArgs = { + ...parseBenchArguments(process.argv), + recipeTasks: new Map(), + recipeBindings: new Map(), + }; + const regrade = args.gradeFrom ? inspectGradeSource(args.gradeFrom, + { level: args.gradeLevel, checkKeys: args.checkKeys }) : null; + if (regrade) { + const { parent, declared, aliases } = regrade; + const requested = declared.selection.requested; + Object.assign(args, { backend: parent.payload.backend, track: parent.payload.track, + runIndex: parent.payload.backendLease.runIndex, + levels: String(declared.level), levelList: [declared.level], recipe: declared.recipe.id, + agentAdapter: parent.identities.agentAdapter!.id, model: parent.payload.model, + providerRoute: parent.payload.providerRoute, + maxOutputTokens: parent.payload.maxOutputTokens, + featureIds: requested.features, checkKeys: args.checkKeys.length ? args.checkKeys : requested.checks, + requestedSpecifications: requested.specifications?.requested ?? [], + expectedSpecifications: requested.specifications?.expected ?? [], + observedSpecifications: [], gradingCredentialAliases: aliases }); + if (declared.selection.observedChecks?.length) throw new Error('--grade-from does not support observed checks'); + const output = resolve(args.out!); + for (let path = output; dirname(path) !== path; path = dirname(path)) { + if (existsSync(path) && lstatSync(path).isSymbolicLink()) { + throw new Error('regrade output must not pass through a symbolic link'); + } + } + const overlaps = (base: string, target: string) => { + const rel = relative(base, target); + return !rel || (rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel)); + }; + if (overlaps(regrade.root, output) || overlaps(output, regrade.root) + || (existsSync(output) && readdirSync(output).length)) { + throw new Error('--grade-from requires an empty output directory outside the original execution'); + } + if (process.env.STACK_BENCH_IMAGE && process.env.STACK_BENCH_IMAGE !== parent.payload.runtime.buildImage) { + throw new Error('regrade build image differs from the original run'); + } + process.env.STACK_BENCH_IMAGE = parent.payload.runtime.buildImage; + if (regrade.serverUri) { + process.env[parent.payload.backend === 'convex' ? 'STACK_BENCH_CONVEX_URI' : 'STACK_BENCH_STDB_URI'] = regrade.serverUri; + } + } + let repairGrant = null; + if (args.repairFrom) { + const repairLevel = args.repairLevel; + if (typeof repairLevel !== 'number' || !Number.isSafeInteger(repairLevel) || repairLevel < 1) { + throw new Error('--repair-from requires a positive --repair-level'); + } + repairGrant = createRepairGrant(args.repairFrom, + { level: repairLevel, repairs: args.repairs }); + const config = repairGrant.configuration; + if (config.buildImage && process.env.STACK_BENCH_IMAGE + && config.buildImage !== process.env.STACK_BENCH_IMAGE) { + throw new Error('repair continuation build image differs from its parent run'); + } + if (config.buildImage) process.env.STACK_BENCH_IMAGE = config.buildImage; + Object.assign(args, { + backend: config.backend, + track: config.track, + recipe: config.recipe, + levels: String(config.level), + levelList: [config.level], + runIndex: config.runIndex, + agentAdapter: config.agentAdapter, + model: config.model, + providerRoute: config.providerRoute, + maxOutputTokens: config.maxOutputTokens, + guidance: config.guidance, + guidanceDocument: config.guidanceDocument, + productionQuality: config.condition?.productionQuality === true || (!config.condition && config.productionQuality === true), + condition: config.condition, + selectionRequest: campaignSelection(config.selectionRequest, 'repair configuration.selectionRequest'), + skills: config.skills, + packIds: [...(campaignSelection(config.selectionRequest, + 'repair configuration.selectionRequest').packs ?? [])], + checkKeys: [...(campaignSelection(config.selectionRequest, + 'repair configuration.selectionRequest').checks ?? [])], + featureIds: [], + requestedSpecifications: [], + expectedSpecifications: [], + observedSpecifications: [], + seedFrom: repairGrant.sourcePath, + url: config.url, + parentAttemptId: repairGrant.parent.id, + repairGrant, + }); + } + if (!args.backend) throw new Error('benchmark run requires a backend'); + const stackAdapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const materializeCodingOutput = stackAdapter.id !== 'stub'; + const agentAdapter = AGENT_ADAPTER_REGISTRY.get(args.agentAdapter); + args.providerRoute = validateProviderRoute(agentAdapter.provider, args.providerRoute); + args.maxOutputTokens = validateProviderOutputLimit(agentAdapter.provider, args.maxOutputTokens); + // Credential aliases keep the fixture passwords out of an agent's prompt, + // so grading expects the aliases. A reference fixture is the fixture itself, + // seeded with the real credentials, and is graded with them. + if (!regrade && agentAdapter.gradesWithFixtureCredentials) args.gradingCredentialAliases = {}; + if (process.env.STACK_BENCH_APPLIANCE !== '1' && agentAdapter.costLimit !== 'non-billable') { + throw new Error(`agent adapter ${agentAdapter.id} requires the Docker appliance`); + } + if (repairGrant) { + const currentAgent = agentAdapterIdentity(agentAdapter); + const parentAgent = repairGrant.parentArtifact.identities.agentAdapter; + if (currentAgent.id !== parentAgent?.id || currentAgent.version !== parentAgent?.version + || currentAgent.sha256 !== parentAgent?.sha256) { + throw new Error('repair continuation agent adapter differs from its parent run'); + } + if (stackAdapter.id !== repairGrant.parentArtifact.identities.stackAdapter?.id + || stackAdapter.version !== repairGrant.parentArtifact.identities.stackAdapter?.version) { + throw new Error('repair continuation stack adapter differs from its parent run'); + } + } + if (!regrade) applyAgentCredential(args, agentAdapter); + args.model ??= agentAdapter.defaultModel; + if (!args.model) throw new Error(`agent adapter ${agentAdapter.id} has no default model`); + if (args.pricing !== undefined) { + args.pricing = validatePricingAuthority(args.pricing, { at: '--pricing-json' }); + } else if (args.maxBudgetUsd != null && agentAdapter.costLimit === 'native') { + const rates = claudeRatesForModel(args.model); + if (!rates) throw new Error(`no default pricing is recorded for model ${args.model}`); + args.pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } else { + args.pricing = null; + } + if (args.retainBackend && !stackAdapter.runPolicy.retainHostSupported) { + throw new Error(`stack adapter ${args.backend} does not support --retain-backend`); + } + const stackRuntime = stackAdapter.orchestrator.config( + { root: ROOT, env: process.env, helpers: { exists: existsSync } }); + Object.assign(process.env, stackRuntime.environment); + process.env.STACK_BENCH_NODE_BIN = process.platform === 'win32' ? 'node.exe' : process.execPath; + const track = loadTrack(args.track); + const auditsTranscripts = agentAdapter.costLimit !== 'non-billable'; + // Resolve the requested scope for every level before probing the sandbox, + // acquiring a backend lease or paying for a build. A pack that exists at L2 + // but not L1 is not a late grading surprise; it is an invalid run request. + args.selectionRequest ??= { packs: [...args.packIds], checks: [...args.checkKeys] }; + for (const level of args.levelList) { + const declared = args.condition?.requested?.levels?.find(entry => entry.level === level) ?? null; + const modularSelection = args.selectionRequest.levels?.find(entry => entry.level === level) ?? null; + if (declared?.selection?.schemaVersion === 3) { + const expected = args.featureCatalog + ? { level, recipe: declared.recipe.id } + : { level, recipe: declared.recipe.id, + features: declared.selection.requested.features, + checks: declared.selection.requested.checks }; + if (canonicalDefinitionJson(modularSelection) !== canonicalDefinitionJson(expected)) { + throw new Error(`campaign selection changed before L${level}`); + } + } else if (modularSelection) { + throw new Error(`campaign selection declares modular L${level} without a modular condition`); + } + const declaredRecipe = declared?.recipe.id ?? null; + const binding = resolveRecipeRelease(track, level, declaredRecipe ?? args.recipe); + if (!binding && (args.packIds.length || args.checkKeys.length)) { + throw new Error(`L${level} has no recipe release, so --pack/--check cannot be resolved`); + } + if (binding) { + args.recipeBindings.set(level, binding); + if (args.featureCatalog) { + validateProgressionCampaignLevelScope(binding, args.featureCatalog, declared, level); + } + const requested = declared?.selection?.requested; + const progressionSelection = args.featureCatalog + ? resolveProgressionRecipeLevelSelection(binding, args.featureCatalog, level, + { cumulative: Boolean(args.progression) }) : null; + const resolved = progressionSelection === null + ? createBoundRecipeTaskRequest(binding, requested?.features + ? { featureIds: requested.features, + requestedSpecifications: requested.specifications?.requested, + expectedSpecifications: requested.specifications?.expected, + observedSpecifications: requested.specifications?.observed, + checkKeys: requested.checks } + : regrade?.dependency ? { ...args, taskMode: regrade.declared.task.mode, + dependencyExpansion: regrade.declared.selection.requested.dependencyExpansion } : args) : null; + const grader = progressionSelection?.grader ?? resolved; + if (!grader) throw new Error(`L${level} has no recipe task request`); + if (args.condition && !declared) { + throw new Error(`study condition does not bind requested L${level}`); + } + const graderIdentity = recipeRequestIdentity(grader.request); + if (declared && (declared.recipe.contentSha256 !== graderIdentity.recipeSha256 + || declared.selection.sha256 !== graderIdentity.selectionSha256 + || JSON.stringify(declared.selection.taskPacks) !== JSON.stringify(graderIdentity.taskPacks) + || declared.task.sha256 !== graderIdentity.taskSha256)) { + throw new Error(`study condition requested scope changed before L${level}`); + } + if (progressionSelection) { + const progressionGrader = progressionSelection.grader; + args.recipeTasks.set(level, { + request: progressionGrader.request, + selection: progressionGrader.selection, + task: progressionGrader.task, + agentRequest: progressionSelection.agent.request, + }); + } else if (resolved) { + args.recipeTasks.set(level, { + ...resolved, + agentRequest: createAgentVisibleTaskRequest(binding, resolved), + }); + } + } + } + if (args.progression) { + const state = progressionEngine.initialize(args.progression.definition); + const declared = args.condition?.requested?.levels + ?.find(entry => entry.level === state.level) ?? null; + const binding = resolveRecipeRelease(track, state.level, + declared?.recipe.id ?? null); + if (!binding) throw new Error(`L${state.level} has no recipe release`); + resolveProgressionRecipeAction(binding, state); + if (!args.progressionOwner) { + throw new Error('live dependency progression requires an exact compiled campaign attempt'); + } + } + if (repairGrant) { + const expectedSelection = repairGrant.level.selection?.sha256 ?? null; + const repairTask = args.recipeTasks.get(repairGrant.level.level); + const resolvedSelection = repairTask ? recipeRequestIdentity(repairTask.request).selectionSha256 : null; + if (resolvedSelection !== expectedSelection) { + throw new Error('repair continuation test selection differs from its parent run'); + } + } + if (regrade) { + const task = args.recipeTasks.get(regrade.declared.level)!; + const keys = (checks: readonly { stableKey: string; points: number }[]) => + checks.map(check => `${check.stableKey}:${check.points}`).sort(); + const selected = 'scoredChecks' in task.selection ? task.selection.scoredChecks : task.selection.checks; + if (regrade.dependency) { + if (canonicalDefinitionJson(selected.map(check => check.stableKey).sort()) + !== canonicalDefinitionJson([...new Set(args.checkKeys)].sort())) { + throw new Error('dependency diagnostic changed the explicitly selected check scope'); + } + } else if (task.task.contractSha256 !== regrade.declared.task.contractSha256 + || task.task.requirementSha256 !== regrade.declared.task.requirementSha256 + || canonicalDefinitionJson(keys(selected)) + !== canonicalDefinitionJson(keys(regrade.declared.selection.scoredChecks!.filter(check => + !args.checkKeys.length || args.checkKeys.includes(check.stableKey))))) { + throw new Error('regrade changed the original product contract or scored check scope'); + } + } + if (!args.selectionRequest.levels && (JSON.stringify(args.selectionRequest.packs) !== JSON.stringify(args.packIds) + || JSON.stringify(args.selectionRequest.checks) !== JSON.stringify(args.checkKeys))) { + throw new Error('campaign pack/check selection changed before execution'); + } + // Caller-owned mutation inputs are pure request data. Reject them before + // checking credentials, Docker, ports, or any other ambient runner state so + // an invalid experiment can never be masked by an unrelated preflight error. + validateMutationInput(args); + // The deterministic adapter/stack is the model-free unit loop. Real runs + // prove the exact requested scope, engine, image, credentials, storage and + // ports before any paid coding session begins. + const performPreflight = (smoke = false) => { + const preflight = args.backend === 'stub' ? null : runPreflight({ + backends: [stackAdapter.id], track: args.track, levels: args.levels, + levelList: args.levelList, runIndex: args.runIndex, agentAdapter: args.agentAdapter, + providerRoute: args.providerRoute, + maxOutputTokens: args.maxOutputTokens, + modelFree: regrade !== null, + guidance: args.guidance, + recipe: args.recipe, + ...(args.condition?.requested ? { requestedScopes: [args.condition.requested] } : {}), + ...(args.featureCatalog ? { featureCatalog: args.featureCatalog } : {}), + ...(args.runMode ? { mode: args.runMode } : {}), + agentSkills: args.skills ?? null, + packIds: args.packIds, checkKeys: args.checkKeys, smoke, + ...(process.env.STACK_BENCH_SUPERVISOR_STATE + ? { supervisorState: process.env.STACK_BENCH_SUPERVISOR_STATE } : {}), + image: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, + resultsDir: resolve(args.out ?? stackBenchResultsRoot(ROOT)), + }, { ownedLease: { path: leasePath, runId }, env: args.apiKey && agentAdapter.apiKeyEnvironmentVariable + ? { ...process.env, [agentAdapter.apiKeyEnvironmentVariable]: '' } + : process.env }); + if (preflight) writeArtifact(join(outputDir, ARTIFACT_FILE.preflight), { + kind: 'preflight', id: `${runId}-preflight`, + attempt: { id: `${runId}-preflight`, parentId: runId }, + identities: emptyArtifactIdentities({ + agentAdapter: agentAdapterIdentity(agentAdapter), + stackAdapter: { id: stackAdapter.id, version: stackAdapter.version }, + }), + payload: preflight, + }); + if (preflight && !preflight.ok) { + const failures = preflight.checks.filter(check => check.status === 'fail'); + console.error('\nPREFLIGHT FAILED — no model session was started.'); + for (const failure of failures) { + console.error(` ${failure.id}: ${failure.summary}`); + if (failure.remediation) console.error(` hint: ${failure.remediation}`); + } + throw new Error('preflight failed; no model session was started'); + } + if (preflight) console.log(` preflight ... ${preflight.summary.passed} checks passed` + + `${preflight.summary.warnings ? `, ${preflight.summary.warnings} warning(s)` : ''}`); + }; + if (process.env.STACK_BENCH_APPLIANCE === '1') { + console.log(' sandbox ... coding container is isolated from the controller and grading files'); + } + const assignedPorts = portsFor(track, args.backend, args.runIndex); + let url = args.url ?? `http://localhost:${assignedPorts.vite}`; + const runDir = resultsName(track, args.backend, args.runIndex); + const runId = newRunId({ track: args.track, backend: args.backend, runIndex: args.runIndex }); + const artifactLabel = `${runDir}-${runId}`; + // Default results never reuse a directory. The stable backend/run name is a + // grouping directory only; every artifact beneath it belongs to one run id. + args.out ??= join(stackBenchResultsRoot(ROOT), runDir, runId); + if (!args.out) throw new Error('benchmark run has no results directory'); + const outputDir = args.out; + mkdirSync(args.out, { recursive: true }); + if (existsSync(join(args.out, ARTIFACT_FILE.run))) { + throw new Error(`refusing to reuse result directory containing ${ARTIFACT_FILE.run}: ${args.out}`); + } + + // Validate caller-owned source before acquiring a backend slot so a bad + // fixture cannot leave leased resources behind. + const ownWorkDir = !args.app; + const appDir = args.app ?? join(workDirFor(track, args.backend, args.runIndex, runId), 'app'); + if (args.app) mkdirSync(appDir, { recursive: true }); + privateGradingDirectory(appDir, join(outputDir, 'grading')); + if (args.repairGrant && url.startsWith('file:')) { + url = pathToFileURL(join(appDir, 'index.html')).href; + } + + // Bind destructive and lifecycle operations to exact resource identities and + // an ownership token. Targets come only from the lease, never generated code. + const runtimeRoot = resolve(process.env.STACK_BENCH_RUNTIME_DIR + ?? join(tmpdir(), 'stack-bench-runtime')); + const runtimeDir = join(runtimeRoot, runId); + const leasePath = join(runtimeDir, ARTIFACT_FILE.backendLease); + const preparedLease = stackAdapter.lease.prepare({ + track, + runIndex: args.runIndex, + runtimeDir, + serverUri: stackRuntime.lease.serverUri, + env: process.env, + helpers: { containerIdentity: runningContainerIdentity, dbName, moduleName }, + }); + const initialLease = createBackendLease({ + runId, + backend: args.backend, + track: args.track, + runIndex: args.runIndex, + ...preparedLease.lease, + }); + const lockScope = resourceLockScope(); + const lockKeys = backendResourceLockKeys(initialLease, assignedPorts, + [...preparedLease.lockKeys, ...(args.app ? [`workspace:${realpathSync(appDir)}`] : [])]); + let privateSupervisorStatePath = null; + try { + if ((args.campaignFile || args.campaignAdmissionId) && args.backend !== 'stub') { + const executionId = process.env.STACK_BENCH_CAMPAIGN_EXECUTION; + if (!executionId || !args.experimentIdentity || !args.campaignAdmissionId + || !borrowCampaignReservation({ env: process.env, + campaignSha256: args.experimentIdentity.sha256, admissionId: args.campaignAdmissionId, + executionId, output: resolve(outputDir), leasePath, lease: initialLease, keys: lockKeys })) { + throw new Error('campaign worker requires private resource delegation'); + } + } else { + await claimBackendResourcesWhenAvailable(leasePath, initialLease, { ...lockScope, keys: lockKeys }); + } + const supervisorState = process.env.STACK_BENCH_SUPERVISOR_STATE + ?? (process.env.STACK_BENCH_SUPERVISOR_DIR + ? join(resolve(process.env.STACK_BENCH_SUPERVISOR_DIR), `${runId}.json`) : null); + if (supervisorState) { + // Private handoff to an outer timeout supervisor. It contains the lease + // token, so create it once with owner-only permissions and never place it + // in the results tree. + privateSupervisorStatePath = resolve(supervisorState); + mkdirSync(dirname(privateSupervisorStatePath), { recursive: true, mode: 0o700 }); + writeFileSync(privateSupervisorStatePath, `${JSON.stringify({ + version: SUPERVISOR_STATE_VERSION, runId, backend: args.backend, runtimeDir, leasePath, + ownershipToken: initialLease.ownershipToken, output: resolve(args.out), + })}\n`, { flag: 'wx', mode: 0o600 }); + } + } catch (error) { + // A refused release keeps the lease: it is the only record of what still runs. + if (existsSync(leasePath) && releaseBackendLease(leasePath, initialLease.ownershipToken) + && !initialLease.campaignDelegation) rmSync(runtimeDir, { recursive: true, force: true }); + throw error; + } + process.env.STACK_BENCH_LEASE = leasePath; + process.env.STACK_BENCH_LEASE_TOKEN = initialLease.ownershipToken; + const auditNetwork = () => auditsTranscripts ? runAuditNetworkContext(track, { backend: stackAdapter.id, runIndex: args.runIndex }, + readBackendLease(leasePath, { token: initialLease.ownershipToken, backend: args.backend, runId })) + : { ownEndpoints: [], isolatedLoopback: false }; + + if (process.platform === 'win32') { + // When Windows resolves `bash` through WSL, WSLENV must carry lease paths + // and tokens into lifecycle scripts with path translation. + const bridge = ['STACK_BENCH_LEASE/p', 'STACK_BENCH_LEASE_TOKEN', + 'STACK_BENCH_NODE_BIN', ...stackRuntime.windowsEnvironmentBridge]; + const existing = (process.env.WSLENV ?? '').split(':').filter(Boolean); + process.env.WSLENV = [...new Set([...existing, ...bridge])].join(':'); + } + + let tornDown = false; + let activeRun: BenchmarkRunRecord | null = null; + const recoveryPath = join(outputDir, ARTIFACT_FILE.recovery); + const writeLeaseEvidence = (knownLease: BackendLease | null = null) => { + const lease = knownLease ?? readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId }); + const out = join(outputDir, ARTIFACT_FILE.backendLease); + const evidence = publicBackendLease(lease); + const id = `${runId}-backend-lease`; + writeArtifact(out, { + kind: 'backend_lease_evidence', id, + attempt: { id, parentId: runId }, + timestamps: { startedAt: evidence.createdAt, completedAt: new Date().toISOString() }, + identities: emptyArtifactIdentities({ stackAdapter: { id: args.backend } }), + payload: evidence, + }); + return evidence; + }; + const teardown = ({ reason = null, retainBackend = args.retainBackend }: + { reason?: string | null; retainBackend?: boolean } = {}) => { + if (tornDown) return; + activeAgentCancellation?.abort(); + activeAgentCancellation = null; + // Preserve restart failures before removing the only filesystem that holds + // their stderr. A 500 after restart is otherwise impossible to distinguish + // from an application defect, a dead dependency, or host pressure. + if (activeRun) { + try { + activeRun.backendDiagnostics = captureApplicationDiagnostics(join(outputDir, 'backend.log')); + } catch (error) { + activeRun.backendDiagnostics = { captured: false, + reason: errorMessage(error).split(/\r?\n/)[0] }; + } + } + let released = false; + let cleanupError: unknown = null; + try { + released = releaseBackendLease(leasePath, initialLease.ownershipToken, + { retainBackend }); + } catch (error) { cleanupError = error; } + let finalLease = initialLease; + try { + finalLease = readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId }); + } catch (error) { cleanupError ??= error; released = false; } + const evidence = writeLeaseEvidence(finalLease); + writeRecoveryArtifact(recoveryPath, finalLease, { cleanupSucceeded: released, + retained: Boolean(retainBackend), + reason: cleanupError === null ? reason ?? (released ? null : 'authenticated cleanup refused') + : errorMessage(cleanupError) }); + if (activeRun) { + activeRun.backendLease = evidence; + activeRun.outcome ??= aggregateRunOutcome(activeRun.levels); + persistRun(join(outputDir, ARTIFACT_FILE.run), activeRun); + } + tornDown = released; + if (released && !retainBackend && !finalLease.campaignDelegation) { + rmSync(runtimeDir, { recursive: true, force: true }); + if (privateSupervisorStatePath) rmSync(privateSupervisorStatePath, { force: true }); + } + if (cleanupError) throw cleanupError; + if (!released) throw new Error(`backend teardown refused: listener no longer matches lease ${runId}`); + }; + emergencyTeardown = teardown; + + try { + performPreflight(); + stackAdapter.lifecycle.activate({ + leasePath, leaseToken: initialLease.ownershipToken, lease: initialLease, + ports: assignedPorts, + ...stackRuntime.lifecycle, + }); + performPreflight(true); + } catch (error) { + try { teardown({ reason: `backend activation failed: ${errorMessage(error)}`, retainBackend: false }); } + catch (cleanupError) { + console.error(` activation cleanup quarantined: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`); + } + throw error; + } + + // Teardown stops only resources recorded in this run's lease. + const interrupt = (signal: NodeJS.Signals, exitCode: number) => { + console.log(`interrupted by ${signal} — stopping exact owned resources`); + try { teardown({ reason: `interrupted by ${signal}` }); } + catch (error) { console.error(` cleanup quarantined: ${errorMessage(error).split(/\r?\n/)[0]}`); } + process.exit(exitCode); + }; + process.on('SIGINT', () => interrupt('SIGINT', 130)); + process.on('SIGTERM', () => interrupt('SIGTERM', 143)); + process.on('exit', () => { + if (!tornDown) { + try { teardown(); } catch (error) { + console.error(` cleanup failed: ${errorMessage(error).split('\n')[0]}`); + } + } + }); + + // Seed source only; the upgrade session installs its own dependencies. + if (args.seedFrom) { + const from = resolve(args.seedFrom); + if (!existsSync(from)) { console.error(`--seed-from path does not exist: ${from}`); process.exit(2); } + seedAppSource(from, appDir); + if (args.progressionSeed) { + const seeded = hashAppSource(appDir); + if (seeded.sha256 !== args.progressionSeed.sourceSha256 + || seeded.files.length !== args.progressionSeed.sourceFiles) { + throw new Error('extension source does not match its recorded identity'); + } + } + console.log(args.repairGrant + ? ` restored L${args.levelList[0]} checkpoint from ${from} for a bounded repair continuation` + : ` seeded from ${from} — level ${args.levelList[0]} will UPGRADE it, not rebuild`); + } + + if (regrade) { + const startedAt = new Date().toISOString(); + let bundle: GradeBundlePayload | null = null; + let failure: string | null = null; + let diagnostics: unknown = null; + let cleanupFailure: unknown = null; + try { + seedAppSource(regrade.sourcePath, appDir); + // prepare-only owns the container but never invokes an agent or broker. + writeFileSync(join(dirname(appDir), '.stack-bench-isolation'), 'container'); + writeFileSync(join(dirname(appDir), '.stack-bench-backend'), args.backend); + sh(process.execPath, [compiledEntrypoint('container', 'run-build.js'), '--app', appDir, + '--backend', args.backend, '--image', process.env.STACK_BENCH_IMAGE!, + '--ports', [assignedPorts.vite, assignedPorts.express].filter(Boolean).join(','), '--prepare-only'], + { stdio: 'inherit', timeout: COMMAND_TIMEOUT_MS }); + const copiedSource = hashAppSource(appDir); + if (copiedSource.sha256 !== regrade.source.sha256 || copiedSource.files.length !== regrade.source.files) { + throw new Error('regrade source changed during preparation'); + } + await materializeAcceptedSource(regrade.sourcePath, appDir, restartSpecFor(args, appDir, track)); + bundle = grade(args, appDir, url, `${args.backend}-regrade`, regrade.declared.level, + track, runId, { out: join(outputDir, 'grading'), sourceSha256: regrade.source.sha256 }); + if (!bundle || bundle.source?.sha256 !== regrade.source.sha256) { + throw new Error('regrade produced no matching source-bound grade bundle'); + } + if (hashAppSource(appDir).sha256 !== regrade.source.sha256) { + throw new Error('application source changed during regrading'); + } + } catch (error) { + failure = errorMessage(error); + if (object(error) && typeof error.startLog === 'string') { + writeFileSync(join(outputDir, 'application-start.log'), error.startLog); + } + throw error; + } finally { + try { diagnostics = captureApplicationDiagnostics(join(outputDir, 'backend.log')); } + catch (error) { diagnostics = { captured: false, reason: errorMessage(error) }; } + try { teardown({ reason: failure, retainBackend: false }); } + catch (error) { failure ??= errorMessage(error); cleanupFailure = error; } + finally { + writeFileSync(join(outputDir, 'regrade.json'), JSON.stringify({ schemaVersion: 1, + kind: 'saved-source-regrade', diagnosticOnly: true, + id: runId, startedAt, completedAt: new Date().toISOString(), + parent: { id: regrade.parent.id, runSha256: regrade.runSha256, + completedAt: regrade.parent.timestamps.completedAt, + engine: regrade.parent.identities.engine, source: regrade.source, + serverUri: regrade.serverUri, + selectionSha256: regrade.declared.selection.sha256, + recipe: regrade.declared.recipe, task: regrade.declared.task, + mode: regrade.parent.payload.mode, outcome: regrade.parent.payload.outcome, + candidate: regrade.candidate }, + engine: currentEngineIdentity(), buildImage: process.env.STACK_BENCH_IMAGE, + controllerImage: process.env.STACK_BENCH_CONTROLLER_IMAGE_ID, + dependencyVolume: process.env.STACK_BENCH_RELEASE_DEPS_VOLUME, + backend: args.backend, track: args.track, level: regrade.declared.level, + recipe: args.recipeTasks.get(regrade.declared.level)!.request.recipe, + task: args.recipeTasks.get(regrade.declared.level)!.request.task, + selectionSha256: args.recipeTasks.get(regrade.declared.level)!.selection.sha256, + selectedChecks: checksForGrade(args.recipeTasks.get(regrade.declared.level), 'scored') + .map(check => check.stableKey), + grading: bundle ? 'grading/bundle.json' : null, + gradingSha256: bundle ? sha256(readFileSync(join(outputDir, 'grading', 'bundle.json'))) : null, + outcome: bundle ? classifyBundle(bundle) : null, failure, diagnostics, + cleanupSucceeded: tornDown, + modelCalls: 0, additionalModelCostUsd: 0, + comparisonNote: regrade.dependency + ? 'Dependency first-build diagnostic; not terminal completion, a new build, or a replacement for the original outcome or spend.' + : 'Regrades the original saved app; not an independent build or a replacement for its model cost.' }, null, 2) + '\n', + { flag: 'wx' }); + if (tornDown && ownWorkDir) rmSync(dirname(appDir), { recursive: true, force: true }); + } + } + if (cleanupFailure) throw cleanupFailure; + console.log(`Saved-source regrade: ${classifyBundle(bundle).kind}; evidence ${outputDir}`); + process.exitCode = runExitCode(classifyBundle(bundle)); + return; + } + const started = Date.now(); + const run: BenchmarkRunRecord = { id: runId, + ...(args.repairGrant ? { kind: 'repair_continuation', + continuation: structuredClone(args.repairGrant.grant) } : {}), + startedAt: new Date(started).toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + experiment: args.experimentIdentity ?? null, + agentAdapter: agentAdapterIdentity(agentAdapter), + stackAdapter: { id: stackAdapter.id, version: stackAdapter.version }, + }), + mode: args.runMode ?? { id: args.progression ? 'dependency' : 'sequential' }, + track: args.track, backend: args.backend, model: args.model, + ...(args.providerRoute ? { providerRoute: args.providerRoute } : {}), + ...(args.maxOutputTokens ? { maxOutputTokens: args.maxOutputTokens } : {}), + pricing: args.pricing, + guidance: args.guidance, condition: args.condition ?? null, + ...(args.productionQuality && agentAdapter.provider ? { productionQuality: true } : {}), + skills: args.skills ?? [], + runtime: { buildImage: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, url }, + selectionRequest: args.selectionRequest, + featureCatalog: args.featureCatalog?.identity ?? null, + dependencyPolicy: args.dependencyPolicy?.identity ?? null, + ...(args.progressionOwner ? { progressionOwner: args.progressionOwner } : {}), + ...(args.progressionSeed ? { progressionSeed: { + fromDepth: args.progressionSeed.fromDepth, + sourceSha256: args.progressionSeed.sourceSha256, + sourceFiles: args.progressionSeed.sourceFiles, + parent: structuredClone(args.progressionSeed.parent), + validatedDepths: [], + } } : {}), + backendLease: publicBackendLease(readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId })), + validation: { + ladder: { policy: args.progression ? args.progression.identity.policy : 'pass-before-next-level', + requestedLevels: [...args.levelList], + completedLevels: [], stoppedAfterLevel: null, blockedLevels: [] } }, levels: [] }; + activeRun = run; + + const progressionOwner = args.progression ? { + ...args.progressionOwner, + workspace: { appDirectory: 'source' }, + } : null; + const progressionExecution = args.progression ? createLiveProgressionExecution({ + progression: args.progression, + featureCatalogIdentity: args.featureCatalog?.identity, + dependencyPolicyIdentity: args.dependencyPolicy?.identity, + owner: progressionOwner, + statePath: join(args.out, ARTIFACT_FILE.progressionState), + runId, + outputDir: args.out, + appDir, + track: args.track, + backend: args.backend, + identities: run.identities, + recipeBindings: args.recipeBindings, + retainPriorContracts: args.retainPriorContracts ?? true, + resumeFrom: args.progressionResumeFrom ?? null, + getRunArtifact: () => { + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + return readArtifact(join(outputDir, ARTIFACT_FILE.run)); + }, + onState: status => { + run.progressionStatus = status; + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + }, + }) : null; + const progressionStart = progressionExecution?.initialize() ?? null; + if (progressionStart?.resumed) { + const prior = progressionStart.priorRun; + if (!prior) throw new Error('resumed dependency progression has no prior run artifact'); + const actionLevel = progressionStart.action.type === 'terminal' + ? Number.MAX_SAFE_INTEGER : progressionStart.action.level; + const inheritedLevels = (prior.payload.levels ?? []) + .filter(level => level.level < actionLevel).map(level => level.level); + run.levels = (prior.payload.levels ?? []) + .filter(level => inheritedLevels.includes(level.level)).map(level => structuredClone(level)); + run.validation.ladder.completedLevels = [...inheritedLevels]; + run.progressionResume = { + priorRunId: prior.id, + priorRunSha256: sha256(canonicalDefinitionJson(prior)), + stateSha256: progressionStart.stateSha256, + action: progressionStart.action.type === 'terminal' + ? { type: 'terminal' } + : { type: progressionStart.action.type, level: progressionStart.action.level }, + inheritedLevels, + priorTotals: prior.payload.totals ?? null, + }; + run.progressionStatus = progressionStart.status; + persistRun(join(args.out, ARTIFACT_FILE.run), run); + } + + const bindProgressionAction = (level: number): ProgressionRecipeAction | null => { + if (!progressionExecution) return null; + const selected = progressionExecution.bind(level); + if (!isProgressionWorkRecipeAction(selected)) return selected; + if (!args.recipeTasks) throw new Error('recipe task map is unavailable'); + args.recipeTasks.set(level, { + request: selected.grader.request, + selection: selected.grader.selection, + task: selected.grader.task, + agentRequest: selected.agent.request, + progressionAction: selected.action, + }); + return selected; + }; + + const progressionBundles = new Map(); + const recordProgressionGrade = (input: Parameters['record']>[0]) => { + const next = progressionExecution?.record(input) ?? null; + const last = progressionExecution?.state?.attempts.at(-1) ?? null; + if (input.bundle && input.selected && isProgressionWorkRecipeAction(input.selected) + && last?.outcome === 'conclusive') { + progressionBundles.set(input.level, input.bundle as GradeBundlePayload); + } + return next; + }; + + const appendLevelRecord = (record: RunLevelRecord): void => { + pendingLevel = null; + if (args.dependencyPolicy?.definition.workSelection !== 'feature') { + run.levels.push(record); + return; + } + const index = run.levels.findIndex(candidate => candidate.level === record.level); + if (index < 0) run.levels.push(record); + else run.levels[index] = mergeFeatureLevelRecord(run.levels[index] ?? null, record); + }; + + const runAgentForLevel = async (mode: AgentMode, level: number, + onFailure?: () => Promise): Promise => { + try { + clearPrivateGradingEvidence(appDir); + pendingLevel ??= { level, graded: false, score: null, max: null, selection: null, + outcome: { kind: 'ungraded' }, buildSessions: [], repairSessions: [] }; + sessionInFlight = true; + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + const result = await runAgent(args, agentAdapter, mode, level, appDir); + sessionInFlight = false; + const paid = runSessionRecord(result); + if (mode === 'fix') { + pendingLevel.repairSessions!.push(paid); + pendingLevel.repairCostUsd = addCostUsd(pendingLevel.repairCostUsd, paid.costUsd); + pendingLevel.repairs = (pendingLevel.repairs ?? 0) + 1; + } else if (mode === 'resume') { + pendingLevel.resumeSession = paid; + pendingLevel.resumeCostUsd = paid.costUsd; + } else { + pendingLevel.buildSessions!.push(paid); + pendingLevel.buildCostUsd = addCostUsd(pendingLevel.buildCostUsd, paid.costUsd); + } + pendingLevel.costUsd = addCostUsd(pendingLevel.buildCostUsd, pendingLevel.resumeCostUsd, + pendingLevel.repairCostUsd); + pendingLevel.sessionTotals = summarizeSessions([...pendingLevel.buildSessions!, + ...(pendingLevel.resumeSession ? [pendingLevel.resumeSession] : []), ...pendingLevel.repairSessions!]); + if (result.costComplete !== true) runCostComplete = false; + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + return result; + } catch (error) { + await onFailure?.(); + const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'agent execution failed'; + const capped = reason.startsWith('Cost cap reached:'); + run.outcome = { kind: capped ? 'provider_failure' : 'harness_failure', phase: capped ? 'cost-cap' : `agent-${mode}`, + reason, appFailures: [], inconclusive: [], harnessFailures: capped ? [] : [reason] }; + run.validation.ladder.stoppedAfterLevel = run.levels.at(-1)?.level ?? null; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + if (progressionExecution) { + recordProgressionGrade({ selected: progressionExecution.bind(level), bundle: null, level, + failure: progressionFailure(run.outcome) }); + run.progressionStatus = progressionExecution.status(); + synchronizeProgressionSummary(run, requireProgressionState(progressionExecution.state)); + } + finalizeRunTotals(run, started, { costComplete: false }); + run.completedAt = new Date().toISOString(); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + throw error; + } + }; + + // Stop before grading if a coding session read protected material or if the + // audit itself failed. Keep the paid session and exact cost in the run artifact even + // though no score may be used. + const abortUnusableSession = (whichSession: string, audit: ContaminationAudit, + levelRecord: UnknownRecord & { level: number }, + selected: ProgressionRecipeAction | null, completedRepair = false) => { + const reason = audit.evidence.join('; '); + const outcome: RunOutcome = { kind: audit.kind === 'harness_failure' ? 'harness_failure' : 'ungraded', + phase: 'contamination-audit', reason, + appFailures: [], inconclusive: [], + harnessFailures: audit.kind === 'harness_failure' ? [reason] : [] }; + run.contaminated = audit.kind === 'contaminated'; + run.contamination = { evidence: audit.evidence, verdict: audit.verdict, + detectedAt: whichSession }; + const record: RunLevelRecord = { ...levelRecord, error: reason, outcome, + level: levelRecord.level, graded: false, score: null, max: null, selection: null }; + appendLevelRecord(record); + if (progressionExecution) { + recordProgressionGrade({ selected, bundle: null, level: levelRecord.level, + failure: progressionFailure(outcome), + completedRepair }); + run.progressionStatus = progressionExecution!.status(); + synchronizeProgressionSummary(run, requireProgressionState(progressionExecution.state)); + } + run.validation.ladder.stoppedAfterLevel = run.levels.at(-2)?.level ?? null; + run.validation.ladder.blockedLevels = args.levelList + .filter(candidate => candidate >= levelRecord.level); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + run.outcome = outcome; + run.completedAt = new Date().toISOString(); + if (run.contaminated) { + console.log(`\n CONTAMINATED at ${whichSession}:`); + for (const evidence of audit.evidence) console.log(` ${evidence}`); + console.log(' Scores from this run must not be quoted.'); + } else { + console.log(`\n HARNESS FAILURE at ${whichSession}:`); + for (const evidence of audit.evidence) console.log(` ${evidence}`); + console.log(' The audit did not establish a usable result.'); + } + try { persistRun(join(outputDir, ARTIFACT_FILE.run), run); } catch { /* best effort */ } + try { archiveTranscripts(appDir, artifactLabel); } catch { /* best effort */ } + teardown(); + process.exit(4); + }; + + for (let levelIndex = 0; levelIndex < args.levelList.length; levelIndex += 1) { + const pauseDepth = args.pauseAfterDepth; + const activeState = progressionExecution?.state; + if (pauseDepth !== undefined && activeState?.phase === 'active' + && depthPauseDue(run, pauseDepth, activeState.level)) { + const context = readDepthPauseContext(); + if (!context || context.depth !== pauseDepth) throw new Error('planned depth pause has no controller authority'); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + run.pausedDurationMs = await waitAtDepthBoundary(outputDir, appDir, context); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + } + // A previous checkpoint cannot cover a newer session or partial grade. + clearTimeContinuationBoundary(outputDir); + const level = args.levelList[levelIndex]!; + const t0 = Date.now(); + const continuing = Boolean(args.repairGrant); + console.log(`\n================ ${args.backend} — level ${level} ================`); + + let progressionSelection = bindProgressionAction(level); + if (progressionSelection?.action.type === 'terminal') break; + if (args.dependencyPolicy?.definition.workSelection === 'all-at-once' + && progressionSelection?.action.level !== level) continue; + const applicationControl = materializeCodingOutput + ? restartSpecFor(args, appDir, track) : null; + const featureActionSequence = args.dependencyPolicy?.definition.workSelection === 'feature' + ? requireProgressionState(progressionExecution?.state ?? null).attempts.length + 1 : null; + const featureActionSuffix = featureActionSequence === null + ? '' : `-action${String(featureActionSequence).padStart(3, '0')}`; + // A clean-source start that fails voids a grade. Its launch log is the + // only account of why, so it stays beside the run. + const keepStartLog = (error: unknown, label: string): void => { + const startLog = error !== null && typeof error === 'object' && 'startLog' in error + ? error.startLog : null; + if (typeof startLog !== 'string' || !startLog) return; + writeFileSync(join(outputDir, `${label}-start.log`), `${startLog}\n`); + }; + const restoreFeatureAcceptedSource = async (resetDatabase = true): Promise => { + if (featureActionSequence === null) return; + const source = join(outputDir, 'source'); + try { + if (applicationControl) { + const restore = resetDatabase ? restoreRepairSource : materializeAcceptedSource; + await restore(source, appDir, applicationControl); + } else resetAppToSource(source, appDir); + } catch (error) { + console.log(` accepted feature restore failed: ${errorMessage(error)}`); + try { keepStartLog(error, `${args.backend}-l${level}${featureActionSuffix}-feature-restore`); } + catch (logError) { console.log(` could not preserve start log: ${errorMessage(logError)}`); } + // Failed coding/grading callers must still finalize their original evidence. + // The durable accepted source remains in outputDir even if local restore fails. + try { resetAppToSource(source, appDir); } + catch (restoreError) { console.log(` source restore failed: ${errorMessage(restoreError)}`); } + } + }; + if (featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection) + && !featureActionNeedsCoding(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null))) { + await restoreFeatureAcceptedSource(false); + const bundle = grade(args, appDir, url, + `${args.backend}-l${level}${featureActionSuffix}-regrade`, level, track, runId); + const outcome = classifyBundle(bundle); + const repair = { status: levelGradeIsUsable(outcome) ? 'not-needed' as const : 'ungraded' as const, + limit: 0, used: 0, stopReason: 'accepted-source-regrade' }; + let next = recordProgressionGrade({ selected: progressionSelection, bundle, + level }); + let finalOutcome = outcome; + const currentState = requireProgressionState(progressionExecution?.state ?? null); + if (next?.type === 'build' && next.level === level + && !featureActionNeedsCoding(progressionSelection, currentState)) { + const reason = 'accepted source still has ungraded checks after regrade'; + finalOutcome = { kind: 'harness_failure', phase: 'feature-regrade', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + next = recordProgressionGrade({ selected: progressionSelection, bundle: null, + level, failure: progressionFailure(finalOutcome) }); + } + const state = requireProgressionState(progressionExecution?.state ?? null); + const graded = levelGradeIsUsable(finalOutcome) + && state.attempts.at(-1)?.outcome === 'conclusive'; + let checkpoint = null; + if (graded && (state.phase === 'terminal' || state.level > level)) { + checkpoint = preserveLevelCheckpoint({ appDir, outputDir, runId, + identities: run.identities, track: args.track, backend: args.backend, level, + repair, outcome: finalOutcome, + selectionSha256: bundle?.selection?.sha256 ?? null }); + } + appendLevelRecord({ level, graded, + score: graded ? bundle?.totals?.score ?? null : null, + max: graded ? bundle?.totals?.max ?? null : null, + selection: graded ? bundle?.selection ?? null : null, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + code: bundle?.code ?? null, + repair, + checkpoint, + sessionTotals: summarizeSessions([]), + costUsd: 0, + repairs: 0, + durationSec: Math.round((Date.now() - t0) / 1000), + outcome: finalOutcome }); + if (graded && (state.phase === 'terminal' || state.level > level) + && !run.validation.ladder.completedLevels.includes(level)) { + run.validation.ladder.completedLevels.push(level); + } + run.progressionStatus = progressionExecution!.status(); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + if (state.phase === 'terminal') break; + if (state.attempts.at(-1)?.outcome === 'inconclusive') { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + break; + } + if (next?.type !== 'terminal' && next?.level === level) { + levelIndex -= 1; + continue; + } + if (next?.type !== 'terminal' && next && next.level < level) { + throw new Error(`dependency progression moved backward from L${level}`); + } + continue; + } + if (args.seedThrough !== undefined && level <= args.seedThrough) { + if (!progressionSelection || !isProgressionWorkRecipeAction(progressionSelection) + || progressionSelection.action.level !== level || !args.seedFrom || !run.progressionSeed) { + throw new Error(`extension cannot validate depth ${level}`); + } + let applicationFailure: RunOutcome | null = null; + if (applicationControl) { + try { + await materializeAcceptedSource(args.seedFrom, appDir, applicationControl); + } catch (error) { + applicationFailure = materializationAppFailure(error); + keepStartLog(error, `${args.backend}-extension-l${level}`); + } + } else { + resetAppToSource(args.seedFrom, appDir); + } + const bundle = grade(args, appDir, url, `${args.backend}-extension-l${level}`, + level, track, runId, { applicationFailure }); + const outcome = applicationFailure ?? classifyBundle(bundle); + const next = recordProgressionGrade({ selected: progressionSelection, bundle, level }); + const progressionState = requireProgressionState(progressionExecution?.state ?? null); + const progressionAttempt = progressionState.attempts.at(-1) ?? null; + const graded = levelGradeIsUsable(outcome, progressionAttempt); + const passed = graded && outcome.kind === 'passed'; + const repair = { status: passed ? 'not-needed' as const : 'incomplete' as const, + limit: 0, used: 0, + stopReason: passed ? 'not-needed' : 'extension-validation-failed' }; + let checkpoint = null; + try { + checkpoint = preserveLevelCheckpoint({ appDir, outputDir: args.out, runId, + identities: run.identities, track: args.track, backend: args.backend, level, + repair, outcome, selectionSha256: bundle?.selection?.sha256 ?? null }); + } catch (error) { + throw new Error(`could not preserve extension depth ${level}: ${errorMessage(error)}`); + } + const source = hashAppSource(appDir); + appendLevelRecord({ level, graded, + score: graded ? bundle?.totals?.score ?? null : null, + max: graded ? bundle?.totals?.max ?? null : null, + selection: bundle?.selection ?? null, + baseline: { kind: 'extension-validation', source: { + sha256: source.sha256, files: source.files.length } }, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + code: bundle?.code ?? null, + repair, + checkpoint, + sessionTotals: summarizeSessions([]), + costUsd: 0, + repairs: 0, + durationSec: Math.round((Date.now() - t0) / 1000), + outcome }); + if (progressionAttempt?.outcome === 'conclusive') { + if (!run.validation.ladder.completedLevels.includes(level)) { + run.validation.ladder.completedLevels.push(level); + } + } + run.progressionStatus = progressionExecution!.status(); + if (passed) run.progressionSeed.validatedDepths.push(level); + if (!passed) { + run.validation.ladder.stoppedAfterLevel = level > 1 ? level - 1 : null; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + persistRun(join(args.out, ARTIFACT_FILE.run), run); + if (!next || next.type === 'terminal' || next.level === undefined || next.level <= level) { + throw new Error(`extension did not advance after depth ${level}`); + } + continue; + } + const resumedRepair = progressionStart?.resumed === true + && progressionStart.action.type === 'repair'; + // The interrupted repair was charged but never graded. Grade its preserved + // source before any coding session. + const resumedGrade = resumedRepair && progressionStart?.action.type === 'repair' + && progressionStart.action.repair.awaitingGrade === true; + const resumedRegression = resumedRepair + ? savedRepairRegression(progressionExecution?.state ?? null, progressionSelection) : null; + const resumedRegressionReport = resumedRegression + ? join(args.out, 'repair-reports', + `rejected-regression-l${level}${featureActionSuffix}-resume.md`) : null; + const priorRepairs = resumedRepair + ? progressionStart.priorRun?.payload.levels?.find(item => item.level === level) + ?.repair?.used ?? 0 + : args.progression ? 0 : run.levels.reduce((sum, item) => sum + (item.repairs ?? 0), 0); + const repairBudgetFor = (selected: ProgressionRecipeAction | null, + completedRepairs: number) => selected + && isProgressionWorkRecipeAction(selected) + ? dependencyRepairBudget(selected.action, completedRepairs) + : args.repairs; + const levelRepairNodeIds = new Set(progressionSelection?.action.repair.nodeIds ?? []); + let progressionRepairLimit = repairBudgetFor( + progressionSelection, priorRepairs); + const trackProgressionBudget = (selected: ProgressionRecipeAction | null, + completedRepairs: number) => { + if (!selected || !isProgressionWorkRecipeAction(selected)) return; + selected.action.repair.nodeIds.forEach(nodeId => levelRepairNodeIds.add(nodeId)); + progressionRepairLimit = Math.max( + progressionRepairLimit, repairBudgetFor(selected, completedRepairs)); + }; + if (resumedRepair && !resumedGrade) { + let reportFailure: string | null = null; + try { + if (resumedRegressionReport && resumedRegression) { + mkdirSync(dirname(resumedRegressionReport), { recursive: true }); + writeFileSync(resumedRegressionReport, resumedRegression.report); + } + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + '--history-json', '[]', '--archive', join(args.out, 'repair-reports', + `bug-report-l${level}${featureActionSuffix}-resume.md`), + ...(resumedRegressionReport ? ['--prior-regression', resumedRegressionReport] : []), + ...repairReportArgs(progressionSelection)], + { stdio: 'pipe' }); + } catch (error) { + reportFailure = errorMessage(error).split(/\r?\n/)[0] + ?? 'repair report generation failed'; + } + if (reportFailure) { + const outcome: RunOutcome = { + kind: 'harness_failure', phase: 'repair-report', reason: reportFailure, + appFailures: [], inconclusive: [], harnessFailures: [reportFailure], + }; + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: progressionFailure(outcome) }); + appendLevelRecord({ level, graded: false, score: null, max: null, + selection: null, error: reportFailure, outcome, + repair: { status: 'ungraded', limit: progressionRepairLimit, + used: priorRepairs, stopReason: 'repair-report' }, + repairCostUsd: 0, repairSessions: [], repairs: 0, priorRepairs, + cumulativeRepairs: priorRepairs, sessionTotals: summarizeSessions([]), + costUsd: 0, durationSec: Math.round((Date.now() - t0) / 1000) }); + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + } + + const firstMode = resumedRepair ? 'fix' + : continuing ? 'resume' : args.seedFrom ? 'upgrade' : 'build'; + const build = resumedGrade ? null : await runAgentForLevel( + resumedRepair || run.levels.length === 0 ? firstMode : 'upgrade', level, + featureActionSequence === null ? undefined : restoreFeatureAcceptedSource); + // Only a resumed grade runs a level without a coding session. + const requireBuild = (): NonNullable => { + if (!build) throw new Error(`level ${level} has no coding session`); + return build; + }; + const buildFailure = build ? agentSessionFailure(build) : null; + const buildLeak = build && !buildFailure ? auditContamination(appDir, auditNetwork(), auditsTranscripts) : null; + if (buildLeak) { + const session = requireBuild(); + const buildSession = runSessionRecord(session, + resumedRepair ? priorRepairs + 1 : null); + const sessionTotals = summarizeSessions([buildSession]); + abortUnusableSession(`level ${level} ${firstMode}`, buildLeak, { + level, graded: false, score: null, max: null, selection: null, + ...(resumedRepair + ? { repairCostUsd: session.costUsd, repairSessions: [buildSession], repairs: 1, + priorRepairs, cumulativeRepairs: priorRepairs + 1 } + : continuing + ? { resumeCostUsd: session.costUsd, resumeSession: buildSession } + : { buildCostUsd: session.costUsd, buildSessions: [buildSession] }), + sessionTotals, costUsd: session.costUsd, durationMs: Date.now() - t0, + }, progressionSelection, resumedRepair); + } + // Record the session setup needed to compare runs. + if (build) run.setup ??= build.setup; + if (continuing) { + const session = requireBuild(); + requireContinuation(run).resumeSetup = { + sessionId: session.sessionId ?? null, + costUsd: session.costUsd, + durationMs: session.durationMs, + sourceVerified: false, + }; + } + // No session, no app. Grading an empty directory yields a real-looking zero + // that is a harness failure, not a result for this backend. + if (buildFailure) { + const session = requireBuild(); + await restoreFeatureAcceptedSource(); + console.log(` ABORTED: ${buildFailure.reason}. Details will be kept in ${join(args.out, ARTIFACT_FILE.run)}`); + const failedSession = runSessionRecord(session); + if (progressionExecution) { + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: buildFailure }); + } + appendLevelRecord({ level, graded: false, score: null, max: null, + selection: null, error: buildFailure.reason, + outcome: buildFailure, + ...(continuing + ? { resumeSession: failedSession, resumeCostUsd: session.costUsd } + : { buildSessions: [failedSession], buildCostUsd: session.costUsd }), + sessionTotals: summarizeSessions([session]), + costUsd: session.costUsd, durationMs: Date.now() - t0 }); + break; + } + const gradeAcceptedSource = async (sourcePath: string, + label: string): Promise => { + let failure: RunOutcome | null = null; + if (applicationControl) { + try { + await materializeAcceptedSource(sourcePath, appDir, applicationControl); + } catch (error) { + failure = materializationAppFailure(error); + keepStartLog(error, label); + } + } else { + resetAppToSource(sourcePath, appDir); + } + return grade(args, appDir, url, label, level, track, runId, + { applicationFailure: failure }); + }; + const checkpointGrade = (phase: RunCheckpoint['phase'], measured: GradeBundlePayload, + sessions: RunSessionRecord[], accepted: boolean): void => { + if (!args.condition?.requested?.levels.length || !measured.source?.sha256 + || !measured.selection?.sha256) return; + const source = join(privateGradingDirectory(appDir), ARTIFACT_FILE.gradeBundle); + const evidence = readArtifactPayload(source, { expectedKind: 'grade_bundle' }); + if (evidence.source?.sha256 !== measured.source.sha256 + || evidence.selection?.sha256 !== measured.selection.sha256) { + throw new Error('grading checkpoint does not match the current source and selection'); + } + const path = `checkpoints/${(run.checkpoints?.length ?? 0) + 1}.json`; + mkdirSync(join(outputDir, 'checkpoints'), { recursive: true }); + copyFileSync(source, join(outputDir, path)); + const prior = progressionStart?.priorRun?.payload; + const priorCheckpoint = Array.isArray(prior?.checkpoints) + ? prior.checkpoints.map(value => checkpointSchema.parse(value)).findLast(checkpoint => checkpoint.accepted) + : undefined; + const prompt = progressionSelection?.action.prompt; + recordRunCheckpoint({ ...run, condition: args.condition, checkpoints: run.checkpoints ??= [] }, + { phase, level, bundle: evidence, + sourceSha256: measured.source.sha256, + evidence: { path, sha256: sha256(readFileSync(join(outputDir, path))) }, + extraSessions: sessions, accepted, + ...(prior ? { priorCost: runCostEvidence(prior), + initialChecks: priorCheckpoint?.checks } : {}), + workNodeIds: object(prompt) && Array.isArray(prompt.nodeIds) + ? prompt.nodeIds.filter((id): id is string => typeof id === 'string') : [] }); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + }; + const restoreAcceptedRepair = async (sourcePath: string, gradingPath: string, + completedRepair = true): Promise => { + try { + try { + if (applicationControl) await restoreRepairSource(sourcePath, appDir, applicationControl); + else resetAppToSource(sourcePath, appDir); + } finally { + restorePrivateGradingEvidence(appDir, gradingPath); + } + return true; + } catch (error) { + let reason = `could not restore the accepted repair source: ${errorMessage(error)}`; + const preserved = join(outputDir, `repair-rollback-l${level}${featureActionSuffix}-round${repairs}`); + try { + snapshotSource(sourcePath, join(preserved, 'source')); + if (existsSync(gradingPath)) cpSync(gradingPath, join(preserved, 'grading'), { recursive: true }); + keepStartLog(error, `${args.backend}-l${level}${featureActionSuffix}-rollback${repairs}`); + reason += `; accepted source and available grading evidence kept at ${preserved}`; + } catch (preserveError) { + reason += `; could not preserve the accepted rollback snapshot: ${errorMessage(preserveError)}`; + } + console.log(` ${reason}; stopping repairs`); + recordRepairHarnessFailure('repair-restore', reason, null, completedRepair); + return false; + } + }; + // Keep a repaired source whose grade did not finish beside the run, bound + // by hash, so a resume can grade exactly what the paid session produced. + const preserveRepairCandidate = (directory: string): RunRepairCandidate => { + const path = join(outputDir, directory); + rmSync(path, { recursive: true, force: true }); + const live = hashAppSource(appDir); + snapshotSource(appDir, path); + const preserved = hashDirectory(path); + if (live.sha256 !== preserved.sha256 || live.files.length !== preserved.files.length) { + throw new Error('preserved repair source differs from the live application source'); + } + return { directory, sha256: preserved.sha256, files: preserved.files.length }; + }; + if (resumedGrade) { + const candidate = progressionStart?.priorRun?.payload.levels + ?.find(item => item.level === level)?.repair?.candidate; + const candidateRoot = args.progressionResumeFrom ?? outputDir; + let candidateFailure: string | null = null; + if (!candidate || !/^[A-Za-z0-9._-]+$/.test(candidate.directory) + || !existsSync(join(candidateRoot, candidate.directory))) { + candidateFailure = 'the interrupted repair left no source to grade'; + } else { + const candidatePath = join(candidateRoot, candidate.directory); + const preserved = hashDirectory(candidatePath); + if (preserved.sha256 !== candidate.sha256 + || preserved.files.length !== candidate.files) { + candidateFailure = 'the interrupted repair source does not match its record'; + } else { + resetAppToSource(candidatePath, appDir); + console.log(` restored the interrupted repair source from ${candidatePath}`); + } + } + if (candidateFailure) { + const outcome: RunOutcome = { + kind: 'harness_failure', phase: 'repair-candidate', reason: candidateFailure, + appFailures: [], inconclusive: [], harnessFailures: [candidateFailure], + }; + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: progressionFailure(outcome) }); + appendLevelRecord({ level, graded: false, score: null, max: null, + selection: null, error: candidateFailure, outcome, + repair: { status: 'ungraded', limit: progressionRepairLimit, + used: priorRepairs, stopReason: 'repair-candidate' }, + repairCostUsd: 0, repairSessions: [], repairs: 0, priorRepairs, + cumulativeRepairs: priorRepairs, sessionTotals: summarizeSessions([]), + costUsd: 0, durationSec: Math.round((Date.now() - t0) / 1000) }); + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + } + if (continuing) { + // A resume may restore runtime state but cannot change checkpoint source. + const resumed = hashAppSource(appDir); + const repairGrant = args.repairGrant; + if (!repairGrant) throw new Error('repair continuation has no grant'); + if (resumed.sha256 !== repairGrant.checkpoint.payload.source.sha256 + || resumed.files.length !== repairGrant.checkpoint.payload.source.files) { + throw new Error('resume setup changed the parent checkpoint source'); + } + const continuation = requireContinuation(run); + if (!continuation.resumeSetup) throw new Error('repair continuation did not record resume setup'); + continuation.resumeSetup.sourceVerified = true; + } + if (args.referenceMutationOnly) { + const session = requireBuild(); + appendLevelRecord({ level, score: null, max: null, graded: false, contractPass: null, + selection: null, + outcome: { kind: 'ungraded', phase: 'reference-mutation-only', + reason: 'the parent qualification owns the full clean grade', + appFailures: [], inconclusive: [], harnessFailures: [] }, + buildSessions: [runSessionRecord(session)], + buildCostUsd: session.costUsd, sessionTotals: summarizeSessions([session]), + costUsd: session.costUsd, durationMs: Date.now() - t0 }); + break; + } + const firstBuildDirectory = continuing + ? `baseline-l${level}${featureActionSuffix}` + : `first-build-l${level}${featureActionSuffix}`; + const firstBuildPath = join(args.out, firstBuildDirectory); + let firstBuildSource = null; + let materializationOutcome: RunOutcome | null = null; + try { + const liveSource = hashAppSource(appDir); + snapshotSource(appDir, firstBuildPath); + const preservedSource = hashDirectory(firstBuildPath); + if (liveSource.sha256 !== preservedSource.sha256) { + throw new Error('preserved first-build source differs from the live application source'); + } + firstBuildSource = { sha256: liveSource.sha256, files: liveSource.files.length }; + if (applicationControl) { + await materializeAcceptedSource(firstBuildPath, appDir, applicationControl); + } + console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} source at ${firstBuildPath}`); + } catch (error) { + if (firstBuildSource) { + materializationOutcome = materializationAppFailure(error); + keepStartLog(error, `${args.backend}-l${level}${featureActionSuffix}`); + } + console.log(materializationOutcome + ? ` application failure: ${materializationOutcome.reason}` + : ` harness failure: could not bind the first-build source: ${errorMessage(error).split('\n')[0]}`); + } + const firstBuildLabel = `${args.backend}-l${level}${featureActionSuffix}`; + let bundle = firstBuildSource + ? grade(args, appDir, url, firstBuildLabel, level, track, runId, + { applicationFailure: materializationOutcome }) : null; + let reusableRepairEvidence: { + bundle: GradeBundlePayload; + results: string; + } | null = null; + + // Keep the unaided result separate from the result after repairs. + const firstBuild: FirstBuildRecord = { + score: bundle?.totals?.score ?? null, + max: bundle?.totals?.max ?? null, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + outcome: materializationOutcome ?? sourceBoundFirstBuildOutcome(bundle, firstBuildSource), + source: firstBuildSource, + missed: Object.values(bundle?.suites ?? {}).flatMap(s => + (s?.features ?? []).flatMap(f => + (f.criteria ?? []).filter(c => !evidencePassed(criterionEvidence(c))) + .map(c => `${f.name}/${c.id}`))), + }; + + if (continuing) { + const repairGrant = args.repairGrant; + if (!repairGrant) throw new Error('repair continuation has no grant'); + if (firstBuild.score === null || firstBuild.max === null || firstBuildSource === null) { + throw new Error('repair continuation did not produce a source-bound baseline score'); + } + const reproduction = compareRepairBaseline(repairGrant.level, { + score: firstBuild.score, + max: firstBuild.max, + selectionSha256: bundle?.selection?.sha256 ?? null, + sourceSha256: firstBuildSource.sha256, + expectedSourceSha256: repairGrant.checkpoint.payload.source.sha256, + outcome: repairOutcome(firstBuild.outcome), + }); + requireContinuation(run).baseline = { + score: firstBuild.score, + max: firstBuild.max, + selectionSha256: bundle?.selection?.sha256 ?? null, + sourceSha256: firstBuildSource?.sha256 ?? null, + outcome: firstBuild.outcome, + ...reproduction, + }; + if (!reproduction.reproduced) { + const reason = `restored checkpoint did not reproduce its parent: ${reproduction.mismatches.join(', ')}`; + console.log(` CONTINUATION STOPPED: ${reason}`); + const failure = { kind: 'harness_failure', phase: 'continuation-baseline', reason, + appFailures: [], inconclusive: [], harnessFailures: [] }; + firstBuild.outcome = failure; + bundle = { ...bundle, outcome: failure }; + } + } + + if (bundle && firstBuildSource && levelGradeIsUsable(firstBuild.outcome)) { + const accepted = !(featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection)) + || featureCandidateAccepted(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null), bundle); + checkpointGrade(resumedRepair ? 'repair' : 'first-build', bundle, + build ? [runSessionRecord(build)] : [], accepted); + } + + const selectedObservedChecks = checksForGrade(args.recipeTasks?.get(level), 'observed'); + if (!continuing && !resumedRepair && selectedObservedChecks.length) { + const observationOut = join(args.out, + `first-build-l${level}${featureActionSuffix}-observed`); + let observationBundle = null; + let observationOutcome; + if (!firstBuildSource) { + observationOutcome = { kind: 'harness_failure', phase: 'first-build-source', + reason: 'observed specifications require a source-bound first build' }; + } else if (!ladderMayContinue(firstBuild.outcome)) { + observationOutcome = { kind: 'ungraded', phase: 'first-build-observation', + reason: 'scored first-build grading did not establish a usable environment' }; + } else { + observationBundle = grade(args, appDir, url, `${args.backend}-l${level}-observed`, level, + track, runId, { observation: 'observed', out: observationOut, + sourceSha256: firstBuildSource.sha256 }); + observationOutcome = classifyBundle(observationBundle); + } + firstBuild.observations = { + sourceSha256: firstBuildSource?.sha256 ?? null, + selectionSha256: args.recipeTasks?.get(level)?.selection.sha256 ?? null, + selectedChecks: selectedObservedChecks.map(check => check.stableKey), + reportedChecks: observationBundle?.selection?.reportedChecks ?? [], + passedPoints: observationBundle?.totals?.score ?? null, + observedPoints: observationBundle?.totals?.max ?? null, + scoreContribution: false, + repairVisible: false, + artifact: observationBundle + ? `first-build-l${level}${featureActionSuffix}-observed/${ARTIFACT_FILE.gradeBundle}` : null, + outcome: observationOutcome, + }; + } + + let initialProgressionFailure: ProgressionFailure | null = null; + if (featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection)) { + const candidateOutcome = classifyBundle(bundle); + archiveCandidateGrade(appDir, outputDir, `l${level}${featureActionSuffix}`); + if (!levelGradeIsUsable(candidateOutcome)) { + await restoreFeatureAcceptedSource(); + initialProgressionFailure = progressionFailure(candidateOutcome); + } else if (!featureCandidateAccepted(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null), + bundle)) { + bundle = await gradeAcceptedSource(join(outputDir, 'source'), + `${args.backend}-l${level}${featureActionSuffix}-restored`, + ); + const restoredOutcome = classifyBundle(bundle); + if (!levelGradeIsUsable(restoredOutcome)) { + initialProgressionFailure = progressionFailure(restoredOutcome); + } + } + } + + // Preserve the first source and scored grading before repair overwrites the + // app. Observed evidence remains in its own source-bound result directory. + const acceptedGradingDirectory = continuing + ? `baseline-l${level}${featureActionSuffix}-grading` + : `first-build-l${level}${featureActionSuffix}-grading`; + try { + const gradingFrom = privateGradingDirectory(appDir); + if (existsSync(gradingFrom)) { + const gradingTo = join(args.out, acceptedGradingDirectory); + cpSync(gradingFrom, gradingTo, { + recursive: true, + filter: src => !/[\\/]media([\\/]|$)/.test(src), + }); + if (bundle) reusableRepairEvidence = { bundle, results: gradingTo }; + console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} grading at ${join(args.out, acceptedGradingDirectory)}`); + } + } catch (e) { + // Never worth losing a run over: the score is already recorded. + console.log(` harness failure: could not keep the first build: ${errorMessage(e).split('\n')[0]}`); + } + + // A resumed grade settles a repair that was already charged; only a + // resumed coding session is a new repair. + let repairs = resumedRepair && !resumedGrade ? 1 : 0; + let repairCost = resumedRepair && build ? build.costUsd : 0; + const repairSessions = resumedRepair && build + ? [runSessionRecord(build, priorRepairs + 1)] : []; + const repairHistory: ReturnType[] = []; + let repairCandidate: RunRepairCandidate | null = null; + let priorRegressionReport = resumedRegressionReport; + let priorRegressionOwner = repairOwnerNodeIds(progressionSelection).join('\n') || null; + let regressed = false; + let repairStopReason: string | null = null; + let repairProgress = repairProgressState(null, bundle); + const pauseForRepeatedFindings = () => { + if (args.progression) return false; + repairProgress = repairProgressState(repairProgress, bundle); + if (args.maxStalledRepairs === 0 + || repairProgress.stalledRounds < args.maxStalledRepairs) return false; + repairStopReason = 'repeated-findings'; + console.log(` pausing after ${repairProgress.stalledRounds} repairs ` + + 'with the same failed checks and no score gain'); + return true; + }; + const initialBundleOutcome = classifyBundle(bundle); + const initialProgressionAttempt = args.progression + ? requireProgressionState(progressionExecution?.state ?? null).attempts.at(-1) ?? null + : null; + const initialGradeUsable = levelGradeIsUsable(initialBundleOutcome, + initialProgressionAttempt); + if (!initialGradeUsable) { + repairStopReason = 'initial-grading-failed'; + console.log(' repairs skipped: the initial grade did not complete, so there are no reliable findings to fix'); + } + + let progressionNext = recordProgressionGrade({ + selected: progressionSelection, + bundle: initialProgressionFailure ? null : bundle, + level, + failure: initialProgressionFailure, + completedRepair: resumedRepair && !resumedGrade, + }); + // Never start a paid repair with nothing left to spend or while a charged + // repair still waits for its grade. + const progressionMayRepair = () => !args.progression + || (progressionNext?.type === 'repair' && progressionNext.repair.remaining > 0 + && progressionNext.repair.awaitingGrade !== true); + // One allowance for both modes: the engine's remaining repairs in + // dependency mode, the run-wide total less every repair so far otherwise. + const repairsRemaining = (): number => args.progression + ? (progressionNext?.type === 'repair' ? progressionNext.repair.remaining : 0) + : args.repairs - priorRepairs - repairs; + const mayRepair = (): boolean => args.progression + ? progressionMayRepair() : repairsRemaining() > 0; + const recordRepairProgression = ({ failure = null, repairRegression = null, + completedRepair = false }: { + failure?: ProgressionFailure | null; + repairRegression?: ProgressionRepairRegression | null; + completedRepair?: boolean; + } = {}) => { + progressionNext = recordProgressionGrade({ + selected: progressionSelection, + bundle: failure ? null : bundle, + level, + failure, + repairRegression, + completedRepair, + }); + return progressionMayRepair(); + }; + const recordRepairHarnessFailure = (phase: string, reason: string, + failedBundle: GradeBundlePayload | null = null, completedRepair = false): void => { + const failure: RunOutcome = { + kind: 'harness_failure', phase, reason, + appFailures: [], inconclusive: [], harnessFailures: [reason], + }; + bundle = failedBundle ? { ...failedBundle, outcome: failure } : { outcome: failure }; + repairStopReason = phase; + if (args.progression) recordRepairProgression({ + failure: progressionFailure(failure), completedRepair, + }); + const failedLevel = progressionExecution?.state?.attempts.at(-1)?.level; + const prior = run.levels.find(record => record.level === failedLevel); + if (prior) prior.outcome = failure; + }; + const restoreProgressionGrade = (accepted: GradeBundlePayload | null, + label: string): boolean => { + const expected = progressionSelection && isProgressionWorkRecipeAction(progressionSelection) + ? progressionSelection.grader.selectionSha256 : null; + if (!expected || accepted?.selection?.sha256 === expected) { + bundle = accepted; + return true; + } + bundle = grade(args, appDir, url, label, level, track, runId); + const outcome = classifyBundle(bundle); + if (levelGradeIsUsable(outcome)) return true; + recordRepairHarnessFailure('repair-restore-grading', + outcome.reason ?? 'restored source did not produce a reliable grade', bundle, true); + return false; + }; + const writeRepairReport = (results: string | null = null): + { status: 0 | 3 | 4 } | { status: 'failed'; reason: string } => { + try { + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + ...(results ? ['--results', results] : []), + '--history-json', JSON.stringify(repairHistory), + '--archive', join(outputDir, 'repair-reports', + `bug-report-l${level}${featureActionSuffix}-round${repairs + 1}.md`), + ...(priorRegressionReport ? ['--prior-regression', priorRegressionReport] : []), + ...repairReportArgs(progressionSelection)], { stdio: 'pipe' }); + return { status: 0 }; + } catch (error) { + const failure = commandFailure(error); + if (failure.status === 3 || failure.status === 4) return { status: failure.status }; + return { status: 'failed', reason: errorMessage(failure).split(/\r?\n/)[0] + ?? 'repair report generation failed' }; + } + }; + const recordMissingRepairFeedback = (status: 3 | 4): void => { + repairStopReason = 'no-actionable-findings'; + if (!args.progression) return; + const reason = status === 3 + ? 'selected repair checks contain no failures' + : 'selected repair checks produced no actionable findings'; + const failure: RunOutcome = { + kind: 'harness_failure', phase: 'repair-report', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason], + }; + bundle = { outcome: failure }; + recordRepairProgression({ failure: progressionFailure(failure) }); + }; + + // Hand back findings and let the agent fix, until clean or out of rounds. + while (levelGradeIsUsable(classifyBundle(bundle), args.progression + ? requireProgressionState(progressionExecution?.state ?? null).attempts.at(-1) ?? null + : null) && mayRepair()) { + let reportReady = false; + const acceptedBundle = bundle; + let repairBaselineBundle = bundle; + if (args.progression) { + progressionSelection = bindProgressionAction(level); + const repairOwner = repairOwnerNodeIds(progressionSelection).join('\n') || null; + if (repairOwner !== priorRegressionOwner) { + priorRegressionReport = null; + priorRegressionOwner = repairOwner; + } + trackProgressionBudget(progressionSelection, priorRepairs + repairs); + if (progressionSelection && isProgressionWorkRecipeAction(progressionSelection) + && bundle?.selection?.sha256 !== progressionSelection.grader.selectionSha256) { + const sequence = requireProgressionState(progressionExecution?.state ?? null).attempts.length + 1; + const targetChecks = repairCheckKeys(progressionSelection); + const sourceSha256 = hashAppSource(appDir).sha256; + const hasCurrentTargetEvidence = bundle?.source?.sha256 === sourceSha256 + && targetChecks.every(check => bundle?.selection?.reportedChecks?.includes(check)); + const reusable = reusableRepairEvidence?.bundle.source?.sha256 === sourceSha256 + && targetChecks.every(check => + reusableRepairEvidence?.bundle.selection?.reportedChecks?.includes(check)) + ? reusableRepairEvidence : null; + let repairResults = hasCurrentTargetEvidence ? null : reusable?.results ?? null; + if (reusable && !hasCurrentTargetEvidence) repairBaselineBundle = reusable.bundle; + if (!hasCurrentTargetEvidence && !reusable) { + const binding = args.recipeBindings.get(level); + if (!binding) throw new Error(`L${level} has no recipe binding`); + const targetTask = resolveProgressionRepairTarget(binding, + requireProgressionState(progressionExecution?.state ?? null)); + repairResults = join(outputDir, 'repair-grades', + `l${level}${featureActionSuffix}-round-${repairs + 1}`); + repairBaselineBundle = grade(args, appDir, url, + `${args.backend}-l${level}-repair-target${sequence}`, + level, track, runId, { out: repairResults, recipeTask: targetTask }); + } + const refreshOutcome = classifyBundle(repairBaselineBundle); + const refreshUsable = levelGradeIsUsable(refreshOutcome); + if (!refreshUsable) { + recordRepairHarnessFailure('refresh-grading-failed', + refreshOutcome.reason ?? 'repair target did not produce a reliable grade', + repairBaselineBundle); + break; + } + const refreshReport = writeRepairReport(repairResults); + if (refreshReport.status === 'failed') { + recordRepairHarnessFailure('repair-report', refreshReport.reason); + break; + } + if (refreshReport.status === 3) { + bundle = grade(args, appDir, url, + `${args.backend}-l${level}-repair-refresh${sequence}`, + level, track, runId); + if (!levelGradeIsUsable(classifyBundle(bundle))) { + recordRepairHarnessFailure('refresh-grading-failed', + classifyBundle(bundle).reason ?? 'repair refresh did not produce a reliable grade', + bundle); + break; + } + checkpointGrade('final', bundle!, + [...(build && !resumedRepair ? [runSessionRecord(build)] : []), ...repairSessions], true); + recordRepairProgression(); + continue; + } + if (refreshReport.status === 4) { + recordMissingRepairFeedback(refreshReport.status); + break; + } + reportReady = true; + } + } + const report = reportReady ? { status: 0 as const } : writeRepairReport(); + if (report.status === 'failed') { + recordRepairHarnessFailure('repair-report', report.reason); + break; + } + if (report.status !== 0) { + recordMissingRepairFeedback(report.status); + break; + } + + const before = repairBaselineBundle?.totals?.score ?? 0; + const beforeMax = repairBaselineBundle?.totals?.max ?? 0; + const beforeBundle = repairBaselineBundle; + // Keep the accepted source outside paths visible to the coding session. + const snapshot = join(tmpdir(), `stack-bench-snapshot-${args.backend}-${args.track}-run${args.runIndex}-l${level}`); + const gradingSnapshot = `${snapshot}-grading`; + const acceptedSource = hashAppSource(appDir); + snapshotSource(appDir, snapshot); + rmSync(gradingSnapshot, { recursive: true, force: true }); + if (existsSync(privateGradingDirectory(appDir))) { + cpSync(privateGradingDirectory(appDir), gradingSnapshot, { recursive: true }); + } + const cleanupRepairSnapshots = () => { + rmSync(snapshot, { recursive: true, force: true }); + rmSync(gradingSnapshot, { recursive: true, force: true }); + }; + try { + const displayedRepairBudget = args.progression + ? progressionRepairLimit + : args.repairs; + if (args.dependencyPolicy?.definition.repair.selection === 'feature' + && progressionSelection && isProgressionWorkRecipeAction(progressionSelection)) { + const [nodeId] = progressionSelection.action.repair.nodeIds; + const node = requireProgressionState(progressionExecution?.state ?? null).definition.nodes + .find(candidate => candidate.id === nodeId); + if (!nodeId || !node) { + throw new Error('feature repair has no selected feature'); + } + const used = requireProgressionState(progressionExecution?.state ?? null) + .nodes[nodeId]?.repairs.used ?? 0; + console.log(`--- feature repair ${used + 1}: ${node.title} ---`); + } else { + console.log(`--- repair ${priorRepairs + repairs + 1}/${displayedRepairBudget} ---`); + } + const fix = await runAgentForLevel('fix', level, + featureActionSequence === null ? undefined : restoreFeatureAcceptedSource); + repairCost += fix.costUsd; + repairSessions.push(runSessionRecord(fix, priorRepairs + repairs + 1)); + + const fixFailure = agentSessionFailure(fix); + if (fixFailure) { + if (featureActionSequence !== null) { + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot, false)) break; + } + console.log(` coding session failed: ${fixFailure.reason}; stopping repairs`); + bundle = { outcome: fixFailure }; + repairHistory.push(repairHistoryEntry(repairs + 1, beforeBundle, bundle, + 'agent session failed')); + repairStopReason = 'agent-session-failure'; + recordRepairProgression({ failure: progressionFailure(fixFailure) }); + break; + } + repairs += 1; + + // Reject contaminated repairs before spending time on grading. + const fixLeak = auditContamination(appDir, auditNetwork(), auditsTranscripts); + if (fixLeak) { + const buildSession = build ? runSessionRecord(build) : null; + const sessions = resumedRepair || !buildSession + ? repairSessions : [buildSession, ...repairSessions]; + const sessionTotals = summarizeSessions(sessions); + cleanupRepairSnapshots(); + abortUnusableSession(`repair ${repairs}`, fixLeak, { + level, graded: false, score: null, max: null, + selection: bundle?.selection ?? null, + ...(resumedRepair + ? { resumedRepair: firstBuild } + : continuing + ? { baseline: firstBuild, resumeCostUsd: requireBuild().costUsd, + resumeSession: runSessionRecord(requireBuild()) } + : { firstBuild, buildCostUsd: requireBuild().costUsd, + buildSessions: [runSessionRecord(requireBuild())] }), + repairCostUsd: addCostUsd(repairCost), repairSessions, repairs, + ...(resumedRepair ? { priorRepairs, + cumulativeRepairs: priorRepairs + repairs } : {}), + repair: { status: 'ungraded', limit: displayedRepairBudget, + used: priorRepairs + repairs, + stopReason: fixLeak.kind === 'harness_failure' ? 'audit-failure' : 'contaminated' }, + sessionTotals, + costUsd: resumedRepair ? addCostUsd(repairCost) + : addCostUsd(requireBuild().costUsd, repairCost), + durationMs: Date.now() - t0, + }, progressionSelection, true); + } + if (hashAppSource(appDir).sha256 === acceptedSource.sha256) { + // A source hash does not cover installed dependencies or a live process. + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + const reason = 'repair made no source change'; + console.log(` ${reason}; ${args.progression + ? 'counting the failed attempt' + : 'pausing before another paid round'}`); + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, beforeBundle, reason)); + if (args.progression) { + if (!restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}-unchanged${repairs}`)) break; + if (!recordRepairProgression({ completedRepair: true })) break; + continue; + } + repairStopReason = 'no-source-change'; + break; + } + const repairedSource = `${snapshot}-accepted`; + snapshotSource(appDir, repairedSource); + try { + bundle = await gradeAcceptedSource(repairedSource, `${args.backend}-l${level}-fix${repairs}`); + } finally { + rmSync(repairedSource, { recursive: true, force: true }); + } + + const repairedOutcome = classifyBundle(bundle); + if (!levelGradeIsUsable(repairedOutcome)) { + const reason = repairedOutcome.reason + ?? 'the repaired source did not produce a reliable grade'; + // The session is paid for and charged. Keep what it produced so a + // resume grades it instead of buying another repair; the feature's + // status waits for that grade. + const candidateDirectory = `repair-candidate-l${level}${featureActionSuffix}`; + let preserveFailure: string | null = null; + try { + repairCandidate = preserveRepairCandidate(candidateDirectory); + console.log(` repair grade failed: ${reason}; kept the repaired source at ` + + `${join(outputDir, candidateDirectory)} for grading on resume`); + } catch (error) { + preserveFailure = errorMessage(error).split(/\r?\n/)[0] + ?? 'could not keep the repaired source'; + console.log(` repair grade failed: ${reason}; ${preserveFailure}; restoring the accepted source`); + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + } + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + 'repair completed; grading failed')); + recordRepairHarnessFailure('repair-grading', + preserveFailure ? `${reason}; ${preserveFailure}` : reason, bundle, true); + break; + } + + if (featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection) + && !featureCandidateAccepted(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null), bundle)) { + const rejectedBundle = bundle; + if (rejectedBundle) checkpointGrade('repair', rejectedBundle, + [...(build && !resumedRepair ? [runSessionRecord(build)] : []), ...repairSessions], false); + archiveCandidateGrade(appDir, outputDir, `l${level}${featureActionSuffix}-repair${repairs}`); + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + if (!restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}${featureActionSuffix}-rollback${repairs}`)) break; + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, rejectedBundle, + 'rolled back because the feature still failed or earlier behavior regressed')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + + const after = bundle?.totals?.score ?? 0; + const afterMax = bundle?.totals?.max ?? 0; + const repairedBundle = bundle; + // Lost or inconclusive evidence cannot hide a repair regression. + let decision = repairEvidenceDecision(beforeBundle, bundle); + const regressionDecision = repairRegressionDecision(acceptedBundle, bundle); + if (regressionDecision.action === 'rollback-regression') decision = regressionDecision; + if (bundle) checkpointGrade('repair', bundle, + [...(build && !resumedRepair ? [runSessionRecord(build)] : []), ...repairSessions], + !decision.action.startsWith('rollback-')); + const shared = decision.shared; + if (decision.action === 'keep-setup-repair') { + console.log(afterMax > 0 + ? ` application setup is now gradeable (${after}/${afterMax}); keeping this repair` + : ' application setup is still failing; keeping the attempted repair for the next round'); + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + afterMax > 0 + ? 'kept because the app became gradeable' + : 'kept to continue repairing application setup')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + if (decision.action === 'rollback-no-comparison') { + console.log(' no criteria were conclusively scored in both rounds; rolling back this fix'); + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + if (!restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}-rollback${repairs}`)) break; + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, repairedBundle, + 'rolled back because the result could not be compared')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + if (shared.points < Math.min(beforeMax, afterMax)) { + console.log(` comparing ${shared.points} point(s) across ${shared.count} criteria scored in both rounds` + + ` (${before}/${beforeMax} -> ${after}/${afterMax} overall)`); + } + if (decision.action === 'rollback-regression') { + if (shared.regressions.length) { + console.log(` broke ${shared.regressions.length} earlier passing check(s); rolling back this fix`); + } else if (shared.lostEvidence.length) { + console.log(` lost conclusive evidence for ${shared.lostEvidence.length} criterion/criteria; rolling back this fix`); + } else if (shared.definitionChanges.length) { + console.log(' rubric points changed between grades; rolling back this fix'); + } else { + console.log(` regressed (${shared.before} -> ${shared.after} on shared criteria); rolling back this fix`); + } + let repairRegression: ProgressionRepairRegression | null = null; + let regressionReportFailure: string | null = null; + try { + if (shared.regressions.length) { + const path = join(outputDir, 'repair-reports', + `rejected-regression-l${level}${featureActionSuffix}-round${repairs}.md`); + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + '--out', path, '--checks-json', JSON.stringify(shared.regressions), + '--regression-context'], { stdio: 'pipe' }); + repairRegression = { + ownerNodeIds: repairOwnerNodeIds(progressionSelection), + report: readFileSync(path, 'utf8'), + }; + priorRegressionReport = path; + } + } catch (error) { + regressionReportFailure = errorMessage(error).split(/\r?\n/)[0] + ?? 'regression report generation failed'; + } + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + if (regressionReportFailure) { + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + 'repair completed; regression reporting failed')); + recordRepairHarnessFailure('repair-regression-report', regressionReportFailure, + null, true); + break; + } + if (!restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}-rollback${repairs}`)) break; + regressed = true; + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, repairedBundle, + 'rolled back because earlier behavior regressed')); + if (!recordRepairProgression({ repairRegression, completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + priorRegressionReport = null; + if (shared.after === shared.before) { + const remaining = displayedRepairBudget - priorRepairs - repairs; + console.log(` ${formatRepairProgress(shared, { before, beforeMax, after, afterMax })}; ` + + (remaining > 0 ? `${remaining} repair(s) remain` : 'repair budget exhausted')); + } + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + shared.after === shared.before ? 'kept with no score gain' : 'kept')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + } finally { + cleanupRepairSnapshots(); + } + } + + // Missing grade evidence is not a zero score. + const progressionState = progressionExecution + ? requireProgressionState(progressionExecution.state) : null; + const progressionAttempt = progressionState + ? progressionState.attempts.findLast(attempt => attempt.level === level) ?? null + : null; + const levelBundle = progressionAttempt?.outcome === 'inconclusive' + ? bundle : progressionBundles.get(level) ?? bundle; + const finalBundleOutcome = classifyBundle(levelBundle); + // Progression uses stricter evidence rules than a regular scored bundle. + // Store one answer when a selected check is not measured: the raw bundle + // remains available for diagnosis, but the level is not a usable grade. + const graded = levelGradeIsUsable(finalBundleOutcome, + args.progression ? progressionAttempt : null); + const finalTotals = graded ? levelBundle?.totals ?? null : null; + const nodeRepairs = progressionState + ? dependencyRepairRecords(progressionState, level, levelRepairNodeIds) + : null; + const repairLimit = progressionExecution + ? Math.max(priorRepairs + repairs, progressionRepairLimit) + : args.repairs; + const progressionStopReason = progressionNext?.type !== 'repair' + ? dependencyRepairStopReason(nodeRepairs ?? []) : null; + const repairBudgetExhausted = progressionExecution + ? finalBundleOutcome.kind === 'app_failure' && progressionStopReason !== null + && progressionStopReason !== 'repeated-findings' + : priorRepairs + repairs >= args.repairs; + repairStopReason ??= progressionStopReason; + const repairStatus: RepairStatus = repairStopReason === 'no-source-change' ? 'incomplete' + : !graded ? 'ungraded' + : finalBundleOutcome.kind === 'passed' ? (repairs > 0 ? 'corrected' : 'not-needed') + : repairBudgetExhausted ? 'budget-exhausted' : 'incomplete'; + const stopReasons: Record = { + 'not-needed': 'not-needed', + corrected: 'passed', + 'budget-exhausted': 'budget-exhausted', + incomplete: null, + ungraded: null, + }; + const stopReason = repairStopReason ?? stopReasons[repairStatus]; + const repair = { + status: repairStatus, + limit: repairLimit, + used: priorRepairs + repairs, + ...(!args.progression ? { stallLimitRounds: args.maxStalledRepairs } : {}), + stopReason, + ...(nodeRepairs ? { nodeRepairs } : {}), + ...(repairCandidate ? { candidate: repairCandidate } : {}), + }; + const latestProgressionAttempt = progressionState?.attempts.at(-1) ?? null; + const featureDepthContinues = featureActionSequence !== null + && progressionState?.phase === 'active' && progressionState.level === level; + if (continuing) { + const continuation = requireContinuation(run); + continuation.cumulativeRepairsAfter = continuation.cumulativeRepairsBefore + repairs; + } + let checkpoint = null; + if (graded && !featureDepthContinues + && (!latestProgressionAttempt || latestProgressionAttempt.level === level)) { + try { + checkpoint = preserveLevelCheckpoint({ + appDir, + outputDir: args.out, + runId, + identities: run.identities, + track: args.track, + backend: args.backend, + level, + repair, + outcome: finalBundleOutcome, + selectionSha256: levelBundle?.selection?.sha256 ?? null, + }); + console.log(` kept the L${level} source checkpoint at ${join(args.out, checkpoint.directory)}`); + } catch (error) { + console.log(` harness failure: could not keep the L${level} source checkpoint: ${errorMessage(error).split('\n')[0]}`); + } + } + if (!graded) { + console.log(` L${level}: GRADING DID NOT COMPLETE — no usable bundle. ` + + `Score is unknown, not zero; re-grade this level before using the run.`); + } + const buildSession = build ? runSessionRecord(build) : null; + const requireBuildSession = (): RunSessionRecord => { + if (!buildSession) throw new Error(`level ${level} has no coding session`); + return buildSession; + }; + const sessionTotals = summarizeSessions(resumedRepair || !buildSession ? repairSessions + : [buildSession, ...repairSessions]); + appendLevelRecord({ + level, + graded, + score: finalTotals?.score ?? null, + max: finalTotals?.max ?? null, + // Preserve earlier-level guarantees in the durable result. + regression: levelBundle?.totals?.regression ?? null, + selection: levelBundle?.selection ?? null, + ...(resumedRepair + ? { resumedRepair: firstBuild } + : continuing + ? { baseline: firstBuild, resumeCostUsd: requireBuild().costUsd, + resumeSession: requireBuildSession() } + : featureActionSequence !== null + ? { buildCostUsd: requireBuild().costUsd, buildSessions: [requireBuildSession()] } + : { firstBuild, buildCostUsd: requireBuild().costUsd, + buildSessions: [requireBuildSession()] }), + contractPass: levelBundle?.totals?.contractPass ?? null, + code: levelBundle?.code ?? null, + repairCostUsd: addCostUsd(repairCost), + repairSessions, + repairHistory, + sessionTotals, + tokens: sessionTotals.tokens, + usage: sessionTotals.usage, + turns: sessionTotals.turns, + promptBytes: sessionTotals.promptBytes, + tokensPerTurn: sessionTotals.turns + ? Math.round(sessionTotals.tokens / sessionTotals.turns) : null, + // Record actual reasoning because the provider default is not pinned. + thinking: sessionTotals.thinking, + repairs, + ...(resumedRepair ? { priorRepairs, + cumulativeRepairs: priorRepairs + repairs } : {}), + repair, + checkpoint, + // Keep the summary flag derived from the typed status so the two cannot drift. + stalled: repairStatus === 'budget-exhausted' + || ['repeated-findings', 'no-source-change'].includes(repairStopReason ?? ''), + regressed, + outcome: finalBundleOutcome, + durationSec: Math.round((Date.now() - t0) / 1000), + }); + if (!args.progression || requireProgressionState(progressionExecution?.state ?? null).attempts + .some(attempt => attempt.level === level && attempt.outcome === 'conclusive')) { + if (!featureDepthContinues && !run.validation.ladder.completedLevels.includes(level)) { + run.validation.ladder.completedLevels.push(level); + } + } + if (progressionState) synchronizeProgressionSummary(run, progressionState); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + run.outcome = aggregateRunOutcome(run.levels, progressionExecution?.state?.terminalOutcome); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + if (progressionState && runCostComplete) { + commitTimeContinuationBoundary(outputDir, progressionState, level); + } + const blockedLevels = args.levelList.filter(candidate => candidate > level); + if (args.progression) { + const progressionState = requireProgressionState(progressionExecution?.state ?? null); + if (progressionState.phase === 'terminal') { + if (blockedLevels.length) run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = blockedLevels; + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + if (progressionState.level <= level) { + if (progressionState.attempts.at(-1)?.outcome === 'inconclusive') { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = [level, ...blockedLevels]; + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + if (featureActionSequence !== null && progressionState.level === level) { + levelIndex -= 1; + continue; + } + throw new Error(`dependency progression did not leave L${level} after its repair budget`); + } + continue; + } + if (blockedLevels.length && !ladderMayAdvance(finalBundleOutcome)) { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = blockedLevels; + persistRun(join(args.out, ARTIFACT_FILE.run), run); + console.log(` ladder paused after L${level}: L${level} must pass before ` + + `${blockedLevels.map(candidate => `L${candidate}`).join(', ')} can start`); + console.log(' inspect the failures, then explicitly grant more repairs or correct the benchmark'); + break; + } + } + + if (args.mutations) { + console.log(`\n================ ${args.backend} mutation control ================`); + const pristineOutcome = aggregateRunOutcome(run.levels, progressionExecution?.state?.terminalOutcome); + if (args.referenceMutationOnly || mutationControlEligible(pristineOutcome)) { + args.parentAttemptId = runId; + const baselineBundle = pristineMutationBaselinePath(args); + if (baselineBundle) args.mutationBaselineBundle = baselineBundle; + else delete args.mutationBaselineBundle; + run.mutationControl = runMutationControl(args, appDir, url, track, + run.setup?.isolation?.imageId ?? null); + } else { + console.log(` skipped: pristine outcome is ${pristineOutcome.kind}`); + run.mutationControl = { ok: false, skipped: true, + outcome: { kind: pristineOutcome.kind, phase: 'mutation-control-prerequisite', + reason: `pristine outcome is ${pristineOutcome.kind}` } }; + } + persistRun(join(args.out, ARTIFACT_FILE.run), run); + } + + // Record a final transcript audit in addition to the per-session hard gates. + // The same retry and diagnostic path is used at both gates. + let finalAuditFailure = null; + const finalAudit = auditContamination(appDir, auditNetwork(), auditsTranscripts); + if (!finalAudit) { + run.contaminated = false; + run.contamination = { evidence: 'no agent access to private benchmark files detected', + verdict: 'private-access audit passed' }; + } else if (finalAudit.kind === 'contaminated') { + run.contaminated = true; + run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict }; + console.log('\n CONTAMINATED: restricted file or network access attempts were detected:'); + for (const evidence of finalAudit.evidence) console.log(` ${evidence}`); + console.log(' Scores from this run must not be quoted.'); + } else { + run.contaminated = false; + run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict }; + const reason = finalAudit.evidence.join('; '); + finalAuditFailure = { kind: 'harness_failure', phase: 'contamination-audit', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + console.log('\n HARNESS FAILURE: the contamination audit did not complete. Scores from this run must not be quoted.'); + } + + // Keep the transcript evidence outside the provider CLI's prunable store. + try { archiveTranscripts(appDir, artifactLabel); } + catch { console.log(' (transcript archiving failed — evidence is on a 30-day timer)'); } + + if (args.progression && progressionExecution?.state) { + synchronizeProgressionSummary(run, requireProgressionState(progressionExecution.state)); + } + run.outcome = finalAuditFailure ?? (args.referenceMutationOnly && run.mutationControl?.ok + ? { kind: 'passed', phase: 'mutation-control', reason: null, + appFailures: [], inconclusive: [], harnessFailures: [] } + : aggregateRunOutcome(run.levels, progressionExecution?.state?.terminalOutcome)); + if (args.mutations && !run.mutationControl?.ok && !run.mutationControl?.skipped) { + run.outcome = { kind: run.mutationControl?.outcome?.kind === 'incomplete' + ? 'incomplete' : 'harness_failure', phase: 'mutation-control', + reason: run.mutationControl?.outcome?.reason + ?? run.mutationControl?.processError + ?? 'one or more declared mutations were not cleanly caught', + appFailures: [], inconclusive: [] }; + } + + if (finalPackageEvidenceRequired(run.outcome, run.levels)) { + try { + preserveFinalPackageEvidence({ appDir, outputDir }); + console.log(` source kept at ${join(outputDir, 'source')}`); + console.log(` grading detail kept at ${join(outputDir, 'grading')}`); + } catch (error) { + const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'evidence preservation failed'; + run.outcome = { kind: 'harness_failure', phase: 'evidence-preservation', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + console.log(` stopped: ${reason}`); + } + } + + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + if (args.repairGrant) { + const continuation = requireContinuation(run); + const totals = requireRunTotals(run); + continuation.cumulativeCostAfterUsd = addCostUsd(continuation.cumulativeCostBeforeUsd, totals.costUsd); + continuation.cumulativeDurationAfterSec = continuation.cumulativeDurationBeforeSec + totals.durationSec; + } + run.completedAt = new Date().toISOString(); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + + console.log(`\n================ ${args.backend} summary ================`); + for (const l of run.levels) { + console.log(` ${formatLevelSummary(l)}`); + } + const totals = requireRunTotals(run); + console.log(` TOTAL ${totals.score}/${totals.max} ` + + `$${totals.costUsd} ${totals.repairs} repair(s) ${totals.durationSec}s`); + console.log(` ${join(outputDir, ARTIFACT_FILE.run)}`); + + teardown(); + + // Remove only the temporary directory created by this run. + if (ownWorkDir) { + try { + rmSync(dirname(appDir), { recursive: true, force: true }); + } catch (error) { + console.log(` could not remove work directory ${dirname(appDir)}: ${errorMessage(error)}`); + } + } + process.exitCode = runExitCode(run.outcome); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch(error => { + console.error(redactCredentials(error instanceof Error ? error.stack ?? error.message : errorMessage(error))); + try { emergencyTeardown?.(); } + catch (cleanupError) { + console.error(`cleanup after failure also failed: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`); + } + process.exitCode = 1; + }); +} diff --git a/tools/stack-bench/commands/campaign-cli.ts b/tools/stack-bench/commands/campaign-cli.ts new file mode 100644 index 00000000000..9c208a8f084 --- /dev/null +++ b/tools/stack-bench/commands/campaign-cli.ts @@ -0,0 +1,385 @@ +#!/usr/bin/env node + +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { CAMPAIGN_MODE_REGISTRY } from '../src/campaigns/campaign-mode.js'; +import { executeCampaign, inspectCampaign, reconcileCampaign } + from '../src/campaigns/campaign-runner.js'; +import { inspectCampaignSummary } from '../src/campaigns/campaign-inspection.js'; +import { exportCampaignReport, generateCampaignReport } from '../src/campaigns/campaign-report.js'; +import { grantCampaignDependencyRepairs } + from '../src/campaigns/campaign-progression-grant.js'; +import { requestCampaignTimeGrant } from '../src/campaigns/campaign-time-grant.js'; +import { readCampaignProviderContinuationStatus, requestCampaignProviderContinuation } + from '../src/campaigns/campaign-provider-continuation.js'; +import { auditProgressionReferenceCampaign, formatProgressionReferenceCampaignAudit } + from '../src/campaigns/progression-reference-campaign-audit.js'; +import type { ReferenceCampaignAudit } + from '../src/campaigns/progression-reference-campaign-audit.js'; +import { prepareCampaignExtension } from '../src/campaigns/campaign-extension.js'; +import { campaignDepthPauseStatus, continueCampaignDepth } from '../src/campaigns/campaign-depth-pause.js'; +import { statusWord } from '../src/evidence/status-words.js'; +import { readCampaignLock, requestCampaignCancellation } from '../src/campaigns/campaign-lock.js'; + +interface CampaignSummaryPlan { + id: string; + version: string; + contentSha256: string; +} + +interface CampaignSummaryState { + status: string; + summary: unknown; + attempts: Array<{ + plan: { id: string }; + status: string; + executions: Array<{ + id: string; + outcome: unknown; + reason: string | null; + }>; + }>; +} + +interface ReferenceCampaignPlan { + attempts: Array<{ + mode?: { id?: string }; + agentAdapter?: string; + }>; +} + +interface ReferenceCampaignState { + status: string; +} + +interface ResumeCampaign { + plan: { + contentSha256: string; + definition: { mode?: { id?: string } }; + }; + state: { + status: string; + attempts: Array<{ executions: readonly unknown[] }>; + }; +} + +type ReferenceCampaignAuditFunction = (directory: string) => ReferenceCampaignAudit | null; + +export type CampaignArgs = + | { command: 'pause-status'; directory: string } + | { command: 'continue-depth'; directory: string } + | { command: 'continue-provider'; directory: string; attemptId: string; requestId: string } + | { command: 'continuation-status'; directory: string; attemptId: string; json: boolean } + | { command: 'grant-time'; directory: string; attemptId: string; grantId: string; minutes: number } + | { command: 'modes' } + | { command: 'validate'; path: string } + | { command: 'show'; path: string } + | { command: 'status'; directory: string; full: boolean } + | { command: 'inspect'; directory: string } + | { command: 'report'; directory: string } + | { command: 'export'; directory: string; output: string } + | { command: 'stop'; directory: string } + | { command: 'audit'; directory: string } + | { command: 'grant-repairs'; directory: string; attemptId: string; grantId: string; + level: number; nodeIds: string[]; repairs: number } + | { command: 'extend'; path: string; parentDirectory: string; fromDepth: number; + directory: string; prepareOnly: boolean } + | { command: 'trial'; path: string; directory: string } + | { command: 'run'; path: string; directory: string } + | { command: 'resume'; path: string; directory: string } + | { command: 'reconcile'; path: string; directory: string }; + +function isOneOf(value: string | undefined, + values: readonly T[]): value is T { + return value !== undefined && values.some(candidate => candidate === value); +} + +export function campaignStateSummary(plan: CampaignSummaryPlan, state: CampaignSummaryState) { + const failures = state.attempts.flatMap(attempt => { + const execution = attempt.executions.at(-1); + if (!execution || execution.outcome === null || execution.outcome === 'passed') return []; + return [{ + attempt: attempt.plan.id, + status: statusWord(attempt.status), + execution: execution.id, + outcome: statusWord(String(execution.outcome)), + reason: execution.reason, + }]; + }); + return { + campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 }, + status: statusWord(state.status), + summary: state.summary, + failures, + }; +} + +export function auditCompletedReferenceCampaign(directory: string, plan: ReferenceCampaignPlan, + state: ReferenceCampaignState, { + audit = auditProgressionReferenceCampaign, +}: { audit?: ReferenceCampaignAuditFunction } = {}): ReferenceCampaignAudit | null { + const hasReferenceProgression = plan.attempts.some(attempt => + attempt.mode?.id === 'dependency' && attempt.agentAdapter === 'reference-fixture'); + return state.status === 'completed' && hasReferenceProgression ? audit(directory) : null; +} + +export function validateResumeCampaignState( + requested: { contentSha256: string }, existing: T): T { + if (requested.contentSha256 !== existing.plan.contentSha256) { + throw new Error('resume requires the exact campaign plan already stored in the output directory'); + } + if (existing.plan.definition.mode?.id !== 'dependency') { + throw new Error('resume is available only for dependency campaigns'); + } + const executions = existing.state.attempts.reduce((total, attempt) => + total + attempt.executions.length, 0); + if (existing.state.status !== 'prepared' || executions < 1) { + throw new Error('resume requires a dependency campaign with scheduled work'); + } + return existing; +} + +export function validateResumeCampaign(path: string, directory: string): ResumeCampaign { + return validateResumeCampaignState(compileCampaignFile(path), inspectCampaign(directory)); +} + +export function parseCampaignArgs(argv: string[]): CampaignArgs { + const [command, path, ...rest] = argv.slice(2); + if ((command === 'pause-status' || command === 'continue-depth') && path && rest.length === 0) { + return { command, directory: resolve(path) }; + } + if ((command === 'continue-provider' || command === 'continuation-status') && path) { + const options = new Map(); + let json = false; + for (let i = 0; i < rest.length; i++) { + const flag = rest[i]!; + if (flag === '--json' && command === 'continuation-status' && !json) { json = true; continue; } + if (!['--attempt', ...(command === 'continue-provider' ? ['--request-id'] : [])].includes(flag) + || options.has(flag) || !rest[i + 1] || rest[i + 1]!.startsWith('--')) { + throw new Error('invalid provider continuation options'); + } + options.set(flag, rest[++i]!); + } + const attemptId = options.get('--attempt'); + if (!attemptId) throw new Error('provider continuation requires --attempt'); + if (command === 'continuation-status') return { command, directory: resolve(path), attemptId, json }; + const requestId = options.get('--request-id'); + if (!requestId) throw new Error('continue-provider requires --request-id'); + return { command, directory: resolve(path), attemptId, requestId }; + } + if (command === 'modes' && path === undefined) return { command }; + if (isOneOf(command, ['validate', 'show']) && path && rest.length === 0) { + return { command, path: resolve(path) }; + } + if (command === 'status' && path + && (rest.length === 0 || (rest.length === 1 && rest[0] === '--full'))) { + return { command, directory: resolve(path), full: rest.length === 1 }; + } + if (isOneOf(command, ['inspect', 'report', 'audit', 'stop']) && path && rest.length === 0) { + return { command, directory: resolve(path) }; + } + if (command === 'export' && path && rest.length === 2 && rest[0] === '--out' && rest[1]) { + return { command, directory: resolve(path), output: resolve(rest[1]) }; + } + if (command === 'grant-time' && path) { + const options = new Map(); + for (let i = 0; i < rest.length; i += 2) { + const flag = rest[i]; const value = rest[i + 1]; + if (!flag || !['--attempt', '--grant-id', '--minutes'].includes(flag) + || !value || options.has(flag)) throw new Error('invalid grant-time options'); + options.set(flag, value); + } + const minutes = Number(options.get('--minutes')); + if (!options.get('--attempt') || !options.get('--grant-id') + || !Number.isSafeInteger(minutes * 60_000) || !Number.isInteger(minutes) || minutes <= 0) { + throw new Error('grant-time requires --attempt, --grant-id, --minutes '); + } + return { command, directory: resolve(path), attemptId: options.get('--attempt')!, + grantId: options.get('--grant-id')!, minutes }; + } + if (command === 'grant-repairs' && path) { + const values: { attemptId?: string; grantId?: string; level?: number; repairs?: number; + nodeIds: string[] } = { nodeIds: [] }; + const seen = new Set(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + const value = rest[index + 1]; + if (flag === undefined || value === undefined + || !['--attempt', '--grant-id', '--level', '--feature', '--repairs'].includes(flag) + || (flag !== '--feature' && seen.has(flag))) { + throw new Error(`invalid or duplicate grant-repairs option ${String(flag)}`); + } + seen.add(flag); + if (flag === '--attempt') values.attemptId = value; + else if (flag === '--grant-id') values.grantId = value; + else if (flag === '--level') values.level = Number(value); + else if (flag === '--repairs') values.repairs = Number(value); + else values.nodeIds.push(value); + } + if (!values.attemptId || !values.grantId || typeof values.level !== 'number' + || !Number.isSafeInteger(values.level) || typeof values.repairs !== 'number' + || !Number.isSafeInteger(values.repairs) || values.nodeIds.length === 0) { + throw new Error('grant-repairs requires --attempt, --grant-id, --level, ' + + 'one or more --feature values, and --repairs'); + } + return { command, directory: resolve(path), attemptId: values.attemptId, + grantId: values.grantId, level: values.level, nodeIds: values.nodeIds, + repairs: values.repairs }; + } + if (command === 'extend' && path && (rest.length === 6 + || (rest.length === 7 && rest[6] === '--prepare-only')) + && rest[0] === '--from' && rest[2] === '--depth' && rest[4] === '--out') { + const fromDepth = Number(rest[3]); + if (!Number.isSafeInteger(fromDepth) || fromDepth < 1) { + throw new Error('extend --depth must be a positive integer'); + } + return { command, path: resolve(path), parentDirectory: resolve(rest[1]!), + fromDepth, directory: resolve(rest[5]!), prepareOnly: rest.length === 7 }; + } + if (isOneOf(command, ['trial', 'run', 'resume', 'reconcile']) + && path && rest.length === 2 && rest[0] === '--out') { + return { command, path: resolve(path), directory: resolve(rest[1]!) }; + } + throw new Error('usage: campaign-cli.js modes | validate|show ' + + '| trial|run|resume|reconcile --out ' + + '| extend --from --depth --out [--prepare-only] ' + + '| status [--full] | inspect|report|audit|stop | export --out ' + + '| grant-repairs --attempt --grant-id --level ' + + '--feature [--feature ...] --repairs ' + + '| grant-time --attempt --grant-id --minutes ' + + '| continue-provider --attempt --request-id ' + + '| continuation-status --attempt [--json] ' + + '| pause-status|continue-depth '); +} + +async function main() { + const args = parseCampaignArgs(process.argv); + if (args.command === 'pause-status' || args.command === 'continue-depth') { + console.log(JSON.stringify(args.command === 'pause-status' + ? campaignDepthPauseStatus(args.directory) : continueCampaignDepth(args.directory), null, 2)); + return; + } + if (args.command === 'modes') { + console.log(JSON.stringify(CAMPAIGN_MODE_REGISTRY.ids.map(value => { + const [id, version] = value.split('@'); + return { id, version }; + }), null, 2)); + return; + } + if (args.command === 'status') { + const campaign = inspectCampaign(args.directory, { requireCurrentInputs: false }); + console.log(JSON.stringify(args.full + ? campaign.state + : campaignStateSummary(campaign.plan, campaign.state), null, 2)); + return; + } + if (args.command === 'inspect') { + console.log(JSON.stringify(inspectCampaignSummary(args.directory), null, 2)); + return; + } + if (args.command === 'report') { + const generated = generateCampaignReport(args.directory); + console.log(`${generated.reportPath}\n${generated.htmlPath}\n${generated.report.contentSha256}`); + return; + } + if (args.command === 'export') { + console.log(exportCampaignReport(args.directory, args.output)); + return; + } + if (args.command === 'stop') { + const lock = readCampaignLock(args.directory); + if (!lock || !requestCampaignCancellation(args.directory, + { id: lock.campaignId, contentSha256: lock.campaignSha256 }, lock.ownershipMarkerSha256)) { + throw new Error('campaign has no current controller to stop'); + } + console.log('Stop requested. The controller will stop its children and release owned resources.'); + return; + } + if (args.command === 'audit') { + const report = auditProgressionReferenceCampaign(args.directory); + if (report === null) throw new Error('campaign has no dependency reference attempts to audit'); + console.log(formatProgressionReferenceCampaignAudit(report)); + if (!report.ok) process.exitCode = 1; + return; + } + if (args.command === 'continuation-status') { + const status = readCampaignProviderContinuationStatus(args.directory, args.attemptId); + console.log(args.json ? JSON.stringify(status, null, 2) + : status.eligible ? 'Waiting: eligible for provider continuation.' : `Ineligible: ${status.reason}`); + return; + } + if (args.command === 'continue-provider') { + console.log(JSON.stringify(requestCampaignProviderContinuation(args.directory, args), null, 2)); + return; + } + if (args.command === 'grant-time') { + console.log(JSON.stringify(requestCampaignTimeGrant(args.directory, { + attemptId: args.attemptId, grantId: args.grantId, minutes: args.minutes }), null, 2)); + return; + } + if (args.command === 'grant-repairs') { + console.log(JSON.stringify(grantCampaignDependencyRepairs(args.directory, { + attemptId: args.attemptId, + grantId: args.grantId, + level: args.level, + nodeIds: args.nodeIds, + repairs: args.repairs, + }), null, 2)); + return; + } + if (args.command === 'extend') { + prepareCampaignExtension(args.path, args.parentDirectory, args.directory, args.fromDepth); + if (args.prepareOnly) { + console.log(JSON.stringify({ status: 'prepared', directory: args.directory, + parentDirectory: args.parentDirectory, fromDepth: args.fromDepth }, null, 2)); + return; + } + const plan = compileCampaignFile(args.path); + const state = await executeCampaign(args.path, args.directory, { mode: 'frozen' }); + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + if (state.status !== 'completed') process.exitCode = 1; + return; + } + const plan = compileCampaignFile(args.path); + if (args.command === 'reconcile') { + const state = reconcileCampaign(args.path, args.directory); + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + return; + } + if (args.command === 'trial' || args.command === 'run' || args.command === 'resume') { + if (args.command === 'resume') validateResumeCampaign(args.path, args.directory); + const cancellation = new AbortController(); + const cancel = () => cancellation.abort(); + process.on('SIGINT', cancel); + process.on('SIGTERM', cancel); + let state; + try { + const executionMode = args.command === 'trial' + || (args.command === 'resume' && plan.state === 'draft') + ? 'model-free-trial' : 'frozen'; + state = await executeCampaign(args.path, args.directory, { + mode: executionMode, + signal: cancellation.signal, + }); + } finally { + process.off('SIGINT', cancel); + process.off('SIGTERM', cancel); + } + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + const audit = auditCompletedReferenceCampaign(args.directory, plan, state); + if (audit !== null) console.log(formatProgressionReferenceCampaignAudit(audit)); + if (state.status !== 'completed' || audit?.ok === false) process.exitCode = 1; + return; + } + if (args.command === 'show') console.log(JSON.stringify(plan, null, 2)); + else console.log(`${plan.id}@${plan.version} ${plan.state}: ${plan.summary.attempts} attempts, ${plan.contentSha256}`); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/check-actions.ts b/tools/stack-bench/commands/check-actions.ts new file mode 100644 index 00000000000..cf9a00f65db --- /dev/null +++ b/tools/stack-bench/commands/check-actions.ts @@ -0,0 +1,116 @@ +#!/usr/bin/env node +// Probes are unauthenticated or malformed and must never mutate data. + +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { emptyArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { probeConvexNamedAction } from '../src/stacks/backends/convex-operations.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; + +interface CheckActionsArgs { + backend?: string; + url?: string; + app?: string; + out?: string; + track?: string; + quiet?: boolean; + parentAttemptId?: string; +} + +import type { NamedAction } from '../src/composition/tracks.js'; + +interface ActionResult { + id: string; + ok: boolean; + status: number; + note: string; +} + +function parseArgs(argv: string[]): CheckActionsArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + backend: { type: 'string' }, url: { type: 'string' }, app: { type: 'string' }, + out: { type: 'string' }, track: { type: 'string' }, quiet: { type: 'boolean' }, + 'parent-attempt-id': { type: 'string' }, + } }); + const a: CheckActionsArgs = { backend: values.backend, url: values.url, app: values.app, + out: values.out, track: values.track, quiet: values.quiet, + parentAttemptId: values['parent-attempt-id'] }; + if (!a.backend) { console.error('--backend is required'); process.exit(2); } + return a; +} + +const args = parseArgs(process.argv); +const backend = args.backend; +if (!backend) throw new Error('--backend is required'); + +// Use non-writing probes declared by the selected track. +const track = args.track ? loadTrack(args.track) : null; +const browserAccounts = backend === 'convex' ? ['signUp', 'signIn'] : []; +const ACTIONS = (track?.actions ?? []).filter(action => !browserAccounts.includes(action.id)); +if (browserAccounts.length && !args.quiet) console.log('Account interfaces are checked through browser sign-up/sign-in, not password mutations.'); +if (!ACTIONS.length) { + if (!args.quiet) console.log(` no named actions declared for track "${args.track ?? '(none)'}" — nothing to check`); + if (args.out) { + const id = `${args.parentAttemptId ?? 'actions'}-action-check`; + writeArtifact(args.out, { kind: 'action_check', id, + attempt: { id, parentId: args.parentAttemptId ?? null }, + identities: emptyArtifactIdentities({ stackAdapter: { id: backend } }), + payload: { backend, results: [], missing: [] } }); + } + process.exit(0); +} + +// SpacetimeDB control targets come from the authenticated lease. Client config +// is app-controlled input and may use environment expressions rather than +// literals; it is neither authoritative nor safe for harness operations. +const adapter = STACK_ADAPTER_REGISTRY.get(backend); +const spacetime = adapter.grading.context({ requireBuildContainer: false }); + +async function probe(action: NamedAction): Promise> { + try { + if (backend === 'convex') return probeConvexNamedAction(action); + const request = adapter.namedAction.request( + { action, input: { args: action.args }, spacetime, url: args.url }); + if (!request?.url) return { ok: false, status: 0, note: 'no --url given for a server-based backend' }; + const r = await fetch(request.url, { + method: request.method ?? 'POST', + headers: { 'Content-Type': 'application/json' }, + body: request.body, + }); + const rejectedByApplication = 'applicationRejectionStatuses' in request + && request.applicationRejectionStatuses.includes(r.status); + const recognizedWithoutRunning = r.status >= 400 && r.status < 500 + && ![404, 405, 429].includes(r.status); + const ok = r.ok || rejectedByApplication || recognizedWithoutRunning; + return { ok, status: r.status, + note: r.status === 404 && 'missingNote' in request ? request.missingNote + : ok ? '' : `action probe returned HTTP ${r.status}` }; + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + return { ok: false, status: 0, note: (message.split('\n')[0] ?? '').slice(0, 90) }; + } +} + +const results: ActionResult[] = await Promise.all(ACTIONS.map(async action => ({ + id: action.id, + ...(await probe(action)), +}))); + +const missing = results.filter(r => !r.ok); +if (!args.quiet) { + for (const r of results) { + console.log(` ${r.ok ? 'ready' : 'UNUSABLE'} ${r.id.padEnd(11)} ${r.status ? `HTTP ${r.status}` : ''} ${r.note}`); + } + console.log(missing.length + ? `\n${missing.length} named action(s) unusable — contention and volume tests cannot be issued against this app.` + : '\nall named actions are ready.'); +} +if (args.out) { + const id = `${args.parentAttemptId ?? 'actions'}-action-check`; + writeArtifact(args.out, { kind: 'action_check', id, + attempt: { id, parentId: args.parentAttemptId ?? null }, + identities: emptyArtifactIdentities({ stackAdapter: { id: backend } }), + payload: { backend, results, missing: missing.map(m => m.id) } }); +} +process.exit(missing.length ? 1 : 0); diff --git a/tools/stack-bench/commands/check-calibration.ts b/tools/stack-bench/commands/check-calibration.ts new file mode 100644 index 00000000000..c7be197c7c4 --- /dev/null +++ b/tools/stack-bench/commands/check-calibration.ts @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { compileCalibrationDefinition, compileCalibrationFile } from '../src/composition/calibration-compiler.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { listTracks, TRACKS_DIR } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js'; + +export interface CalibrationCheckResult { + track: string; + id: string; + recipe: string; + controls: number; + stacks: number; + contentSha256: string; +} + +export function checkCalibrations( + { trackName = null }: { trackName?: string | null } = {}, +): CalibrationCheckResult[] { + const availableTracks = listTracks({ includeInternal: true }); + if (trackName && !availableTracks.includes(trackName)) { + throw new Error(`unknown calibration track ${trackName}`); + } + const tracks = trackName ? [trackName] : availableTracks; + const results: CalibrationCheckResult[] = []; + for (const name of tracks) { + const trackRoot = join(TRACKS_DIR, name); + const directory = join(trackRoot, 'composition', 'calibrations'); + if (!existsSync(directory)) continue; + for (const file of readdirSync(directory).filter(candidate => candidate.endsWith('.json')).sort()) { + const path = join(directory, file); + const source = `composition/calibrations/${file}`; + const input = JSON.parse(readFileSync(path, 'utf8')); + const definition = compileCalibrationDefinition(input, { source }); + const recipePath = resolve(dirname(path), definition.recipe.path); + const release = buildRecipeRelease(recipePath, { trackRoot }); + const plan = compileCalibrationFile(path, { trackRoot, stackBenchRoot: ROOT, release }); + results.push({ track: name, id: plan.id, + recipe: plan.recipe.id, controls: plan.controls.length, stacks: plan.qualification.stacks.length, + contentSha256: plan.contentSha256 }); + } + } + return results; +} + +function main() { + const { values } = parseArgs({ args: process.argv.slice(2), options: { + track: { type: 'string' }, + }, strict: true, allowPositionals: false }); + const results = checkCalibrations({ trackName: values.track ?? null }); + for (const result of results) { + console.log(`${result.track}: ${result.id}; ` + + `${result.controls} controls, ${result.stacks} stacks, ${result.contentSha256.slice(0, 12)}`); + } +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { main(); } + catch (error: unknown) { + console.error(error instanceof Error ? error.stack ?? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/tools/stack-bench/commands/check-composition.ts b/tools/stack-bench/commands/check-composition.ts new file mode 100644 index 00000000000..49a29c5fb3f --- /dev/null +++ b/tools/stack-bench/commands/check-composition.ts @@ -0,0 +1,87 @@ +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { + compileFixtureDefinition, + compilePackDefinition, + compileRecipeFile, + compileRecipeSelectionFile, +} from '../src/composition/composition-compiler.js'; +import { TRACKS_DIR, listTracks } from '../src/composition/tracks.js'; + +function json(path: string): unknown { + try { return JSON.parse(readFileSync(path, 'utf8')); } + catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`cannot read composition source ${path}: ${message}`, { cause: error }); + } +} + +export interface CompositionSummary { + track: string; + packs: number; + fixtures: number; + recipes: number; + checks: number; + selections: number; +} + +export function checkCompositions( + { trackName = null }: { trackName?: string | null } = {}, +): CompositionSummary[] { + const names = trackName ? [trackName] : listTracks({ includeInternal: true }); + const summary = []; + for (const name of names) { + const trackRoot = join(TRACKS_DIR, name); + const root = join(trackRoot, 'composition'); + if (!existsSync(root)) { + if (trackName) throw new Error(`track ${name} has no composition directory`); + continue; + } + const packs = join(root, 'packs'); + const fixtures = join(root, 'fixtures'); + const recipes = join(root, 'recipes'); + const packFiles = readdirSync(packs).filter(file => file.endsWith('.json')).sort(); + const fixtureFiles = readdirSync(fixtures).filter(file => file.endsWith('.json')).sort(); + const recipeFiles = readdirSync(recipes).filter(file => file.endsWith('.json')).sort(); + if (!packFiles.length || !fixtureFiles.length || !recipeFiles.length) { + throw new Error(`track ${name} composition must contain packs, fixtures, and recipes`); + } + for (const file of packFiles) { + const path = join(packs, file); + compilePackDefinition(json(path), { source: path }); + } + for (const file of fixtureFiles) { + const path = join(fixtures, file); + compileFixtureDefinition(json(path), { source: path }); + } + const plans = recipeFiles.map(file => compileRecipeFile(join(recipes, file), { trackRoot })); + const selectionFiles = readdirSync(root).filter(file => file.endsWith('.json')).sort(); + const selections = selectionFiles.reduce((total, file) => total + + compileRecipeSelectionFile(join(root, file), { trackRoot }).entries.length, 0); + summary.push({ track: name, packs: packFiles.length, fixtures: fixtureFiles.length, + recipes: plans.length, checks: plans.reduce((total, plan) => total + plan.checks.length, 0), + selections }); + } + return summary; +} + +function main(): void { + const args = process.argv.slice(2); + let trackName: string | null = null; + for (let index = 0; index < args.length; index += 1) { + const value = args[index + 1]; + if (args[index] === '--track' && value) { + trackName = value; + index += 1; + } else throw new Error(`unknown or incomplete argument ${args[index]}`); + } + const summary = checkCompositions({ trackName }); + if (!summary.length) throw new Error('no composition sources found'); + for (const row of summary) { + console.log(`${row.track}: ${row.packs} packs, ${row.fixtures} fixtures, ${row.recipes} recipes, ${row.checks} selected checks, ${row.selections} recipe selections`); + } +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/check-mutations.ts b/tools/stack-bench/commands/check-mutations.ts new file mode 100644 index 00000000000..042a6d9770b --- /dev/null +++ b/tools/stack-bench/commands/check-mutations.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from 'node:fs'; +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { mutationFileEdits, resolveMutationFile, validateMutationDefinitions } + from '../src/evidence/mutation-analysis.js'; +import type { MutationDefinition } from '../src/evidence/mutation-analysis.js'; + +interface CliArgs { + app: string; + mutations: string; + quiet: boolean; +} + +interface MutationSpec { + anchoredTo?: unknown; + mutations?: MutationDefinition[]; +} + +function parseArgs(argv: string[]): CliArgs { + const { values: { app, mutations, quiet = false } } = parseNodeArgs({ args: argv.slice(2), + options: { app: { type: 'string' }, mutations: { type: 'string' }, quiet: { type: 'boolean' } } }); + if (!app || !mutations) { + console.error('Usage: node dist/commands/check-mutations.js --app --mutations '); + process.exit(2); + } + return { app, mutations, quiet }; +} + +const args = parseArgs(process.argv); +const parsed: unknown = JSON.parse(readFileSync(args.mutations, 'utf8')); +if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('mutation manifest must be an object'); +} +const spec = parsed as MutationSpec; +const say = (...message: unknown[]): void => { if (!args.quiet) console.log(...message); }; + +say(`mutations : ${args.mutations}`); +say(`app : ${args.app}`); +if (spec.anchoredTo) say(`anchored : ${String(spec.anchoredTo).split('.')[0]}`); +say(''); + +let bad = 0; +const definitions = validateMutationDefinitions(spec.mutations); +for (const issue of definitions.issues) { + console.log(` BAD MANIFEST ${issue.mutation ?? ''} -> ${issue.kind}`); + bad += 1; +} +for (const mutation of spec.mutations ?? []) { + const mutationId = String(mutation.id ?? ''); + for (const edit of mutationFileEdits(mutation)) { + let file: string; + try { file = resolveMutationFile(args.app, edit.file); } + catch { + console.log(` UNSAFE FILE ${mutationId} -> ${edit.file} escapes the app directory`); + bad += 1; + continue; + } + if (!existsSync(file)) { + console.log(` DEAD FILE ${mutationId} -> ${edit.file} does not exist in this app`); + bad += 1; + continue; + } + const source = readFileSync(file, 'utf8'); + const matches = source.split(edit.find).length - 1; + if (matches === 1) { + say(` ok ${mutationId} -> ${edit.file}`); + continue; + } + console.log(matches === 0 + ? ` DEAD ANCHOR ${mutationId} -> not found in ${edit.file}` + : ` AMBIGUOUS ${mutationId} -> matches ${matches}x in ${edit.file}; the edit would land in more than one place`); + bad += 1; + } +} + +console.log(bad + ? `\n${bad} problem(s) — these mutations cannot validate anything against this app.` + : '\nall anchors present and unique — this file can validate against this app.'); +process.exit(bad ? 1 : 0); diff --git a/tools/stack-bench/commands/check-scenarios.ts b/tools/stack-bench/commands/check-scenarios.ts new file mode 100644 index 00000000000..5ebbcca0104 --- /dev/null +++ b/tools/stack-bench/commands/check-scenarios.ts @@ -0,0 +1,315 @@ +#!/usr/bin/env node +// Check scenario action names, actors, UI hooks, and score totals without an app. + +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { compileRecipeFile, type CompiledOwnedTaskFragment, type CompiledRecipeRelease } + from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition, type CompiledStep } + from '../src/composition/definition-compiler.js'; +import { DEFAULT_TRACK, listTracks, loadTrack, type Track } + from '../src/composition/tracks.js'; + +interface ScenarioScope { + features: Map>; + contractOwners: Set; + requirementOwners: Set; + contractText: string; + requirementText: string; +} + +interface RecipeSource { + baseRecipe: string | null; + isolatesSelectedSources: boolean; +} + +type HooksByLevel = Map>; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function readJson(path: string): unknown { + return JSON.parse(readFileSync(path, 'utf8')) as unknown; +} + +function readRecipeSource(path: string): RecipeSource { + const source = readJson(path); + if (!isRecord(source)) throw new Error(`${path}: recipe must be an object`); + const task = source.task; + if (!isRecord(task)) throw new Error(`${path}: recipe task must be an object`); + const baseRecipe = task.baseRecipe; + let baseRecipePath: string | null = null; + if (baseRecipe !== undefined) { + if (!isRecord(baseRecipe) || typeof baseRecipe.path !== 'string') { + throw new Error(`${path}: task.baseRecipe.path must be a string`); + } + baseRecipePath = baseRecipe.path; + } + return { + baseRecipe: baseRecipePath, + isolatesSelectedSources: source.execution === 'all-selected-sources', + }; +} + +function contractHookIds(path: string): string[] { + const contract = readJson(path); + if (!isRecord(contract) || !Array.isArray(contract.hooks)) { + throw new Error(`${path}: hooks must be an array`); + } + return contract.hooks.map((hook, index) => { + if (!isRecord(hook) || typeof hook.id !== 'string') { + throw new Error(`${path}: hooks[${index}].id must be a string`); + } + return hook.id; + }); +} + +// Contract levels are cumulative. A level can use hooks introduced earlier. +function hooksByLevel(track: Track): HooksByLevel { + const perFile = new Map(); + for (const file of readdirSync(track.contracts).filter(name => /^\d\d-.*\.json$/.test(name))) { + perFile.set(file.slice(0, 2), contractHookIds(join(track.contracts, file))); + } + const byLevel: HooksByLevel = new Map(); + for (const level of perFile.keys()) { + const ids = [...perFile.entries()] + .filter(([candidate]) => candidate <= level) + .flatMap(([, hookIds]) => hookIds); + byLevel.set(level, new Set(ids)); + } + return byLevel; +} + +function ownedFragment( + fragment: CompiledOwnedTaskFragment, + owners: ReadonlySet, +): boolean { + return fragment.owners.some(owner => owners.has(owner)); +} + +function recipeScenarioScopes(track: Track, recipeFile: string): Map { + const recipeDir = join(track.dir, 'composition', 'recipes'); + const chain: CompiledRecipeRelease[] = []; + const seen = new Set(); + let currentFile: string | null = recipeFile; + let isolatesSelectedSources = false; + while (currentFile !== null) { + if (seen.has(currentFile)) throw new Error(`recipe base cycle at ${currentFile}`); + seen.add(currentFile); + const path = join(recipeDir, currentFile); + chain.push(compileRecipeFile(path, { trackRoot: track.dir })); + const source = readRecipeSource(path); + if (chain.length === 1) isolatesSelectedSources = source.isolatesSelectedSources; + currentFile = source.baseRecipe; + } + + const recipe = chain[0]; + if (recipe === undefined) throw new Error(`recipe chain is empty for ${recipeFile}`); + const packs = new Map(recipe.packs.map(pack => [pack.id, pack])); + const contracts = isolatesSelectedSources + ? recipe.recipe.task.contracts + : chain.flatMap(release => release.recipe.task.contracts); + const requirements = isolatesSelectedSources + ? recipe.recipe.task.requirements + : chain.flatMap(release => release.recipe.task.requirements); + const scopes = new Map(); + + const ownersFor = (check: CompiledRecipeRelease['checks'][number]): Set => { + const found = new Set([check.packId, ...(check.requiresFeatures ?? [])]); + const visit = (id: string): void => { + const pack = packs.get(id); + if (pack === undefined) return; + for (const reference of pack.requiresPacks) { + if (found.has(reference)) continue; + found.add(reference); + visit(reference); + } + }; + [...found].forEach(visit); + return found; + }; + + for (const check of recipe.checks) { + const source = check.source.replace(/^scenarios\//, ''); + const scope = scopes.get(source) ?? { + features: new Map>(), + contractOwners: new Set(), + requirementOwners: new Set(), + contractText: '', + requirementText: '', + }; + const criteria = scope.features.get(check.featureId) ?? new Set(); + criteria.add(check.criterionId); + scope.features.set(check.featureId, criteria); + for (const owner of ownersFor(check)) { + scope.contractOwners.add(owner); + scope.requirementOwners.add(owner); + } + scopes.set(source, scope); + } + + for (const scope of scopes.values()) { + const selectedContracts = isolatesSelectedSources + ? contracts.filter(fragment => ownedFragment(fragment, scope.contractOwners)) + : contracts; + const selectedRequirements = isolatesSelectedSources + ? requirements.filter(fragment => ownedFragment(fragment, scope.requirementOwners)) + : requirements; + scope.contractText = selectedContracts.map(fragment => fragment.text).join('\n'); + scope.requirementText = selectedRequirements.map(fragment => fragment.text).join('\n'); + } + return scopes; +} + +function normalizeText(text: string): string { + return text.replace(/\*\*/g, '').replace(/—/g, '-').toLowerCase().replace(/\s+/g, ' ').trim(); +} + +function promptFor(track: Track, level: string): string | null { + const dir = join(track.dir, 'prompts'); + if (!existsSync(dir)) return null; + const file = readdirSync(dir).find(name => name.startsWith(`${level}-`) && name.endsWith('.md')); + return file === undefined ? null : normalizeText(readFileSync(join(dir, file), 'utf8')); +} + +function referencedActors(step: CompiledStep): string[] { + return [step.do === 'expectCrashCheckout' ? undefined : step.from, step.fromActor] + .filter((actor): actor is string => actor !== undefined); +} + +export function referencedInterfaceValues(value: unknown, key: 'testid' | 'attribute'): string[] { + if (Array.isArray(value)) return value.flatMap(child => referencedInterfaceValues(child, key)); + if (!isRecord(value)) return []; + return Object.entries(value).flatMap(([name, child]) => name === key && typeof child === 'string' + ? [child] : referencedInterfaceValues(child, key)); +} + +function referencedTestIds(step: CompiledStep): string[] { + return referencedInterfaceValues(step, 'testid'); +} + +function main(args: readonly string[]): number { + const { values } = parseArgs({ args: [...args], options: { + track: { type: 'string' }, + recipe: { type: 'string' }, + }, strict: true, allowPositionals: false }); + const trackArg = values.track ?? null; + const recipeArg = values.recipe ?? null; + const availableTracks = listTracks(); + const trackNames = trackArg === null + ? (availableTracks.length > 0 ? availableTracks : [DEFAULT_TRACK]) + : [trackArg]; + if (recipeArg !== null && trackNames.length !== 1) { + throw new Error('--recipe requires one --track'); + } + + const knownActions = new Set(ACTION_REGISTRY.ids); + let problems = 0; + let unstatedWarnings = 0; + let staleStatementWarnings = 0; + const fail = (where: string, message: string): void => { + console.log(` ${where}: ${message}`); + problems += 1; + }; + + for (const name of trackNames) { + const track = loadTrack(name); + console.log(`# track: ${name}`); + const contracts = hooksByLevel(track); + const recipeScopes = recipeArg === null ? null : recipeScenarioScopes(track, recipeArg); + for (const file of readdirSync(track.scenarios).filter(candidate => candidate.endsWith('.json'))) { + const recipeScope = recipeScopes?.get(file); + if (recipeScopes !== null && recipeScope === undefined) continue; + const scenarioPath = join(track.scenarios, file); + let spec; + try { + spec = compileScenarioDefinition(readJson(scenarioPath), { source: scenarioPath }); + } catch (error: unknown) { + fail(file, error instanceof Error ? error.message : String(error)); + continue; + } + const level = String(spec.level).padStart(2, '0'); + const hooks = recipeScope === undefined ? (contracts.get(level) ?? null) : null; + + console.log(file); + const prompt = recipeScope === undefined + ? promptFor(track, level) + : normalizeText(recipeScope.requirementText); + for (const feature of spec.features) { + const selectedCriteria = recipeScope?.features.get(feature.id); + if (recipeScope !== undefined && selectedCriteria === undefined) continue; + const criteria = selectedCriteria === undefined + ? feature.criteria + : feature.criteria.filter(criterion => selectedCriteria.has(criterion.id)); + for (const criterion of criteria) { + if (criterion.statedBy !== undefined) { + if (recipeScope === undefined && prompt !== null + && !prompt.includes(normalizeText(criterion.statedBy))) { + staleStatementWarnings += 1; + console.log(` warn F${feature.id} ${criterion.id}: statedBy text is not in the level ${level} prompt`); + } + } else if (recipeScope === undefined && criterion.points > 0) { + unstatedWarnings += 1; + console.log(` warn F${feature.id} ${criterion.id}: carries ${criterion.points} point(s) with no statedBy - the requirement may be unstated`); + } + } + + const actors = new Set(feature.actors ?? []); + const steps = [...feature.setup, ...criteria.flatMap(criterion => criterion.steps)]; + const declared = (actor: string): boolean => actors.has(actor) + || [...actors].some(candidate => actor.startsWith(`${candidate}-`)); + for (const step of steps) { + const at = `F${feature.id} ${step.do}`; + if (!knownActions.has(step.do)) fail(at, `unknown step type "${step.do}"`); + if (step.actor !== undefined && actors.size > 0 && !declared(step.actor)) { + fail(at, `actor "${step.actor}" is not in the feature's actor list`); + } + for (const actor of referencedActors(step)) { + if (actors.size > 0 && !declared(actor)) { + fail(at, `actor "${actor}" is not in the feature's actor list`); + } + } + if (hooks !== null) { + for (const id of referencedTestIds(step)) { + if (!hooks.has(id)) fail(at, `testid "${id}" is not in the contract`); + } + } + if (recipeScope !== undefined) { + for (const id of referencedTestIds(step)) { + if (!recipeScope.contractText.includes(`\`${id}\``)) { + fail(at, `testid "${id}" is not in the selected recipe contracts`); + } + } + for (const attribute of referencedInterfaceValues(step, 'attribute')) { + if (attribute.startsWith('data-') && !recipeScope.contractText.includes(`\`${attribute}\``)) { + fail(at, `attribute "${attribute}" is not in the selected recipe contracts`); + } + } + } + } + + const points = criteria.reduce((total, criterion) => total + criterion.points, 0); + if (recipeScope === undefined && feature.max !== undefined && points !== feature.max) { + fail(`F${feature.id}`, `criteria total ${points} but max says ${feature.max}`); + } + } + } + } + + const warnings = unstatedWarnings + staleStatementWarnings; + console.log(problems > 0 + ? `\n${problems} error(s); ${warnings} warning(s)` + : warnings > 0 + ? `\n0 errors; ${warnings} warning(s) (${unstatedWarnings} point-carrying criteria lack statedBy; ${staleStatementWarnings} statedBy references are outside level prompts)` + : '\n0 errors; 0 warnings'); + return problems > 0 ? 1 : 0; +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + process.exitCode = main(process.argv.slice(2)); +} diff --git a/tools/stack-bench/commands/composition-cli.ts b/tools/stack-bench/commands/composition-cli.ts new file mode 100644 index 00000000000..13411828c2c --- /dev/null +++ b/tools/stack-bench/commands/composition-cli.ts @@ -0,0 +1,315 @@ +#!/usr/bin/env node + +import { existsSync, readdirSync, realpathSync } from 'node:fs'; +import { join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { compilePackDefinition, compileRecipeFile, resolveTaskFragment } from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { canonicalDefinitionJson, readDefinitionJson } + from '../src/composition/definition-plan.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { composeSelectedRecipeTask, selectRecipeRelease } from '../src/composition/recipe-selection.js'; +import { TRACKS_DIR } from '../src/composition/tracks.js'; +import type { CompiledPackDefinition, CompiledRecipePlan } from '../src/composition/composition-compiler.js'; +import type { RecipeRelease } from '../src/composition/recipe-release.js'; +import type { RecipeSelectionOptions, SelectedRecipeRelease } from '../src/composition/recipe-selection.js'; + +export { selectRecipeRelease } from '../src/composition/recipe-selection.js'; + +interface TrackRootOptions { + trackRoot: string; +} + +interface PackIndexEntry { + pack: CompiledPackDefinition; + path: string; +} + +interface CalibrationValue { + id: string; + recipe?: { id?: string; contentSha256?: string }; +} + +type RecipeOptions = TrackRootOptions & RecipeSelectionOptions; +type RecipeTaskKind = 'requirements' | 'contracts'; + +function contained(root: string, path: string, label: string): string { + const absoluteRoot = realpathSync(resolve(root)); + const candidate = resolve(path); + const lexical = relative(absoluteRoot, candidate); + if (lexical === '..' || lexical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`); + if (!existsSync(candidate)) throw new Error(`${label} does not exist: ${candidate}`); + const absolute = realpathSync(candidate); + const physical = relative(absoluteRoot, absolute); + if (physical === '..' || physical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`); + return absolute; +} + +function packIndex(trackRoot: string): Map { + const directory = join(trackRoot, 'composition', 'packs'); + const byRef = new Map(); + for (const name of readdirSync(directory).filter(file => file.endsWith('.json')).sort()) { + const path = join(directory, name); + const pack = compilePackDefinition(readDefinitionJson(path, 'pack'), { + source: relative(trackRoot, path).replaceAll('\\', '/'), + }); + if (byRef.has(pack.id)) throw new Error(`duplicate pack id ${pack.id}`); + byRef.set(pack.id, { pack, path: realpathSync(path) }); + } + for (const [ref, { pack }] of byRef) { + for (const dependency of [...pack.requiresPacks, ...pack.conflictsWith]) { + if (!byRef.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`); + } + } + return byRef; +} + +export function validatePackFile(path: string, options: Partial = {}) { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('pack validation requires trackRoot'); + const root = realpathSync(resolve(trackRoot)); + const absolute = contained(join(root, 'composition'), path, 'pack path'); + const pack = compilePackDefinition(readDefinitionJson(absolute, 'pack'), { + source: relative(root, absolute).replaceAll('\\', '/'), + }); + const packs = packIndex(root); + const indexed = packs.get(pack.id); + if (!indexed || indexed.path !== absolute) throw new Error(`${pack.id} is not the indexed source ${absolute}`); + for (const ref of [...pack.requiresPacks, ...pack.conflictsWith]) { + if (!packs.has(ref)) throw new Error(`${pack.id} references missing pack ${ref}`); + } + const sourceCache = new Map(); + for (const kind of ['requirements', 'contracts'] satisfies RecipeTaskKind[]) { + for (const fragment of pack.task[kind]) { + resolveTaskFragment(fragment, { trackRoot: root, + source: `${relative(root, absolute).replaceAll('\\', '/')}.task.${kind}.${fragment.id}`, + sourceCache }); + } + } + const state = new Map(); + const visit = (ref: string, chain: string[] = []): void => { + if (state.get(ref) === 'done') return; + if (state.get(ref) === 'visiting') throw new Error(`pack dependency cycle: ${[...chain, ref].join(' -> ')}`); + state.set(ref, 'visiting'); + const entry = packs.get(ref); + if (!entry) throw new Error(`missing pack release ${ref}`); + for (const dependency of entry.pack.requiresPacks) { + if (!packs.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`); + visit(dependency, [...chain, ref]); + } + state.set(ref, 'done'); + }; + visit(pack.id); + let criteria = 0; + for (const check of pack.checks) { + const scenarioPath = contained(root, join(root, check.source), `${pack.id}.${check.id}.source`); + const scenario = compileScenarioDefinition(readDefinitionJson(scenarioPath, 'scenario'), { + source: relative(root, scenarioPath).replaceAll('\\', '/'), + }); + const feature = scenario.features.find(candidate => candidate.id === check.feature); + if (!feature) throw new Error(`${pack.id}.${check.id} references missing feature ${check.feature}`); + criteria += feature.criteria.length; + } + return { id: pack.id, path: absolute, + checkGroups: pack.checks.length, criteria, requiresPacks: pack.requiresPacks }; +} + +export function validateRecipeFile(path: string, options: Partial = {}): { + plan: CompiledRecipePlan; + release: RecipeRelease; +} { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('recipe validation requires trackRoot'); + const absolute = contained(join(trackRoot, 'composition'), path, 'recipe path'); + const plan = compileRecipeFile(absolute, { trackRoot }); + const release = buildRecipeRelease(absolute, { trackRoot }); + return { plan, release }; +} + +export function showRecipeFile(path: string, options: RecipeOptions): SelectedRecipeRelease & { + builderTask: ReturnType & { note: string }; +} { + const compiled = validateRecipeFile(path, options); + const selected = selectRecipeRelease(compiled.release, options); + const builderTask = composeSelectedRecipeTask(compiled.plan, selected.selection); + return { + ...selected, + builderTask: { + ...builderTask, + note: 'Pack selection defines the requested task; a check-only filter narrows measurement inside it.', + }, + }; +} + +const same = (left: unknown, right: unknown): boolean => + canonicalDefinitionJson(left) === canonicalDefinitionJson(right); + +function meaningView(release: RecipeRelease) { + return { + track: release.track, + task: release.task, + checks: release.checkCatalog.map(({ stableKey, packId, checkGroupId, role, source, + featureId, criterionId, description }) => ({ stableKey, packId, checkGroupId, role, + source, featureId, criterionId, description })), + }; +} + +function scoringView(release: RecipeRelease) { + return { scoring: release.scoring, + checks: release.checkCatalog.map(({ stableKey, points }) => ({ stableKey, points })) }; +} + +function metadataView(release: RecipeRelease) { + return { id: release.id, title: release.title, + sequence: release.sequence, sourceManifestSha256: release.sourceManifestSha256 }; +} + +function matchingCalibrations(trackRoot: string, release: RecipeRelease): Array<{ + path: string; + value: CalibrationValue; +}> { + const directory = join(trackRoot, 'composition', 'calibrations'); + if (!existsSync(directory)) return []; + return readdirSync(directory).filter(name => name.endsWith('.json')).sort() + .map(name => ({ path: join(directory, name), + value: readDefinitionJson(join(directory, name), 'calibration') })) + .filter(({ value }) => value.recipe?.id === release.id + && value.recipe?.contentSha256 === release.contentSha256); +} + +export function diffRecipeFiles(fromPath: string, toPath: string, options: Partial = {}) { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('recipe diff requires trackRoot'); + const from = validateRecipeFile(fromPath, { trackRoot }).release; + const to = validateRecipeFile(toPath, { trackRoot }).release; + const categories = { + meaning: !same(meaningView(from), meaningView(to)), + scoring: !same(scoringView(from), scoringView(to)), + fixtures: !same(from.components.fixture, to.components.fixture), + execution: from.executionSha256 !== to.executionSha256, + metadata: !same(metadataView(from), metadataView(to)), + }; + const recipeBindingChanged = from.id !== to.id + || from.meaningSha256 !== to.meaningSha256 || from.executionSha256 !== to.executionSha256 + || from.contentSha256 !== to.contentSha256; + const calibrations = matchingCalibrations(trackRoot, from).map(({ path, value }) => { + const invalidated = []; + if (recipeBindingChanged) invalidated.push('recipe binding'); + if (categories.fixtures) invalidated.push('fixture binding'); + if (categories.scoring) invalidated.push('zero-point control policy'); + if (categories.meaning || categories.scoring || categories.execution || categories.fixtures) { + invalidated.push('reference repetitions', 'mutation repetitions'); + } + if (categories.meaning || categories.scoring || categories.fixtures) invalidated.push('null repetitions'); + if (recipeBindingChanged) invalidated.push('selection binding'); + return { id: value.id, + path: relative(trackRoot, path).replaceAll('\\', '/'), invalidated: [...new Set(invalidated)] }; + }); + const fragmentDiff = (kind: RecipeTaskKind) => { + const before = new Map(from.task[kind].map(fragment => [fragment.id, fragment])); + const after = new Map(to.task[kind].map(fragment => [fragment.id, fragment])); + return { + added: [...after.keys()].filter(key => !before.has(key)).sort(), + removed: [...before.keys()].filter(key => !after.has(key)).sort(), + changed: [...after.keys()].filter(key => before.has(key) + && !same(before.get(key), after.get(key))).sort(), + }; + }; + return { + from: { id: from.id, meaningSha256: from.meaningSha256, + executionSha256: from.executionSha256, contentSha256: from.contentSha256 }, + to: { id: to.id, meaningSha256: to.meaningSha256, + executionSha256: to.executionSha256, contentSha256: to.contentSha256 }, + categories, + taskFragments: { + requirements: fragmentDiff('requirements'), + contracts: fragmentDiff('contracts'), + composedTaskChanged: from.task.composedSha256 !== to.task.composedSha256, + }, + calibrations, + }; +} + +type CliSubject = 'pack' | 'recipe'; +type CliCommand = 'validate' | 'show' | 'diff'; + +interface ParsedArgs extends RecipeSelectionOptions { + json: boolean; + positional: string[]; + packIds: string[]; + checkKeys: string[]; + track?: string; + trackRoot?: string; +} + +interface CliArgs extends ParsedArgs { + subject: CliSubject; + command: CliCommand; + paths: string[]; + trackRoot: string; +} + +function parse(argv: string[]): CliArgs { + const { positionals, values } = parseArgs({ args: argv.slice(2), allowPositionals: true, + options: { track: { type: 'string' }, 'track-root': { type: 'string' }, + pack: { type: 'string', multiple: true }, check: { type: 'string', multiple: true }, + json: { type: 'boolean' } } }); + const args: ParsedArgs = { json: values.json ?? false, positional: positionals, + packIds: (values.pack ?? []).flatMap(value => value.split(',').filter(Boolean)), + checkKeys: (values.check ?? []).flatMap(value => value.split(',').filter(Boolean)), + track: values.track, + trackRoot: values['track-root'] === undefined ? undefined : resolve(values['track-root']) }; + const [subject, command, ...paths] = args.positional; + if (subject !== 'pack' && subject !== 'recipe') { + throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track '); + } + if (command !== 'validate' && command !== 'show' && command !== 'diff') { + throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track '); + } + if (subject === 'pack' && command !== 'validate') throw new Error(`pack ${command} is not supported`); + if ((command === 'diff' ? paths.length !== 2 : paths.length !== 1)) throw new Error(`${subject} ${command} received the wrong number of paths`); + if (!args.trackRoot && !args.track) throw new Error('--track or --track-root is required'); + if ((args.packIds.length || args.checkKeys.length) && !(subject === 'recipe' && command === 'show')) { + throw new Error('--pack and --check are allowed only with recipe show'); + } + const trackRoot = args.trackRoot ?? join(TRACKS_DIR, args.track ?? ''); + return { ...args, subject, command, paths, trackRoot }; +} + +function main() { + const args = parse(process.argv); + const firstPath = args.paths[0]; + if (firstPath === undefined) throw new Error('command requires a source path'); + let result: object; + if (args.subject === 'pack') result = validatePackFile(firstPath, args); + else if (args.command === 'diff') { + const secondPath = args.paths[1]; + if (secondPath === undefined) throw new Error('recipe diff requires two source paths'); + result = diffRecipeFiles(firstPath, secondPath, args); + } else if (args.command === 'show') result = showRecipeFile(firstPath, args); + else { + const compiled = validateRecipeFile(firstPath, args); + result = { + id: compiled.release.id, + packs: compiled.release.components.packs.length, checks: compiled.release.checkCatalog.length, + points: compiled.release.checkCatalog.reduce((total, check) => total + check.points, 0), + meaningSha256: compiled.release.meaningSha256, + executionSha256: compiled.release.executionSha256, + contentSha256: compiled.release.contentSha256, + }; + } + if (args.json || args.command === 'show' || args.command === 'diff') console.log(JSON.stringify(result, null, 2)); + else if ('id' in result) { + console.log(`${String(result.id)}: valid`); + } else throw new Error('validation result has no release identity'); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} diff --git a/tools/stack-bench/commands/container-smoke.ts b/tools/stack-bench/commands/container-smoke.ts new file mode 100644 index 00000000000..1c7c9702e23 --- /dev/null +++ b/tools/stack-bench/commands/container-smoke.ts @@ -0,0 +1,206 @@ +#!/usr/bin/env node +// Use only ephemeral resources owned by this smoke run. + +import { spawn, execFileSync } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { createServer } from 'node:net'; +import type { AddressInfo } from 'node:net'; +import { basename, join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { killTree, pidsOnPort, processIdentity } from '../src/runtime/platform.js'; +import { buildContainerName } from '../container/reconcile-build-container.js'; +import { createBackendLease, readBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { fetchStatus } from '../src/runtime/readiness.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { containerReachableSpacetimeUri } from '../src/runtime/spacetime-target.js'; +import { CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_SPACETIME_CLI, + codingContainerAgentExecOptions } from '../src/runtime/coding-container-policy.js'; +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; +const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release', + process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli'); +const RUN_BUILD = compiledEntrypoint('container', 'run-build.js'); +const FIXTURE = join(ROOT, 'tests', 'fixtures', 'spacetime-module'); + +interface PreparedContainerIdentity { + containerName: string; + identity: string; + networkMode: string | null; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parsePreparedContainerIdentity(text: string): PreparedContainerIdentity { + const value: unknown = JSON.parse(text.trim().split(/\r?\n/).pop() ?? ''); + if (!isRecord(value)) throw new Error('prepared container identity is invalid'); + const record = value; + if (typeof record.containerName !== 'string' || typeof record.identity !== 'string' + || (record.networkMode !== null && typeof record.networkMode !== 'string')) { + throw new Error('prepared container identity is invalid'); + } + return { containerName: record.containerName, identity: record.identity, networkMode: record.networkMode }; +} + +async function freePort() { + const server = createServer(); + await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port'); + const port: AddressInfo['port'] = address.port; + await new Promise(ok => server.close(ok)); + return port; +} + +async function waitFor(check: () => boolean | Promise, timeoutMs: number, description: string): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await check()) return; + await delay(250); + } + throw new Error(`timed out waiting for ${description}`); +} + +async function main() { + if (!existsSync(CLI)) throw new Error(`local SpacetimeDB CLI is missing: ${CLI}`); + execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' }); + + const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-smoke-')); + const app = join(root, 'app'); + const dataDir = join(root, 'spacetime-data'); + const port = await freePort(); + const uri = `http://127.0.0.1:${port}`; + const module = `stackbench-container-smoke-${process.pid}`; + const containerName = buildContainerName({ runId: basename(root), resources: {} }); + const leasePath = join(root, ARTIFACT_FILE.backendLease); + let host: ChildProcess | null = null; + let dev: ChildProcess | null = null; + let output = ''; + + try { + mkdirSync(app, { recursive: true }); + host = spawn(CLI, ['start', '--listen-addr', `127.0.0.1:${port}`, '--data-dir', dataDir], + { stdio: 'ignore', windowsHide: true }); + await waitFor(async () => { + const status = await fetchStatus(`${uri}/v1/ping`, { timeoutMs: 5000 }); + return status !== null && status >= 200 && status < 300; + }, 120_000, `dedicated SpacetimeDB host on :${port}`); + + const lease = createBackendLease({ runId: basename(root), backend: 'spacetime', + track: 'container-smoke', runIndex: 0, serverUri: uri, module, dataDir }); + lease.state = 'active'; + lease.resources.launchedProcess = host.pid ? processIdentity(host.pid) : null; + lease.resources.listenerProcesses = pidsOnPort(port).map(pid => processIdentity(pid)) + .filter((identity): identity is NonNullable => identity !== null); + writeBackendLease(leasePath, lease); + + const prepared = execFileSync(process.execPath, + [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--image', IMAGE, '--prepare-only'], + { encoding: 'utf8', stdio: 'pipe', maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken, STACK_BENCH_STDB_URI: uri } }); + const identity = parsePreparedContainerIdentity(prepared); + if (identity.containerName !== containerName) { + throw new Error(`prepared unexpected container ${identity.containerName}`); + } + const leasedContainer = readBackendLease(leasePath, + { token: lease.ownershipToken, backend: 'spacetime', active: true }).resources.buildContainer; + if (!leasedContainer || identity.identity.split(' ')[0] !== leasedContainer.id) { + throw new Error('prepared container identity was not recorded in the backend lease'); + } + if (!leasedContainer.image || !/^sha256:[0-9a-f]{64}$/.test(leasedContainer.image)) { + throw new Error(`prepared container did not record an immutable image id: ${leasedContainer.image}`); + } + + cpSync(FIXTURE, join(app, 'spacetimedb'), { recursive: true }); + const agentExec = ['exec', ...codingContainerAgentExecOptions()]; + const browserDom = execFileSync('docker', [...agentExec, containerName, 'chromium', + '--headless', '--no-sandbox', '--disable-dev-shm-usage', '--dump-dom', + 'data:text/html,'], + { encoding: 'utf8', stdio: 'pipe', timeout: 30_000 }); + if (!browserDom.includes('42')) throw new Error('agent browser did not execute JavaScript'); + const cliAccess = execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `stat -c '%a %U %G' ${CODING_CONTAINER_SPACETIME_CLI}; test -x ${CODING_CONTAINER_SPACETIME_CLI}`], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/^755 root root/m.test(cliAccess)) throw new Error(`unexpected CLI access: ${cliAccess.trim()}`); + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && npm install --no-audit --no-fund`], + { stdio: 'pipe' }); + + const startedDev = spawn('docker', [...agentExec, '-i', containerName, 'sh', '-c', + `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && ${CODING_CONTAINER_SPACETIME_CLI} dev ${module} ` + + `--project-path ${CODING_CONTAINER_APP_ROOT}/spacetimedb --module-path . ` + + '--server-only --skip-generate ' + + `-s ${containerReachableSpacetimeUri({ resources: { serverUri: uri, + buildContainer: lease.resources.buildContainer } }, identity.networkMode)} -y`], + { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); + dev = startedDev; + const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-128 * 1024); }; + startedDev.stdout?.on('data', collect); + startedDev.stderr?.on('data', collect); + + await waitFor(() => { + if (/Published successfully!/.test(output)) return true; + if (startedDev.exitCode !== null) throw new Error(`spacetime dev exited ${startedDev.exitCode}:\n${output}`); + return false; + }, 240_000, 'containerized module publish'); + + const sql = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/\bid\s*\|\s*value\b/.test(sql)) throw new Error(`SQL verification failed:\n${sql}`); + if (startedDev.exitCode !== null) throw new Error('spacetime dev did not remain alive as a watcher'); + + // Publishing and log streaming must retain one authenticated identity. A + // prior dev bug published with a token stored only in a Config clone, then + // directly logged in again for logs and received an authorization error. + await delay(2_000); + const logStreamingAuthorized = !/Log streaming error:.*not authorized/s.test(output); + console.log(JSON.stringify({ ok: true, image: IMAGE, container: identity.identity, + host: { uri, listenerPids: pidsOnPort(port) }, published: true, sqlVerified: true, + watcherAlive: true, leasedContainer: true, immutableImagePinned: true, + logStreamingAuthorized }, null, 2)); + if (!logStreamingAuthorized) { + throw new Error('`spacetime dev` published successfully but its log stream was not authorized'); + } + // The grader resets by republishing the same named database from this exact + // leased container. Prove that `-y` retained a reusable local identity, + // rather than merely proving that the first anonymous-looking publish ran. + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + 'for process in /proc/[0-9]*; do ' + + 'test "$(readlink "$process/exe" 2>/dev/null)" = /deps/.spacetimedb-cli ' + + '&& kill -TERM "${process##*/}" || true; done'], { stdio: 'pipe' }); + await waitFor(() => startedDev.exitCode !== null, 15_000, 'spacetime dev to stop before reset publish'); + const targetUri = containerReachableSpacetimeUri({ resources: { serverUri: uri, + buildContainer: lease.resources.buildContainer } }, identity.networkMode); + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && ${CODING_CONTAINER_SPACETIME_CLI} publish ${module} ` + + `--module-path . -s ${targetUri} --delete-data -y`], + { stdio: 'pipe', timeout: 240_000 }); + const afterReset = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/\bid\s*\|\s*value\b/.test(afterReset)) { + throw new Error(`SQL verification after reset publish failed:\n${afterReset}`); + } + console.log(JSON.stringify({ resetRepublished: true, resetSqlVerified: true })); + } finally { + if (dev && dev.exitCode === null) dev.kill('SIGTERM'); + try { execFileSync('docker', ['rm', '-f', containerName], { stdio: 'ignore' }); } catch { /* absent */ } + // The port was proven unused before this script started the host. Kill only + // listeners on that exact ephemeral port, then the wrapper if it remains. + for (const pid of pidsOnPort(port)) killTree(pid); + if (host && host.exitCode === null) killTree(host.pid); + rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/definition-snapshots.ts b/tools/stack-bench/commands/definition-snapshots.ts new file mode 100644 index 00000000000..9f227e19d30 --- /dev/null +++ b/tools/stack-bench/commands/definition-snapshots.ts @@ -0,0 +1,80 @@ +import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { canonicalDefinitionJson, compileTrackPlan } from '../src/composition/definition-plan.js'; +import { listTracks } from '../src/composition/tracks.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +const SNAPSHOT_DIR = join(STACK_BENCH_ROOT, 'tests', 'snapshots', 'definitions'); +const ALL_ACTIONS = join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'definitions', 'all-actions.json'); + +function atomicWrite(path: string, contents: string): void { + mkdirSync(dirname(path), { recursive: true }); + const temporary = `${path}.tmp-${process.pid}`; + writeFileSync(temporary, contents); + renameSync(temporary, path); +} + +interface DefinitionSnapshot { + name: string; + value: unknown; +} + +export function currentDefinitionSnapshots(): DefinitionSnapshot[] { + const entries: DefinitionSnapshot[] = listTracks({ includeInternal: true }).map(name => ({ + name: `${name}.snapshot.json`, + value: compileTrackPlan(name), + })); + entries.push({ + name: 'all-actions.snapshot.json', + value: compileScenarioDefinition(JSON.parse(readFileSync(ALL_ACTIONS, 'utf8')), { + source: ALL_ACTIONS, + }), + }); + return entries.sort((a, b) => a.name.localeCompare(b.name)); +} + +export interface DefinitionSnapshotResult { + checked: number; + changed: string[]; +} + +export function checkDefinitionSnapshots( + { update = false }: { update?: boolean } = {}, +): DefinitionSnapshotResult { + const entries = currentDefinitionSnapshots(); + const changed: string[] = []; + for (const entry of entries) { + const path = join(SNAPSHOT_DIR, entry.name); + const actual = canonicalDefinitionJson(entry.value); + let expected: string | null = null; + try { + expected = readFileSync(path, 'utf8').replaceAll('\r\n', '\n'); + } catch (error: unknown) { + if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error; + } + if (expected === actual) continue; + changed.push(entry.name); + if (update) atomicWrite(path, actual); + } + if (changed.length > 0 && !update) { + throw new Error( + `definition snapshot drift: ${changed.join(', ')}; inspect the semantic change, then run npm run check:definition-snapshots -- --update`, + ); + } + return { checked: entries.length, changed }; +} + +function main(): void { + const args = new Set(process.argv.slice(2)); + for (const arg of args) { + if (arg !== '--update') throw new Error(`unknown argument ${arg}`); + } + const result = checkDefinitionSnapshots({ update: args.has('--update') }); + console.log(`${result.checked} definition snapshots checked${ + result.changed.length > 0 ? `; ${result.changed.length} updated` : '; no drift'}`); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/fault-injection.ts b/tools/stack-bench/commands/fault-injection.ts new file mode 100644 index 00000000000..60087d56e19 --- /dev/null +++ b/tools/stack-bench/commands/fault-injection.ts @@ -0,0 +1,253 @@ +#!/usr/bin/env node +// Fault injection may remove only resources owned by its lease. + +import assert from 'node:assert/strict'; +import { execFileSync, spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { killTree, pidsOnPort } from '../src/runtime/platform.js'; +import { buildContainerName } from '../container/reconcile-build-container.js'; +import { ARTIFACT_FILE, readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; +const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release', + process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli'); +const RUN_BUILD = compiledEntrypoint('container', 'run-build.js'); +interface ContainerIdentity { id: string; running: boolean; } +interface ExitResult { code: number | null; signal: NodeJS.Signals | null; } +interface FaultLeaseResources { + listenerProcesses: Array<{ pid: number; startMarker: string }>; + buildContainer: { id: string; image: string; running: boolean; removedAt?: string }; + locks: { releasedAt?: string }[]; +} +interface FaultLeaseEvidence { + runId: string; + state: string; + stoppedAt?: string; + releasedAt?: string; + resources: FaultLeaseResources; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +async function freePort() { + const server = createServer((_request, response) => response.end('foreign')); + await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port'); + const port: AddressInfo['port'] = address.port; + await new Promise(ok => server.close(ok)); + return port; +} + +function inspectContainer(target: string): ContainerIdentity | null { + try { + const output = execFileSync('docker', ['inspect', '--format', + '{{.Id}} {{.State.Running}}', target], { encoding: 'utf8', stdio: 'pipe' }).trim(); + const [id, running] = output.split(/\s+/, 2); + if (!id) return null; + return { id, running: running === 'true' }; + } catch { return null; } +} + +function startContainer(name: string): ContainerIdentity { + const id = execFileSync('docker', ['run', '-d', '--init', '--name', name, + IMAGE, 'sleep', 'infinity'], { encoding: 'utf8', stdio: 'pipe' }).trim(); + assert.ok(id, `Docker did not return an id for ${name}`); + const container = inspectContainer(id); + if (!container) throw new Error(`Docker did not return a running container for ${name}`); + return container; +} + +function removeExactContainer(identity: ContainerIdentity | { id: string } | null): void { + if (!identity) return; + const current = inspectContainer(identity.id); + if (!current || current.id !== identity.id) return; + execFileSync('docker', ['rm', '-f', identity.id], { stdio: 'ignore' }); +} + +async function waitForExit(child: ChildProcess, timeoutMs: number): Promise { + return new Promise((resolveExit, reject) => { + const timeout = setTimeout(() => reject(new Error( + `benchmark runner did not exit after injected failure within ${timeoutMs}ms`)), timeoutMs); + child.once('exit', (code: number | null, signal: NodeJS.Signals | null) => { + clearTimeout(timeout); + resolveExit({ code, signal }); + }); + }); +} + +async function assertRefusesUnleasedCollision() { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-collision-')); + const app = join(root, 'app'); + const runId = `collision-${process.pid}`; + // Plant the foreign container under the exact name the launcher will claim. + const name = buildContainerName({ runId, resources: {} }); + const leasePath = join(root, ARTIFACT_FILE.backendLease); + let foreign = null; + try { + mkdirSync(app, { recursive: true }); + foreign = startContainer(name); + const lease = createBackendLease({ runId, backend: 'spacetime', + track: 'fault-injection', runIndex: 0, serverUri: 'http://127.0.0.1:1', + module: `collision-${process.pid}`, dataDir: join(root, 'data') }); + lease.state = 'active'; + writeBackendLease(leasePath, lease); + + let refused = false; + try { + execFileSync(process.execPath, + [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--prepare-only'], + { stdio: 'pipe', env: { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken } }); + } catch (error: unknown) { + const childError = error instanceof Error && isRecord(error) ? error : null; + refused = childError?.status === 3 + && /refusing to adopt existing unleased container/.test(String(childError?.stderr)); + } + assert.equal(refused, true, 'launcher did not explicitly refuse an unleased same-name container'); + assert.deepEqual(inspectContainer(foreign.id), foreign, + 'collision refusal changed or stopped the foreign container'); + } finally { + removeExactContainer(foreign); + rmSync(root, { recursive: true, force: true }); + } +} + +async function main() { + assert.ok(existsSync(CLI), `local SpacetimeDB CLI is missing: ${CLI}`); + execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' }); + await assertRefusesUnleasedCollision(); + + const root = mkdtempSync(join(tmpdir(), 'stack-bench-fault-')); + const app = join(root, 'app'); + const out = join(root, 'out'); + const markerPath = join(app, '.fault-ready.json'); + const port = await freePort(); + const uri = `http://127.0.0.1:${port}`; + const foreignName = `stack-bench-foreign-${process.pid}-${Date.now()}`; + let foreignContainer: ContainerIdentity | null = null; + let foreignServer: Server | null = null; + let bench: ChildProcess | null = null; + let marker: { lease: { runId: string; state: string; resources: FaultLeaseResources }; leasePath: string; phase: string } | null = null; + let output = ''; + + try { + mkdirSync(app, { recursive: true }); + mkdirSync(out, { recursive: true }); + foreignContainer = startContainer(foreignName); + const startedForeignServer = createServer((_request, response) => response.end('foreign')); + foreignServer = startedForeignServer; + await new Promise((ok, fail) => startedForeignServer.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const foreignAddress = startedForeignServer.address(); + if (!foreignAddress || typeof foreignAddress === 'string') throw new Error('could not allocate foreign TCP port'); + const foreignUri = `http://127.0.0.1:${foreignAddress.port}`; + + bench = spawn(process.execPath, + [compiledEntrypoint('commands', 'bench.js'), '--backend', 'spacetime', '--track', 'loop', + '--levels', '1', '--agent-adapter', 'fault-injection', '--app', app, '--out', out, + '--url', `file:///${app.replace(/\\/g, '/')}/index.html`], + { env: { ...process.env, STACK_BENCH_STDB_URI: uri, STACK_BENCH_IMAGE: IMAGE, + SPACETIME_BIN: CLI }, + stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); + const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-256 * 1024); }; + bench.stdout?.on('data', collect); + bench.stderr?.on('data', collect); + const exited = await waitForExit(bench, 300_000); + assert.notEqual(exited.code, 0, 'injected coding-agent failure unexpectedly exited zero'); + assert.ok(existsSync(markerPath), `fault marker was not written before failure:\n${output}`); + const markerValue: unknown = JSON.parse(readFileSync(markerPath, 'utf8')); + if (!isRecord(markerValue) || !isRecord(markerValue.lease) || !isRecord(markerValue.lease.resources) + || typeof markerValue.leasePath !== 'string' || typeof markerValue.phase !== 'string' + || typeof markerValue.lease.runId !== 'string' || typeof markerValue.lease.state !== 'string') { + throw new Error('fault marker is invalid'); + } + const markerResources = markerValue.lease.resources; + if (!Array.isArray(markerResources.listenerProcesses) || !isRecord(markerResources.buildContainer) + || typeof markerResources.buildContainer.id !== 'string') throw new Error('fault marker resources are invalid'); + marker = { phase: markerValue.phase, leasePath: markerValue.leasePath, + lease: { runId: markerValue.lease.runId, state: markerValue.lease.state, + resources: { listenerProcesses: markerResources.listenerProcesses.filter((item): item is { + pid: number; startMarker: string } => isRecord(item) && typeof item.pid === 'number' + && typeof item.startMarker === 'string'), buildContainer: { + id: markerResources.buildContainer.id, image: String(markerResources.buildContainer.image ?? ''), + running: markerResources.buildContainer.running === true }, locks: [] } } }; + assert.equal(marker.phase, 'restart-stopped', + 'fault was not injected inside the backend restart window'); + assert.equal(marker.lease.state, 'restarting'); + assert.match(marker.lease.resources.buildContainer.image, /^sha256:[0-9a-f]{64}$/, + 'build container lease did not record an immutable image id'); + + const evidencePath = join(out, ARTIFACT_FILE.backendLease); + assert.ok(existsSync(evidencePath), `teardown did not preserve lease evidence:\n${output}`); + const evidence = readArtifactPayload(evidencePath, { expectedKind: 'backend_lease_evidence' }); + const preflight = readArtifact(join(out, ARTIFACT_FILE.preflight), + { expectedKind: 'preflight' }); + assert.equal(preflight.payload.ok, true, 'paid-run preflight did not pass'); + assert.equal(preflight.attempt.parentId, marker.lease.runId, + 'preflight evidence is not attached to the run it admitted'); + assert.equal(evidence.runId, marker.lease.runId); + assert.equal(evidence.state, 'released', 'benchmark lease did not reach its terminal state'); + assert.ok(evidence.stoppedAt, 'benchmark-owned SpacetimeDB host has no stop evidence'); + assert.ok(evidence.releasedAt, 'benchmark lease has no release evidence'); + assert.deepEqual(evidence.resources.listenerProcesses, []); + assert.equal(evidence.resources.buildContainer.running, false, + 'benchmark-owned build container was not marked removed'); + assert.ok(evidence.resources.buildContainer.removedAt); + assert.ok(evidence.resources.locks.every(lock => lock.releasedAt), + 'one or more resource locks were not released'); + assert.equal(inspectContainer(marker.lease.resources.buildContainer.id), null, + 'benchmark-owned build container survived fatal cleanup'); + assert.equal(pidsOnPort(port).length, 0, 'benchmark-owned listener survived fatal cleanup'); + assert.equal(existsSync(marker.leasePath), false, 'private runtime lease was not removed'); + + assert.equal((await fetch(foreignUri)).status, 200, + 'foreign listener was disturbed by benchmark cleanup'); + assert.deepEqual(inspectContainer(foreignContainer.id), foreignContainer, + 'foreign container was changed or removed by benchmark cleanup'); + + console.log(JSON.stringify({ ok: true, injectedAt: 'restart-stopped-before-replacement', + benchmarkHostStopped: true, benchmarkContainerRemoved: true, locksReleased: true, + privateLeaseRemoved: true, foreignListenerSurvived: true, + foreignContainerSurvived: true, unleasedCollisionRefused: true, + immutableImagePinned: true }, null, 2)); + } finally { + if (bench?.exitCode === null) { + killTree(bench.pid); + await delay(500); + } + if (marker?.lease?.resources?.buildContainer) { + removeExactContainer(marker.lease.resources.buildContainer); + } + for (const identity of marker?.lease?.resources?.listenerProcesses ?? []) { + if (pidsOnPort(port).includes(String(identity.pid))) killTree(identity.pid); + } + if (foreignServer) { + const server = foreignServer; + // The verification fetch uses a keep-alive connection. Waiting on + // close() alone can hold CI open until Undici retires that socket. + server.closeAllConnections(); + await new Promise((ok, fail) => server.close(error => error ? fail(error) : ok())); + } + removeExactContainer(foreignContainer); + rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/job-cli.ts b/tools/stack-bench/commands/job-cli.ts new file mode 100644 index 00000000000..e5d0233919f --- /dev/null +++ b/tools/stack-bench/commands/job-cli.ts @@ -0,0 +1,64 @@ +#!/usr/bin/env node +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; +import { cancelExecutionJob, listExecutionJobs, readExecutionJob, + submitExecutionJob, workExecutionJob } from '../src/campaigns/execution-jobs.js'; +import { runExecutionWorker } from '../src/campaigns/execution-worker.js'; +import { prepareRun, runSetupCatalog, submitPreparedRun } from '../src/campaigns/run-setup.js'; + +export async function jobCommand(argv: string[], env: NodeJS.ProcessEnv = process.env) { + const { values, positionals } = parseArgs({ args: argv, allowPositionals: true, options: { + results: { type: 'string' }, host: { type: 'string' }, after: { type: 'string' }, + limit: { type: 'string' }, concurrency: { type: 'string' }, + } }); + const [command, argument] = positionals; + if (argv[0] !== command) throw new Error('put the job command before its options'); + if (positionals.length > 2) throw new Error('unexpected job arguments'); + const results = resolve(values.results ?? env.STACK_BENCH_RESULTS_DIR ?? 'results'); + if (command === 'options' && !argument) return runSetupCatalog(results, env); + if (command === 'prepare' && argument) return prepareRun(results, + JSON.parse(readFileSync(argument === '-' ? 0 : argument, 'utf8')), env); + if (command === 'start' && argument) { + const host = values.host ?? env.STACK_BENCH_HOST_ID; + if (!host) throw new Error('job start requires --host or STACK_BENCH_HOST_ID'); + const job = submitPreparedRun(results, JSON.parse(readFileSync(argument === '-' ? 0 : argument, 'utf8')), env); + console.log(JSON.stringify({ jobId: job.id, campaignKey: `job-${job.id}` })); + return jobCommand(['work', job.id, '--results', results, '--host', host], env); + } + if (command === 'submit' && argument) return submitExecutionJob(results, + JSON.parse(readFileSync(argument === '-' ? 0 : argument, 'utf8'))); + if (command === 'status' && argument) return readExecutionJob(results, argument); + if (command === 'cancel' && argument) { + cancelExecutionJob(results, argument); return readExecutionJob(results, argument); + } + if (command === 'list' && !argument) return listExecutionJobs(results, + { after: values.after, limit: values.limit === undefined ? undefined : Number(values.limit) }); + if ((command === 'work' && argument) || (command === 'worker' && !argument)) { + const host = values.host ?? env.STACK_BENCH_HOST_ID; + if (!host) throw new Error('job work/worker requires --host or STACK_BENCH_HOST_ID'); + const controller = new AbortController(); + const stop = () => controller.abort(); + process.on('SIGTERM', stop); process.on('SIGINT', stop); + try { + if (command === 'worker') { + await runExecutionWorker(results, host, { env, signal: controller.signal, + concurrency: Number(values.concurrency) }); + return { status: 'stopped' as const }; + } + return await workExecutionJob(results, argument!, host, { env, signal: controller.signal }); + } + finally { process.off('SIGTERM', stop); process.off('SIGINT', stop); } + } + throw new Error('use job options, prepare , start --host , submit , list, status , cancel , work --host , or worker --host --concurrency '); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + jobCommand(process.argv.slice(2)).then(result => { + console.log(JSON.stringify(result, null, 2)); + if ('status' in result && result.status === 'failed') process.exitCode = 1; + }).catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; + }); +} diff --git a/tools/stack-bench/commands/leak-audit.ts b/tools/stack-bench/commands/leak-audit.ts new file mode 100644 index 00000000000..a4f4580c74b --- /dev/null +++ b/tools/stack-bench/commands/leak-audit.ts @@ -0,0 +1,325 @@ +#!/usr/bin/env node +// Use the recorded cwd as the app boundary; transcript folder names are not authority. + +import { readdirSync, readFileSync, existsSync } from 'node:fs'; +import { join, posix, resolve } from 'node:path'; +import { homedir } from 'node:os'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; +import { CODING_CONTAINER_APP_ROOT } from '../src/runtime/coding-container-policy.js'; +import { transcriptDirectories } from '../src/agents/transcript-archive.js'; +import { codexTranscriptDirectory } from '../src/agents/codex-protocol.js'; + +const norm = (value: unknown): string => String(value ?? '') + .replace(/\\/g, '/').replace(/^["']|["']$/g, '').toLowerCase(); + +// Ignore dependencies, build output, and this session's CLI task output. +const IGNORE = /node_modules|\.git[/\\]|package-lock\.json|\/dist\/|\.map$|[/\\]temp[/\\]claude[/\\].*[/\\]tasks[/\\]/; + +// Commands that pull file contents into context. +const READER = /(?:^|[;&|]\s*)(?:cat|head|tail|less|more|type|grep|rg|ack|find|ls\s+-\w*l|sed\s+-n|awk)\s+([^;&|]+)/g; + +// Network targets in a shell command: any URL, and a raw socket target. A +// A verified attempt namespace owns its loopback ports, including temporary +// test servers. Shared namespaces require exact endpoints. Internet targets +// are recorded, not judged. +const URL_TARGET = /https?:\/\/([^\s/'"`]+)/gi; +const SOCKET_TARGET = /(?:^|[;&|]\s*)(?:nc|ncat|netcat)\s+(?:-\S+\s+)*([\w.-]+)\s+(\d{2,5})\b/g; +const LOCAL_HOST = /^(?:127\.\d+\.\d+\.\d+|localhost|0\.0\.0\.0|\[::1\]|host\.docker\.internal|10\.\d+\.\d+\.\d+|172\.(?:1[6-9]|2\d|3[01])\.\d+\.\d+|192\.168\.\d+\.\d+)$/i; +const LOOPBACK_HOST = /^(?:127\.\d+\.\d+\.\d+|localhost|0\.0\.0\.0|\[::1\])$/i; + +export interface AuditNetworkContext { + ownEndpoints?: readonly string[]; + isolatedLoopback?: boolean; +} + +export interface NetworkTarget { + host: string; + port: number | null; +} + +export function networkTargetsFromBash(command: unknown): NetworkTarget[] { + const targets: NetworkTarget[] = []; + const text = String(command ?? ''); + for (const match of text.matchAll(URL_TARGET)) { + const authority = (match[1] ?? '').replace(/^[^@]*@/, ''); + const port = authority.match(/:(\d{1,5})$/)?.[1]; + targets.push({ host: authority.replace(/:\d{1,5}$/, ''), port: port ? Number(port) : null }); + } + for (const match of text.matchAll(SOCKET_TARGET)) { + targets.push({ host: match[1] ?? '', port: Number(match[2]) }); + } + return targets; +} + +const endpointHost = (host: string): string => /^(?:127\.0\.0\.1|0\.0\.0\.0|localhost|\[::1\])$/i.test(host) + ? 'localhost' : host.toLowerCase(); +const endpointKey = (target: NetworkTarget): string => `${endpointHost(target.host)}:${target.port}`; + +const networkKind = (target: NetworkTarget, ownEndpoints: ReadonlySet, isolatedLoopback: boolean): string | null => { + if (isolatedLoopback && LOOPBACK_HOST.test(target.host)) return null; + if (ownEndpoints.has(endpointKey(target))) return null; + if (!LOCAL_HOST.test(target.host)) return 'network (internet)'; + return 'NETWORK / OTHER RUN'; +}; + +const CLASSES: Array = [ + [/[/\\]stack-bench(?:[/\\]|$)/, 'GRADER / TEST SPECS'], + [/\.claude[/\\]projects.*memory|[/\\]memory[/\\].*\.md$/, 'BENCHMARK NOTES'], + [/scenarios[/\\].*\.json|grade\.(?:js|ts)|mutation|check-scenarios/, 'GRADER / TEST SPECS'], + [/contracts[/\\].*\.json|appendix-\d+\.md|walk\.(?:js|ts)|lint\.(?:js|ts)/, 'CONTRACT / LINTER'], + [/prompts[/\\]|test-plans[/\\]|GRADING|RUBRIC/, 'PROMPTS / RUBRIC'], + [/[/\\]skills[/\\]/, 'skill docs (intended)'], + [/backends[/\\].*\.md|CLAUDE\.md|README/, 'setup docs (intended)'], +]; +const classify = (path: string): string => CLASSES.find(([pattern]) => pattern.test(path))?.[1] + ?? 'other'; + +// The shallowest recorded cwd is the app boundary when --app is absent. +function sessionCwd(lines: string[]): string | null { + const seen = new Set(); + for (const l of lines) { + const m = l.match(/"cwd":"((?:[^"\\]|\\.)*)"/); + if (m?.[1]) seen.add(norm(m[1].replace(/\\\\/g, '/'))); + } + if (!seen.size) return null; + return [...seen].sort((a, b) => a.split('/').length - b.split('/').length || a.length - b.length)[0] + ?? null; +} + +export function pathsFromBash(command: unknown): string[] { + const out: string[] = []; + for (const match of String(command).matchAll(READER)) { + const argumentsText = match[1]; + if (!argumentsText) continue; + for (const tokRaw of argumentsText.split(/\s+/)) { + const t = tokRaw.replace(/^["']|["']$/g, ''); + if (!t || t.startsWith('-')) continue; + if (/[*?]/.test(t) || /\//.test(t) || /\\/.test(t) || /\.\w+$/.test(t)) out.push(t); + } + } + return out; +} + +// Count file-tool reads only after their result confirms success. +// Bash reads count unless the command fails. +interface AuditHit { + path: string; + via: string; + kind: string; + unresolved?: boolean; +} + +interface PendingRead { + paths: string[]; + network: Array<{ path: string; kind: string }>; + via: string; +} + +interface TranscriptAudit { + file: string; + cwd: string | null; + fileTool: number; + bashReads: number; + hits: AuditHit[]; + refused: AuditHit[]; +} + +interface AuditResult extends TranscriptAudit { + root: string; +} + +interface TranscriptContent { + type?: string; + name?: string; + id?: string; + tool_use_id?: string; + is_error?: boolean; + input?: { file_path?: string; path?: string; pattern?: string; command?: string }; +} + +// Feed both CLIs through the same path, network, and refusal checks. +function transcriptContent(event: Record): TranscriptContent[] { + const message = event.message as { content?: TranscriptContent[] } | undefined; + if (Array.isArray(message?.content)) return message.content; + if (event.type !== 'item.started' && event.type !== 'item.completed') return []; + const item = event.item as { id?: string; type?: string; command?: string; + aggregated_output?: string; exit_code?: number; status?: string; + changes?: { path: string }[] } | undefined; + if (!item?.id) return []; + const content: TranscriptContent[] = []; + if (item.type === 'command_execution') { + // Codex records the shell launcher around the command. + const command = (item.command ?? '').replace(/^(?:\/\S+\/)?(?:bash|sh|zsh)\s+-[a-z]*c\s+(['"])([\s\S]*)\1$/, '$2'); + content.push({ type: 'tool_use', id: item.id, name: 'Bash', input: { command } }); + } else if (item.type === 'file_change') { + for (const [index, change] of (item.changes ?? []).entries()) { + content.push({ type: 'tool_use', id: `${item.id}:${index}`, name: 'Edit', + input: { file_path: change.path } }); + } + } + if (event.type === 'item.completed') { + // Output can contain a successful read before a later command fails. + const blocked = item.status === 'failed' && !item.aggregated_output?.trim(); + for (const call of [...content]) content.push({ type: 'tool_result', + tool_use_id: call.id, is_error: blocked }); + } + return content; +} + +export function auditTranscript(file: string, boundary: string | null, + { ownEndpoints = [], isolatedLoopback = false }: AuditNetworkContext = {}): TranscriptAudit { + const endpoints = new Set(ownEndpoints.map(endpoint => { + const url = new URL(`http://${endpoint}`); + return endpointKey({ host: url.hostname, port: Number(url.port || 80) }); + })); + const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean); + // Container transcripts use /app; --app is its host path. + const recorded = sessionCwd(lines); + const cwd = recorded === CODING_CONTAINER_APP_ROOT ? recorded : (boundary ?? recorded); + const hits: AuditHit[] = []; + const refused: AuditHit[] = []; + const pending = new Map(); + let fileTool = 0, bashReads = 0; + + for (const line of lines) { + let event: Record; + try { event = JSON.parse(line) as typeof event; } catch { continue; } + const c = transcriptContent(event); + for (const p of c) { + if (p.type === 'tool_result' && p.tool_use_id && pending.has(p.tool_use_id)) { + const completed = pending.get(p.tool_use_id ?? ''); + if (!completed) continue; + const { paths, network, via } = completed; + pending.delete(p.tool_use_id ?? ''); + const blocked = p.is_error === true; + for (const n of paths) (blocked ? refused : hits).push({ path: n, via, kind: classify(n) }); + for (const target of network) (blocked ? refused : hits).push({ ...target, via: `${via} network attempt` }); + continue; + } + if (p.type !== 'tool_use') continue; + const cand = []; + if (/^(Read|Grep|Glob|NotebookRead|Edit)$/.test(p.name ?? '')) { + fileTool++; + cand.push(p.input?.file_path ?? p.input?.path ?? p.input?.pattern ?? ''); + } else if (p.name === 'Bash') { + const found = pathsFromBash(p.input?.command ?? ''); + bashReads += found.length; + cand.push(...found); + } + const paths = []; + const network: Array<{ path: string; kind: string }> = []; + if (p.name === 'Bash') { + for (const target of networkTargetsFromBash(p.input?.command ?? '')) { + const kind = networkKind(target, endpoints, isolatedLoopback); + if (kind) network.push({ path: `${target.host}${target.port === null ? '' : `:${target.port}`}`, kind }); + } + } + for (const raw of cand) { + let n = norm(raw); + if (!n || IGNORE.test(n)) continue; + // The CLI keeps auto-memory for the session's OWN project dir. A build + // A session may read its own memory, never another project's memory. + if (cwd && /[/\\]projects[/\\][^/\\]+[/\\]memory[/\\]/.test(n) + && n.includes(cwd.replace(/[\\/:]/g, '-'))) continue; + const absolute = /^[a-z]:/.test(n) || n.startsWith('/'); + if (!absolute && cwd) n = `${cwd}/${n.replace(/^\.\//, '')}`; + n = posix.normalize(n); + const privateHarnessPath = cwd + && (n === `${cwd}/stack-bench` || n.startsWith(`${cwd}/stack-bench/`)); + if (!privateHarnessPath && !absolute && !cwd) continue; + if (!privateHarnessPath && cwd && (n === cwd || n.startsWith(`${cwd}/`))) continue; + paths.push(n); + } + if ((paths.length || network.length) && p.id && p.name) { + pending.set(p.id, { paths, network, via: p.name }); + } + } + } + // A call whose result never arrived (session cut short) is unresolved, and + // unresolved is not innocent: count it. + for (const { paths, network, via } of pending.values()) { + for (const n of paths) hits.push({ path: n, via, kind: classify(n), unresolved: true }); + for (const target of network) hits.push({ ...target, via: `${via} network attempt`, unresolved: true }); + } + + return { file, cwd, fileTool, bashReads, hits, refused }; +} + +function main(): void { + const { values } = parseArgs({ args: process.argv.slice(2), options: { + app: { type: 'string' }, dir: { type: 'string' }, json: { type: 'boolean' }, + 'own-endpoints': { type: 'string' }, + 'isolated-loopback': { type: 'boolean' }, + } }); + const ownEndpoints = (values['own-endpoints'] ?? '').split(',').filter(Boolean); + const requestedApp = values.app; + const requestedDirectory = values.dir; + if (requestedApp && requestedDirectory) throw new Error('--app and --dir cannot be used together'); + const roots = requestedApp ? transcriptDirectories(requestedApp) + : requestedDirectory ? [resolve(requestedDirectory)] + : [join(homedir(), '.claude', 'projects')]; + // When the caller names the app directory, that is the boundary. Do not + // infer it from a transcript folder name. + const appBoundary = requestedApp ? norm(resolve(requestedApp)) : null; + const results: AuditResult[] = []; +for (const root of roots) { + if (!existsSync(root)) continue; + const stack = [root]; + while (stack.length) { + const d = stack.pop(); + if (!d) continue; + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = join(d, e.name); + if (e.isDirectory()) { + // Codex native rollouts are agent-writable; audit controller event logs only. + if (!/node_modules/.test(p) + && !(requestedApp && root === codexTranscriptDirectory(requestedApp))) stack.push(p); + continue; + } + if (!/\.jsonl$/.test(e.name)) continue; + // Include transcripts from the main session and its subagents. + if (!/transcript|^agent-|^[0-9a-f-]{36}\.jsonl$|\.events\.jsonl$/.test(e.name)) continue; + results.push({ ...auditTranscript(p, appBoundary, { ownEndpoints, + isolatedLoopback: values['isolated-loopback'] === true }), root }); + } + } +} + +if (values.json) { + console.log(JSON.stringify(results, null, 2)); + return; +} + +const label = (file: string): string => file.replace(/\\/g, '/') + .split('/').slice(-3).join('/').slice(0, 62); +console.log('\nBuilds that read outside their own directory'); +console.log('(counts BOTH file tools and Bash cat/grep/find; boundary = the session\'s own cwd)\n'); + +let clean = 0; +for (const r of results.sort((a, b) => b.hits.length - a.hits.length)) { + if (!r.cwd) { console.log(` ?? ${label(r.file)} — no cwd recorded, cannot judge`); continue; } + if (!r.hits.length) { + clean++; + // Blocked attempts are worth printing: they are the sandbox doing its job, + // and they say which paths a build still goes looking for. + if (r.refused?.length) { + const kinds = [...new Set(r.refused.map(h => h.kind))].join(', '); + console.log(` ${label(r.file)}\n clean — ${r.refused.length} attempt(s) BLOCKED by the sandbox (${kinds})`); + } + continue; + } + const byKind: Record = {}; + for (const h of r.hits) (byKind[h.kind] ??= []).push(h.path); + console.log(` ${label(r.file)}`); + console.log(` cwd: ...${r.cwd.slice(-52)} (${r.fileTool} file-tool, ${r.bashReads} bash reads)`); + for (const [k, v] of Object.entries(byKind).sort((a, b) => b[1].length - a[1].length)) { + const example = [...new Set(v)][0] ?? ''; + console.log(` ${String(v.length).padStart(3)}x ${k.padEnd(22)} ${example.split('/').slice(-2).join('/')}`); + } +} +console.log(`\n ${clean} transcript(s) read nothing outside their directory.`); +console.log(` ${results.length} transcript(s) examined.\n`); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/null-control.ts b/tools/stack-bench/commands/null-control.ts new file mode 100644 index 00000000000..c0531a20033 --- /dev/null +++ b/tools/stack-bench/commands/null-control.ts @@ -0,0 +1,287 @@ +#!/usr/bin/env node +// Grade the real validated production scenarios against a reachable app that +// implements nothing. Every point-bearing criterion must conclusively fail. + +import { execFile } from 'node:child_process'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, relative, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { chromium, type BrowserServer } from 'playwright'; +import { readArtifactPayload, writeRunJson } from '../src/evidence/artifacts.js'; +import { calibrationQualificationIdentity, calibrationQualificationRelease, + resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { qualificationScopeIdentity } from '../src/composition/qualification-scope.js'; +import { writeQualificationSnapshot } from '../src/composition/qualification-slices.js'; +import { analyseNullReports } from '../src/evidence/null-control-analysis.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveRecipeSelection } from '../src/composition/recipe-selection.js'; +import { isDeclaredLevel, listTracks, loadTrack, suitesFor } from '../src/composition/tracks.js'; +import { controllerRunner } from '../src/runtime/runner-environment.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import type { RecipeBinding } from '../src/composition/recipe-release.js'; +import type { Track } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const GRADE = compiledEntrypoint('grader', 'grade.js'); +const NULL_CONTROL_WORKERS = 4; + +interface NullControlArgs { + tracks: string[]; + level: number | null; + recipe?: string; + out?: string; + audit: boolean; + parentAttemptId?: string; + selectedChecks: string[]; +} + +export function parseNullControlArgs(argv: string[]): NullControlArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + track: { type: 'string' }, level: { type: 'string' }, recipe: { type: 'string' }, + out: { type: 'string' }, audit: { type: 'boolean' }, 'parent-attempt-id': { type: 'string' }, + 'selected-check': { type: 'string', multiple: true }, + } }); + const args: NullControlArgs = { + tracks: values.track?.split(',').filter(Boolean) ?? listTracks(), + level: values.level === undefined ? null : Number(values.level), audit: values.audit ?? false, + recipe: values.recipe, out: values.out, parentAttemptId: values['parent-attempt-id'], + selectedChecks: values['selected-check'] ?? [], + }; + if (args.level !== null && (!Number.isInteger(args.level) || args.level < 1)) { + throw new Error('--level must be a positive integer'); + } + if (args.level !== null && args.tracks.length !== 1) { + throw new Error('--level requires exactly one --track'); + } + if (args.recipe && args.level === null) throw new Error('--recipe requires --level'); + if (args.selectedChecks.length && args.level === null) throw new Error('--selected-check requires --level'); + return args; +} + +function runGrade(argv: string[], timeoutMs = 300_000): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + execFile(process.execPath, [GRADE, ...argv], { + encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: timeoutMs, + }, (error, stdout, stderr) => { + if (error) { + error.message = `grader failed: ${error.message}\n${stdout}\n${stderr}`; + reject(error); + } else resolve({ stdout, stderr }); + }); + }); +} + +export function nullControlSuites(track: Track, selectedLevel: number | null = null, + binding: RecipeBinding | null = null) { + if (binding) { + if (selectedLevel === null) throw new Error('recipe-bound null control requires one level'); + if (!Array.isArray(binding.execution) || !binding.execution.length) { + throw new Error('recipe-bound null control requires a typed execution plan'); + } + const executionIds = new Set(); + const mappedKeys = new Set(); + const suites = binding.execution.map(execution => { + if (executionIds.has(execution.id)) { + throw new Error(`recipe-bound null control repeats execution ${execution.id}`); + } + executionIds.add(execution.id); + const checks = binding.release.checkCatalog.filter(check => check.executionId === execution.id); + if (!checks.length) { + throw new Error(`recipe-bound null control execution ${execution.id} maps no checks`); + } + for (const check of checks) { + if (mappedKeys.has(check.stableKey)) { + throw new Error(`recipe-bound null control maps check ${check.stableKey} more than once`); + } + mappedKeys.add(check.stableKey); + } + return { id: execution.id, spec: resolve(track.dir, execution.source ?? ''), + level: selectedLevel, checks }; + }); + const missing = binding.release.checkCatalog + .filter(check => !mappedKeys.has(check.stableKey)).map(check => check.stableKey); + if (missing.length) { + throw new Error(`recipe-bound null control leaves checks unmapped: ${missing.join(', ')}`); + } + return suites; + } + if (selectedLevel !== null && !isDeclaredLevel(track, selectedLevel)) { + throw new Error(`L${selectedLevel} is not declared for ${track.name}`); + } + const seen = new Set(); + const suites = []; + const levels = selectedLevel === null + ? Array.from({ length: track.validatedThrough }, (_, index) => index + 1) + : [selectedLevel]; + for (const level of levels) { + for (const suite of suitesFor(track, level)) { + if (seen.has(suite.spec)) continue; + seen.add(suite.spec); + suites.push({ ...suite, level }); + } + } + return suites; +} + +export function selectNullQualificationBinding(binding: RecipeBinding, calibration: CalibrationPlan): RecipeBinding { + const selected = calibrationQualificationRelease(calibration, binding.release, binding.execution); + return { ...binding, release: selected.release, execution: selected.execution }; +} + +export function createNullQualification(binding: RecipeBinding, calibration: CalibrationPlan, + selectedChecks: string[] = []) { + let selectedBinding = selectNullQualificationBinding(binding, calibration); + if (selectedChecks.length) { + const selected = calibrationQualificationRelease({ qualification: { checks: selectedChecks } }, + selectedBinding.release, selectedBinding.execution); + selectedBinding = { ...selectedBinding, ...selected }; + } + const selection = resolveRecipeSelection(selectedBinding.release, { + checkKeys: selectedBinding.release.checkCatalog.map(check => check.stableKey), + }); + return { + binding: selectedBinding, + calibration, + identity: calibrationQualificationIdentity(calibration), + selectionSha256: selection.sha256, + }; +} + +async function listen(server: Server): Promise { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen({ port: 0, host: '127.0.0.1' }, () => resolve()); + }); + return (server.address() as AddressInfo).port; +} + +async function main() { + const args = parseNullControlArgs(process.argv); + const nullAttemptId = `null-control-${new Date().toISOString().replace(/[:.]/g, '-')}`; + const work = mkdtempSync(join(tmpdir(), 'stack-bench-null-')); + const app = join(work, 'app'); + const reportsDir = join(work, 'reports'); + mkdirSync(app, { recursive: true }); + mkdirSync(reportsDir, { recursive: true }); + + // Root navigation succeeds, proving the browser and server are healthy. All + // application/API behavior is absent: non-navigation requests get 404. + const server = createServer((request, response) => { + if (request.method === 'GET' && (request.url === '/' || request.headers.accept?.includes('text/html'))) { + response.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + response.end('Null control'); + } else { + response.writeHead(404, { 'content-type': 'application/json' }); + response.end('{"error":"not implemented"}'); + } + }); + + const started = Date.now(); + const suiteReports = []; + let qualification: ReturnType | null = null; + let browserServer: BrowserServer | undefined; + try { + const port = await listen(server); + const url = `http://127.0.0.1:${port}`; + // This command owns the empty page; it has no generated app or attempt lease. + browserServer = await chromium.launchServer({ headless: true, host: '127.0.0.1' }); + const browserEndpoint = browserServer.wsEndpoint(); + for (const trackName of args.tracks) { + const track = loadTrack(trackName); + let binding: RecipeBinding | null = null; + let recipeTask: ReturnType['request'] | null = null; + if (args.level !== null) { + binding = resolveRecipeRelease(track, args.level, args.recipe); + if (!binding) throw new Error(`${trackName} L${args.level} has no recipe release`); + recipeTask = createBoundRecipeTaskRequest(binding, { + taskMode: binding.plan.recipe.task.mode === 'action' ? 'fresh' : undefined, + }).request; + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, stackBenchRoot: ROOT, alias: `L${args.level}` }); + if (!calibration) throw new Error(`${trackName} L${args.level} has no calibration`); + qualification = createNullQualification(binding, calibration, args.selectedChecks); + binding = qualification.binding; + } + const selectedSuites = nullControlSuites(track, args.level, binding); + const resolvedRecipe = binding?.release.id ?? args.recipe; + for (let index = 0; index < selectedSuites.length; index += NULL_CONTROL_WORKERS) { + const reports = await Promise.all(selectedSuites + .slice(index, index + NULL_CONTROL_WORKERS).map(async suite => { + const reportPath = join(reportsDir, + `${trackName}-l${suite.level}-${suite.id.replaceAll('@', '-')}.json`); + console.log(`${trackName} L${suite.level} ${suite.id} (${basename(suite.spec)})`); + await runGrade(['--url', url, '--level', String(suite.level), '--spec', suite.spec, + '--backend', 'postgres', '--track', trackName, '--app', app, '--out', reportPath, + '--null-control', + '--browser-ws-endpoint', browserEndpoint, + '--parent-attempt-id', nullAttemptId, + ...(resolvedRecipe ? ['--recipe', resolvedRecipe] : []), + ...(binding ? ['--expected-recipe-sha256', binding.release.contentSha256] : []), + ...(recipeTask ? ['--recipe-task-json', JSON.stringify(recipeTask)] : []), + ...(qualification ? ['--selection-sha256', qualification.selectionSha256] : []), + ...(('checks' in suite ? suite.checks : []) ?? []) + .flatMap(check => ['--selected-check', check.stableKey])]); + const report = readArtifactPayload(reportPath, { expectedKind: 'grade' }); + console.log(`${suite.id}: ${report.total}/${report.max}`); + return { track: trackName, level: suite.level, id: suite.id, + scenario: relative(track.dir, suite.spec).replaceAll('\\', '/'), report }; + })); + suiteReports.push(...reports); + } + } + + const analysis = analyseNullReports(suiteReports); + const artifact = { + id: nullAttemptId, + kind: 'null_control', + startedAt: new Date(started).toISOString(), + completedAt: new Date().toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: qualification ? { + recipe: { id: qualification.binding.release.id, + sha256: qualification.binding.release.contentSha256 }, + calibration: { id: qualification.identity.id, + sha256: qualification.identity.contentSha256 }, + } : undefined, + durationMs: Date.now() - started, + runner: controllerRunner(), + ...(qualification ? { qualificationScope: qualificationScopeIdentity({ + kind: 'null', release: qualification.binding.release, stackBenchRoot: ROOT, + }) } : {}), + tracks: args.tracks, + ...analysis, + }; + const outputPath = resolve(args.out ?? join(ROOT, 'results', `${artifact.id}.json`)); + writeRunJson(outputPath, artifact); + if (qualification) writeQualificationSnapshot(`${outputPath}.inputs.json`, + qualification.binding.recipePath, qualification.calibration, ROOT); + console.log(JSON.stringify({ + id: artifact.id, + kind: artifact.kind, + durationMs: artifact.durationMs, + tracks: artifact.tracks, + ok: artifact.ok, + summary: artifact.summary, + artifact: outputPath, + }, null, 2)); + if (!analysis.ok && !args.audit) process.exitCode = 1; + } finally { + try { await browserServer?.close(); } + finally { + await new Promise(resolve => server.close(resolve)); + rmSync(work, { recursive: true, force: true }); + } + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/pack-budget.ts b/tools/stack-bench/commands/pack-budget.ts new file mode 100644 index 00000000000..51c3ffe8b12 --- /dev/null +++ b/tools/stack-bench/commands/pack-budget.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env node + +import { existsSync } from 'node:fs'; +import { dirname, relative, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { artifactPayload, recipeArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { loadPackBudgetEvidence, PACK_BUDGET_POLICY, recommendPackBudgets } + from '../src/composition/pack-budget.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { isDeclaredLevel, listTracks, loadTrack } from '../src/composition/tracks.js'; + +interface PackBudgetArgs { + command: 'recommend'; + track: string; + level: number; + evidence: string[]; + out: string; + recipe?: string; +} + +const USAGE = 'usage: pack-budget.js recommend --track --level ' + + '[--recipe ] --evidence [--evidence ...] ' + + '--out '; + +export function parsePackBudgetArgs(argv: string[]): PackBudgetArgs { + const [command, ...options] = argv.slice(2); + const { values } = parseArgs({ args: options, options: { + track: { type: 'string' }, + level: { type: 'string' }, + recipe: { type: 'string' }, + evidence: { type: 'string', multiple: true }, + out: { type: 'string' }, + } }); + const level = Number(values.level); + const evidence = (values.evidence ?? []).map(path => resolve(path)); + if (command !== 'recommend' || !values.track || !Number.isInteger(level) || level < 1 + || !evidence.length || !values.out) throw new Error(USAGE); + if (new Set(evidence).size !== evidence.length) throw new Error('--evidence paths must be unique'); + return { command, track: values.track, level, evidence, out: resolve(values.out), + ...(values.recipe ? { recipe: values.recipe } : {}) }; +} + +function main(): void { + const args = parsePackBudgetArgs(process.argv); + if (!listTracks().includes(args.track)) throw new Error(`unknown track ${args.track}`); + const track = loadTrack(args.track); + if (!isDeclaredLevel(track, args.level)) throw new Error(`L${args.level} is not declared for ${args.track}`); + const binding = resolveRecipeRelease(track, args.level, args.recipe); + if (!binding) throw new Error(`${args.track} L${args.level} has no recipe release`); + const calibration = resolveCalibrationForRelease(binding.release, { trackRoot: track.dir, alias: `L${args.level}` }); + if (!calibration) throw new Error(`${binding.release.id} has no calibration`); + const loaded = loadPackBudgetEvidence(args.evidence); + const result = recommendPackBudgets({ binding, calibration, evidence: loaded }); + if (existsSync(args.out)) throw new Error(`refusing to replace existing budget measurement: ${args.out}`); + const id = `pack-budget-${args.track}-l${args.level}-${new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}`; + const artifact = writeArtifact(args.out, { kind: 'pack_budget_measurement', id, + identities: recipeArtifactIdentities(binding.release, { + calibration: { id: calibration.id, sha256: calibration.contentSha256 }, + }), + payload: { schemaVersion: 1, track: args.track, level: args.level, policy: PACK_BUDGET_POLICY, + runner: result.measuredRunner, + evidence: loaded.map(item => { + const stackAdapter = item.artifact.identities.stackAdapter; + if (!stackAdapter) throw new Error(`${item.path} has no stack adapter identity`); + return { path: relative(dirname(args.out), item.path).replaceAll('\\', '/'), + sha256: item.sha256, stack: stackAdapter.id }; + }), + samples: result.samples, recommendations: result.recommendations } }); + console.log(JSON.stringify(artifactPayload(artifact), null, 2)); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} diff --git a/tools/stack-bench/commands/preflight-cli.ts b/tools/stack-bench/commands/preflight-cli.ts new file mode 100644 index 00000000000..066389d0363 --- /dev/null +++ b/tools/stack-bench/commands/preflight-cli.ts @@ -0,0 +1,89 @@ +import { resolve } from 'node:path'; +import { parseArgs } from 'node:util'; + +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import type { PreflightReport, PreflightRequest } from '../src/runtime/preflight.js'; + +function splitList(value: unknown): string[] { + return String(value).split(',').map(item => item.trim()).filter(Boolean); +} + +export function parsePreflightArgs( + argv: string[], + { env = process.env }: { env?: NodeJS.ProcessEnv } = {}, +): PreflightRequest { + const { values } = parseArgs({ args: argv.slice(2), options: { + backend: { type: 'string', multiple: true }, + track: { type: 'string' }, + levels: { type: 'string' }, + recipe: { type: 'string' }, + 'run-index': { type: 'string' }, + parallelism: { type: 'string' }, + 'agent-adapter': { type: 'string' }, + 'provider-route': { type: 'string' }, + 'max-output-tokens': { type: 'string' }, + guidance: { type: 'string' }, + pack: { type: 'string', multiple: true }, + check: { type: 'string', multiple: true }, + image: { type: 'string' }, + 'results-dir': { type: 'string' }, + report: { type: 'string' }, + smoke: { type: 'boolean' }, + json: { type: 'boolean' }, + } }); + const request: PreflightRequest = { backends: [], track: 'ecommerce', levels: '1', levelList: [], + runIndex: 0, parallelism: 1, + agentAdapter: 'claude-code', guidance: 'prescribed', packIds: [], checkKeys: [], smoke: false, + image: env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, + resultsDir: stackBenchResultsRoot(STACK_BENCH_ROOT, env) }; + request.backends = (values.backend ?? []).flatMap(splitList); + if (values.track !== undefined) request.track = values.track; + if (values.levels !== undefined) request.levels = values.levels; + if (values.recipe !== undefined) request.recipe = values.recipe; + if (values['run-index'] !== undefined) request.runIndex = Number(values['run-index']); + if (values.parallelism !== undefined) request.parallelism = Number(values.parallelism); + if (values['agent-adapter'] !== undefined) request.agentAdapter = values['agent-adapter']; + if (values['provider-route'] !== undefined) request.providerRoute = values['provider-route']; + if (values['max-output-tokens'] !== undefined) request.maxOutputTokens = Number(values['max-output-tokens']); + if (values.guidance !== undefined) request.guidance = values.guidance; + request.packIds = (values.pack ?? []).flatMap(splitList); + request.checkKeys = (values.check ?? []).flatMap(splitList); + if (values.image !== undefined) request.image = values.image; + if (values['results-dir'] !== undefined) request.resultsDir = resolve(values['results-dir']); + if (values.report !== undefined) request.report = resolve(values.report); + request.smoke = values.smoke ?? false; + request.json = values.json; + if (!request.backends.length) throw new Error('--backend is required (comma-separated values are accepted)'); + if (request.guidance !== 'neutral' && request.guidance !== 'prescribed') { + throw new Error('--guidance must be neutral or prescribed'); + } + request.backends = [...new Set(request.backends)].sort(); + if (!Number.isInteger(request.runIndex) || request.runIndex < 0) { + throw new Error('--run-index must be a non-negative integer'); + } + if (!Number.isInteger(request.parallelism) || (request.parallelism ?? 0) < 1) { + throw new Error('--parallelism must be a positive integer'); + } + const match = String(request.levels).match(/^(\d+)(?:-(\d+))?$/); + if (!match || Number(match[2] ?? match[1]) < Number(match[1])) { + throw new Error('--levels must be N or N-M'); + } + request.levelList = Array.from({ length: Number(match[2] ?? match[1]) - Number(match[1]) + 1 }, + (_, index) => Number(match[1]) + index); + if (request.recipe && request.levelList.length !== 1) { + throw new Error('--recipe requires exactly one requested level'); + } + return request; +} + +export function printPreflightReport(report: PreflightReport): void { + console.log(`Stack Bench preflight: ${report.ok ? 'READY' : 'NOT READY'}`); + for (const check of report.checks) { + const mark = check.status === 'pass' ? 'PASS' : check.status === 'warn' ? 'WARN' : 'FAIL'; + console.log(` ${mark.padEnd(4)} ${check.id.padEnd(28)} ${check.summary}`); + if (check.remediation && check.status === 'fail') console.log(` hint: ${check.remediation}`); + } + console.log(`\n${report.summary.passed} passed, ${report.summary.failed} failed, ${report.summary.warnings} warnings`); +} diff --git a/tools/stack-bench/commands/preflight.ts b/tools/stack-bench/commands/preflight.ts new file mode 100644 index 00000000000..45cb7a40af1 --- /dev/null +++ b/tools/stack-bench/commands/preflight.ts @@ -0,0 +1,19 @@ +#!/usr/bin/env node + +import { parsePreflightArgs, printPreflightReport } from './preflight-cli.js'; +import { runPreflight, writePreflightReport } from '../src/runtime/preflight.js'; + +let request; +try { + request = parsePreflightArgs(process.argv); +} catch (error) { + console.error(`preflight: ${error instanceof Error ? error.message : String(error)}`); + console.error('Usage: stack-bench preflight --backend spacetime[,postgres,mongodb] [--track ecommerce] [--levels 1-2] [--smoke]'); + process.exit(2); +} + +const report = runPreflight(request); +if (request.report) writePreflightReport(request.report, report); +if (request.json) console.log(JSON.stringify(report, null, 2)); +else printPreflightReport(report); +process.exitCode = report.ok ? 0 : 1; diff --git a/tools/stack-bench/commands/progression-graph.ts b/tools/stack-bench/commands/progression-graph.ts new file mode 100644 index 00000000000..82dc0a3be9e --- /dev/null +++ b/tools/stack-bench/commands/progression-graph.ts @@ -0,0 +1,21 @@ +import { dirname, join, resolve } from 'node:path'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { writeProgressionGraph } from '../src/progression/progression-graph.js'; + +interface ProgressionGraph { + nodes: unknown[]; + levels: number; +} + +const definitionPath = process.argv[2]; +if (!definitionPath) { + throw new Error('usage: progression-graph [html-path]'); +} +const resolvedDefinitionPath = resolve(definitionPath); +const graph: ProgressionGraph = writeProgressionGraph({ + definitionPath: resolvedDefinitionPath, + htmlPath: process.argv[3] ?? join(STACK_BENCH_ROOT, 'docs', 'dependency-graph.html'), + trackRoot: dirname(dirname(resolvedDefinitionPath)), +}); +console.log(`Rendered ${graph.nodes.length} nodes across ${graph.levels} levels.`); diff --git a/tools/stack-bench/commands/qualification-cli.ts b/tools/stack-bench/commands/qualification-cli.ts new file mode 100644 index 00000000000..97bd1e37909 --- /dev/null +++ b/tools/stack-bench/commands/qualification-cli.ts @@ -0,0 +1,253 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from 'node:fs'; +import { basename, dirname, extname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { calibrationQualificationIdentity, resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { isDeclaredLevel, listTracks, loadTrack } from '../src/composition/tracks.js'; +import { PACK_BUDGET_POLICY } from '../src/composition/pack-budget.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import { companionReferenceArtifactPath, MAX_MUTATION_WORKERS, validateMutationWorkerCount } + from '../src/references/reference-live.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import type { RecipeBinding, RecipeRelease } from '../src/composition/recipe-release.js'; + +interface QualificationArgs { + command?: string; + track: string | null; + level: number | null; + recipe?: string; + mutationWorkers?: number; +} + +interface QualificationBlocker { + code: string; + path: string; + summary: string; +} + +export function parseQualificationArgs(argv: string[]): QualificationArgs { + const { positionals, values } = parseNodeArgs({ args: argv.slice(2), allowPositionals: true, + options: { track: { type: 'string' }, level: { type: 'string' }, recipe: { type: 'string' }, + 'mutation-workers': { type: 'string' } } }); + const args: QualificationArgs = { command: positionals[0], track: values.track ?? null, + level: values.level === undefined ? null : Number(values.level), + ...(values.recipe === undefined ? {} : { recipe: values.recipe }), + ...(values['mutation-workers'] === undefined ? {} : { + mutationWorkers: validateMutationWorkerCount(Number(values['mutation-workers'])) }) }; + if (args.command !== 'status' || typeof args.track !== 'string' || !args.track + || positionals.length !== 1 || args.level === null || !Number.isInteger(args.level) || args.level < 1) { + throw new Error('usage: node dist/commands/qualification-cli.js status --track --level ' + + '[--recipe ] [--mutation-workers ]'); + } + return args; +} + +function blocker(code: string, path: string, summary: string): QualificationBlocker { + return { code, path, summary }; +} + +function evidencePlan(calibration: CalibrationPlan) { + const stacks = [...calibration.qualification.stacks].sort(); + const evidence = []; + for (const stack of stacks) { + for (let repetition = 1; repetition <= calibration.qualification.referenceRepetitions; repetition += 1) { + evidence.push({ kind: 'reference', stack, repetition }); + } + for (let repetition = 1; repetition <= calibration.qualification.mutationRepetitions; repetition += 1) { + evidence.push({ kind: 'mutation', stack, repetition }); + } + } + for (let repetition = 1; repetition <= calibration.nullControl.repetitions; repetition += 1) { + evidence.push({ kind: 'null', stack: null, repetition }); + } + return evidence; +} + +export interface CalibrationMutationSelection { + mutations: Array<{ backend: string; path: string; targets: Array<{ id: string }> }>; +} + +export function mutationWorkerCount(calibration: CalibrationMutationSelection, stack: string, + readManifest: (path: string) => { mutations?: { id: string }[] } = path => + JSON.parse(readFileSync(resolve(STACK_BENCH_ROOT, path), 'utf8')) as { mutations?: { id: string }[] }, + requestedWorkers = MAX_MUTATION_WORKERS) { + validateMutationWorkerCount(requestedWorkers); + const entry = calibration.mutations.find(candidate => candidate.backend === stack); + if (!entry) return 1; + const manifest = readManifest(entry.path); + const selectedIds = new Set(entry.targets.map(target => target.id)); + const selectedMutations = (manifest.mutations ?? []).filter(mutation => + selectedIds.delete(mutation.id)); + if (selectedIds.size) { + throw new Error(`${stack} calibration selects missing mutations: ${[...selectedIds].sort().join(', ')}`); + } + return Math.min(requestedWorkers, Math.max(1, selectedMutations.length)); +} + +function mutationWorkerOption(calibration: CalibrationPlan, stack: string, requestedWorkers: number) { + const workers = mutationWorkerCount(calibration, stack, undefined, requestedWorkers); + return workers > 1 ? ` --mutation-workers ${workers}` : ''; +} + +function qualificationRunDirectory(artifactPath: string): string { + return join(dirname(artifactPath), `${basename(artifactPath, extname(artifactPath))}.runs`); +} + +function defectCheckCoverage(release: RecipeRelease, calibration: CalibrationPlan) { + const selected = calibration.qualification.checks + ? new Set(calibration.qualification.checks) : null; + const scored = release.checkCatalog.filter(check => check.points > 0 + && (selected === null || selected.has(check.stableKey))); + const scoredByKey = new Map(scored.map(check => [check.stableKey, check])); + const stacks = [...calibration.qualification.stacks].sort(); + return { + required: 'every scored check has an exact known-defect test on every supported stack', + totalChecks: scored.length, + totalPoints: scored.reduce((total, check) => total + check.points, 0), + stacks: stacks.map(stack => { + const covered = new Set(calibration.mutations + .filter(entry => entry.backend === stack) + .flatMap(entry => entry.targets.flatMap(target => target.stableKeys)) + .filter(key => scoredByKey.has(key))); + const missing = scored.filter(check => !covered.has(check.stableKey)); + return { + stack, + coveredChecks: covered.size, + coveredPoints: [...covered].reduce((total, key) => total + (scoredByKey.get(key)?.points ?? 0), 0), + missingChecks: missing.map(check => check.stableKey), + }; + }), + }; +} + +export function qualificationReadiness(trackName: string, level: number, recipe: string | null = null, + mutationWorkers = MAX_MUTATION_WORKERS) { + validateMutationWorkerCount(mutationWorkers); + if (!listTracks().includes(trackName)) throw new Error(`unknown qualification track ${trackName}`); + const track = loadTrack(trackName); + if (!isDeclaredLevel(track, level)) { + throw new Error(`L${level} is not declared for ${trackName}`); + } + const binding: RecipeBinding | null = resolveRecipeRelease(track, level, recipe); + if (!binding) throw new Error(`${trackName} L${level} has no recipe release`); + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, alias: `L${level}` }); + if (!calibration) { + throw new Error(`${binding.release.id} has no L${level} calibration`); + } + const identity = calibrationQualificationIdentity(calibration); + const qualificationLevel = Number(calibration.selection.alias.slice(1)); + const launchBlockers = []; + for (const pack of binding.plan.packs) { + if (pack.budget.status !== 'bounded') { + launchBlockers.push(blocker('pack_budget_unbounded', `packs.${pack.id}.budget`, + `${pack.id} needs a measured maxRuntimeMs before qualification`)); + } + } + + const requiredEvidence = evidencePlan(calibration); + const defectChecks = defectCheckCoverage(binding.release, calibration); + const recorded = new Set(calibration.qualification.evidence.map(entry => + `${entry.kind}:${entry.stack ?? ''}:${entry.repetition}`)); + const qualificationBlockers = [...launchBlockers]; + for (const coverage of defectChecks.stacks.filter(item => item.missingChecks.length > 0)) { + qualificationBlockers.push(blocker('defect_check_coverage_incomplete', + `defectChecks.${coverage.stack}`, + `${coverage.coveredChecks}/${defectChecks.totalChecks} scored checks have exact known-defect tests`)); + } + for (const item of requiredEvidence) { + const key = `${item.kind}:${item.stack ?? ''}:${item.repetition}`; + if (!recorded.has(key)) qualificationBlockers.push(blocker('evidence_missing', `evidence.${key}`, + `${key} has no hash-bound qualification artifact`)); + } + for (const stale of (calibration.qualificationStaleness ?? []) as { + kind: string; stack?: string; repetition: number; reason: string; + }[]) { + const key = `${stale.kind}:${stale.stack ?? ''}:${stale.repetition}`; + qualificationBlockers.push(blocker('qualification_evidence_stale', `evidence.${key}`, + `${key} must be regenerated: ${stale.reason}`)); + } + const output = join(stackBenchResultsRoot(STACK_BENCH_ROOT), 'qualification'); + const stacks = [...calibration.qualification.stacks].sort(); + const budgetEvidence = stacks.map(stack => + `${output}/budget-input/${trackName}-l${qualificationLevel}-${stack}.json`); + const budgetPreparationRequired = launchBlockers.some(item => item.code === 'pack_budget_unbounded'); + const recipeOption = ` --recipe ${binding.release.id}`; + const featureCatalog = calibration.qualification.featureCatalog; + const featureCatalogOption = featureCatalog + ? ` --feature-catalog ${featureCatalog.path}` : ''; + const combinedReferenceEvidence = calibration.qualification.referenceRepetitions + === calibration.qualification.mutationRepetitions; + const artifactStem = `${trackName}-l${qualificationLevel}-${binding.release.contentSha256.slice(0, 12)}`; + const artifactPaths = { + references: Object.fromEntries(stacks.map(stack => [stack, + `${output}/${artifactStem}-${stack}-reference.json`])), + mutations: Object.fromEntries(stacks.map(stack => [stack, + `${output}/${artifactStem}-${stack}-mutation.json`])), + null: `${output}/${artifactStem}-null.json`, + }; + const launchPaths = new Set([artifactPaths.null]); + for (const stack of stacks) { + const mutationPath = artifactPaths.mutations[stack]; + const referencePath = artifactPaths.references[stack]; + if (!mutationPath || !referencePath) throw new Error(`qualification path is missing for ${stack}`); + launchPaths.add(mutationPath); + launchPaths.add(qualificationRunDirectory(mutationPath)); + launchPaths.add(combinedReferenceEvidence + ? companionReferenceArtifactPath(mutationPath) : referencePath); + if (!combinedReferenceEvidence) { + launchPaths.add(qualificationRunDirectory(referencePath)); + } + } + for (const path of [...launchPaths].filter(existsSync).sort()) { + launchBlockers.push(blocker('qualification_output_exists', path, + 'qualification output already exists')); + } + return { + qualificationSchemaVersion: 1, + scope: { track: trackName, level, recipe: { id: binding.release.id, + contentSha256: binding.release.contentSha256 }, + calibration: { ...identity, contentSha256: calibration.contentSha256 }, + runner: calibration.qualification.runner ?? null }, + launch: { ok: launchBlockers.length === 0, blockers: launchBlockers }, + budgetPreparation: { + required: budgetPreparationRequired, + policy: PACK_BUDGET_POLICY, + commands: budgetPreparationRequired ? [ + ...stacks.map((stack, index) => + `qualify-reference --timing-only --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.referenceRepetitions} --out ${budgetEvidence[index]}`), + `pack-budget recommend --track ${trackName} --level ${qualificationLevel}${recipeOption} ${budgetEvidence + .map(path => `--evidence ${path}`).join(' ')} --out ${output}/${trackName}-l${qualificationLevel}-pack-budgets.json`, + ] : [], + }, + requiredEvidence, + defectChecks, + artifactPaths, + commands: qualificationBlockers.length === 0 ? [] : [ + ...stacks.flatMap(stack => [ + ...(!combinedReferenceEvidence ? [ + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.referenceRepetitions} --out ${artifactPaths.references[stack]}`, + ] : []), + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.mutationRepetitions} --mutations --full-mutations${mutationWorkerOption(calibration, stack, mutationWorkers)} --out ${artifactPaths.mutations[stack]}`, + ]), + `qualify-null --track ${trackName} --level ${qualificationLevel}${recipeOption} --out ${artifactPaths.null}`, + ], + qualification: { ready: qualificationBlockers.length === 0, blockers: qualificationBlockers }, + }; +} + +function main() { + const args = parseQualificationArgs(process.argv); + if (!args.track || args.level === null) throw new Error('track and level are required'); + console.log(JSON.stringify(qualificationReadiness(args.track, args.level, args.recipe, args.mutationWorkers), null, 2)); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error: unknown) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 2; } +} diff --git a/tools/stack-bench/commands/recovery.ts b/tools/stack-bench/commands/recovery.ts new file mode 100644 index 00000000000..cf622baf806 --- /dev/null +++ b/tools/stack-bench/commands/recovery.ts @@ -0,0 +1,25 @@ +#!/usr/bin/env node + +import { recoverBackendLease, recoverSupervisedRun } from '../src/runtime/recovery.js'; + +const [command, statePath, option, output] = process.argv.slice(2); +const supervisorRequest = command === 'recover' && statePath !== undefined && process.argv.length === 4; +const leaseRequest = command === 'recover-lease' && statePath !== undefined && option === '--out' + && output !== undefined && process.argv.length === 6; +if (!supervisorRequest && !leaseRequest) { + console.error('Usage:\n' + + ' stack-bench recover \n' + + ' stack-bench recover-lease --out '); + process.exit(2); +} + +try { + const result = leaseRequest + ? recoverBackendLease(statePath, output) + : recoverSupervisedRun(statePath); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.ok ? 0 : 1; +} catch (error) { + console.error(`recovery: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 2; +} diff --git a/tools/stack-bench/commands/repair-cli.ts b/tools/stack-bench/commands/repair-cli.ts new file mode 100644 index 00000000000..b37a06a74a3 --- /dev/null +++ b/tools/stack-bench/commands/repair-cli.ts @@ -0,0 +1,220 @@ +#!/usr/bin/env node + +import { randomUUID } from 'node:crypto'; +import { existsSync, mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { acquireCampaignLock, releaseCampaignLock } from '../src/campaigns/campaign-lock.js'; +import { ARTIFACT_FILE, emptyArtifactIdentities, readArtifact, writeArtifact } + from '../src/evidence/artifacts.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { rescueSupervisedLease } from '../src/runtime/recovery.js'; +import { runBounded } from '../src/runtime/bounded-process.js'; +import type { BoundedProcessResult, RunBoundedOptions } + from '../src/runtime/bounded-process.js'; +import { createRepairGrant, inspectRepairParent } from '../src/runtime/repair-grant.js'; + +const BENCH = join(STACK_BENCH_ROOT, 'dist', 'commands', 'bench.js'); + +export interface RepairStatusArgs { + command: 'status'; + parent: string; + level: number; +} + +export interface RepairGrantArgs { + command: 'grant'; + parent: string; + level: number; + repairs: number; + maxBudgetUsd?: number; + timeoutMinutes: number; +} + +export type RepairArgs = RepairStatusArgs | RepairGrantArgs; + +export function parseRepairArgs(argv: string[]): RepairArgs { + const [command, parent, ...rest] = argv.slice(2); + if (command === 'status' && parent && rest.length === 2 && rest[0] === '--level') { + const level = Number(rest[1]); + if (!Number.isSafeInteger(level) || level < 1) throw new Error('--level must be a positive integer'); + return { command, parent: resolve(parent), level }; + } + if (command !== 'grant' || !parent) { + throw new Error('usage: repair status --level | repair grant --level --repairs [--max-budget-usd ] [--timeout-minutes ]'); + } + const values: { level?: number; repairs?: number; maxBudgetUsd?: number; + timeoutMinutes: number } = { timeoutMinutes: 120 }; + const seen = new Set(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + if (!flag || !['--level', '--repairs', '--max-budget-usd', '--timeout-minutes'].includes(flag) + || index + 1 >= rest.length || seen.has(flag)) { + throw new Error(`invalid or duplicate repair option ${String(flag)}`); + } + seen.add(flag); + const value = Number(rest[index + 1]); + if (flag === '--level') values.level = value; + else if (flag === '--repairs') values.repairs = value; + else if (flag === '--max-budget-usd') values.maxBudgetUsd = value; + else values.timeoutMinutes = value; + } + const level = values.level; + if (level === undefined || !Number.isSafeInteger(level) || level < 1) { + throw new Error('--level must be a positive integer'); + } + const repairs = values.repairs; + if (repairs === undefined || !Number.isSafeInteger(repairs) || repairs < 1) { + throw new Error('--repairs must be a positive safe integer'); + } + if (values.maxBudgetUsd !== undefined + && (!Number.isFinite(values.maxBudgetUsd) || values.maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (!Number.isFinite(values.timeoutMinutes) || values.timeoutMinutes < 10 + || values.timeoutMinutes > 480) { + throw new Error('--timeout-minutes must be from 10 through 480'); + } + return { command, parent: resolve(parent), level, + repairs, timeoutMinutes: values.timeoutMinutes, + ...(values.maxBudgetUsd === undefined ? {} : { maxBudgetUsd: values.maxBudgetUsd }) }; +} + +export function repairStatus(parent: string, level: number): Record { + try { + const inspected = inspectRepairParent(parent, level); + return { eligible: true, parentRunId: inspected.parent.id, level, + score: inspected.level.score, max: inspected.level.max, + used: inspected.cumulativeRepairsBefore, + checkpointSha256: inspected.checkpoint.payload.source.sha256 }; + } catch (error) { + return { eligible: false, level, + reason: error instanceof Error ? error.message : String(error) }; + } +} + +interface RepairExecutionDependencies { + execute?: (command: string, argv: string[], + options: RunBoundedOptions) => Promise; + rescue?: (path: string, output: string) => void; + uuid?: () => string; + env?: NodeJS.ProcessEnv; +} + +interface RepairContinuationPayload { + outcome?: unknown; + continuation?: { + parentRunId?: string; + repairsGranted?: number; + level?: number; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +export async function executeRepairGrant(args: RepairGrantArgs, + { execute = runBounded, rescue = rescueSupervisedLease, uuid = randomUUID, + env = process.env }: RepairExecutionDependencies = {}) { + const resolved = createRepairGrant(args.parent, { level: args.level, repairs: args.repairs }); + const lock = acquireCampaignLock(join(resolved.root, '.repair-control'), { + id: `repair-l${args.level}`, + contentSha256: resolved.checkpoint.payload.source.sha256, + }); + const stamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); + const executionId = `grant-${stamp}-${uuid().replaceAll('-', '').slice(0, 12)}`; + const output = join(resolved.root, 'continuations', executionId); + const privateRoot = join(tmpdir(), 'stack-bench-repair-supervisors'); + const supervisorState = join(privateRoot, `${executionId}.json`); + try { + mkdirSync(output, { recursive: true }); + mkdirSync(privateRoot, { recursive: true, mode: 0o700 }); + const argv = [BENCH, + '--repair-from', resolved.root, + '--repair-level', String(args.level), + '--repairs', String(args.repairs), + '--out', output, + '--no-media']; + if (args.maxBudgetUsd !== undefined) { + argv.push('--max-budget-usd', String(args.maxBudgetUsd)); + } + const childEnv: NodeJS.ProcessEnv = { + ...env, + STACK_BENCH_SUPERVISOR_STATE: supervisorState, + }; + if (resolved.configuration.buildImage) { + childEnv.STACK_BENCH_IMAGE = resolved.configuration.buildImage; + } + const processResult = await execute(process.execPath, argv, { + cwd: STACK_BENCH_ROOT, + env: childEnv, + stdio: 'inherit', + timeoutMs: args.timeoutMinutes * 60_000, + logs: { stdout: join(output, 'process.stdout.log'), + stderr: join(output, 'process.stderr.log') }, + }); + let cleanupError: unknown = null; + if (!processResult.ok && existsSync(supervisorState)) { + try { rescue(supervisorState, output); } + catch (error) { cleanupError = error; } + } + const streams = processResult.logs ? Object.fromEntries(Object.entries(processResult.logs) + .map(([name, value]) => [name, { ...value, path: `process.${name}.log` }])) : null; + writeArtifact(join(output, ARTIFACT_FILE.process), { + kind: 'repair_process', + id: `${executionId}-process`, + attempt: { id: `${executionId}-process`, parentId: resolved.parent.id }, + identities: emptyArtifactIdentities({ + agentAdapter: resolved.parentArtifact.identities.agentAdapter, + stackAdapter: resolved.parentArtifact.identities.stackAdapter, + }), + payload: { schemaVersion: 2, parentRunId: resolved.parent.id, + level: args.level, repairsGranted: args.repairs, + exitCode: processResult.code ?? null, signal: processResult.signal ?? null, + timedOut: processResult.timedOut, streams }, + }); + if (cleanupError) { + const detail = cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + throw new Error(`repair continuation cleanup failed: ${detail}`); + } + const runPath = join(output, ARTIFACT_FILE.run); + if (!existsSync(runPath)) { + throw new Error(`repair continuation produced no run artifact${processResult.timedOut ? ' before its timeout' : ''}`); + } + const run = readArtifact(runPath, + { expectedKind: 'repair_continuation' }); + if (run.attempt.parentId !== resolved.parent.id + || run.payload.continuation?.parentRunId !== resolved.parent.id + || run.payload.continuation?.repairsGranted !== args.repairs + || run.payload.continuation?.level !== args.level) { + throw new Error('repair continuation result does not match its grant'); + } + return { output, process: processResult, run }; + } finally { + rmSync(supervisorState, { force: true }); + releaseCampaignLock(lock); + } +} + +async function main(): Promise { + const args = parseRepairArgs(process.argv); + if (args.command === 'status') { + const status = repairStatus(args.parent, args.level); + console.log(JSON.stringify(status, null, 2)); + if (status.eligible !== true) process.exitCode = 1; + return; + } + const result = await executeRepairGrant(args); + console.log(JSON.stringify({ output: result.output, id: result.run.id, + outcome: result.run.payload.outcome, + continuation: result.run.payload.continuation }, null, 2)); + if (!result.process.ok) process.exitCode = 1; +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/report-bugs.ts b/tools/stack-bench/commands/report-bugs.ts new file mode 100644 index 00000000000..b2ad2268d8a --- /dev/null +++ b/tools/stack-bench/commands/report-bugs.ts @@ -0,0 +1,398 @@ +#!/usr/bin/env node +import { privateGradingDirectory } from '../src/evidence/repair-evidence.js'; +// Turns grading results into a behavioral BUG_REPORT.md for the fix agent. +// +// Report behavior and typed observations, never implementation advice. A setup +// failure must not be described as a failure of the later criterion. + +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { findingStatus, renderRepairFinding } from '../src/actions/action-findings.js'; +import type { Finding } from '../src/actions/action-findings.js'; +import type { ActionEvidence } from '../src/actions/action-contract.js'; +import type { CheckEvidence } from '../src/evidence/check-evidence.js'; +import { sanitiseConsoleError, sanitiseDiagnostic } from '../src/evidence/diagnostic-sanitizer.js'; +import { ARTIFACT_FILE, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { criterionEvidence, evidenceIsRepairable, validateCheckEvidence } from '../src/evidence/check-evidence.js'; +import { assertAgentVisibleText } from '../src/composition/agent-visible-contract.js'; +import { CODING_CONTAINER_BUG_REPORT_FILE, CODING_CONTAINER_START_SCRIPT } + from '../src/runtime/coding-container-policy.js'; + +interface RepairHistoryEntry { + round?: number; + beforeScore?: number; + beforeMax?: number; + afterScore?: number; + afterMax?: number; + result?: string; + remainingFailures?: string[]; +} + +interface ReportBugsArgs { + app: string; + results: string; + out: string; + archive?: string; + history: RepairHistoryEntry[]; + checks: string[] | null; + controls: string[] | null; + priorRegression: string | null; + regressionContext: boolean; +} + +interface ParsedArgs { + app?: string; + results?: string; + out?: string; + archive?: string; + history?: unknown; + checks?: unknown; + controls?: unknown; +} + +interface Criterion { + id?: string; + stableKey?: string; + desc?: string; + statedBy?: string; + points?: number; + evidence?: unknown; +} + +interface GradeFeature { + name?: string; + consoleErrors?: string[]; + criteria?: Criterion[]; + setupEvidence?: unknown; +} + +interface GradePayload { + features?: GradeFeature[]; +} + +interface ContractResult { + id: string; + status: string; + detail?: string; +} + +interface ContractLintPayload { + results?: ContractResult[]; +} + +interface GradeBundlePayload { + backend?: string; + outcome?: { kind?: string; phase?: string; reason?: string }; +} + +interface RepairBug { + area: string; + actor: string | null; + action: string | null; + expected: string | null; + observed: string; + consoleErrors: string[]; + contract: boolean; + context?: string[]; +} + +// Only completed public interactions, never raw action inputs or diagnostics. +// These observations explain where a sequence stopped without teaching a fix. +function observationContext(evidence: CheckEvidence): string[] { + const actions = evidence.actions.map(entry => ({ actor: entry.actor, + evidence: entry.evidence as ActionEvidence })); + const failureIndex = actions.findLastIndex(entry => entry.evidence.status === 'failed'); + const context: string[] = evidence.phase === 'setup' + ? ['Setup stopped before the named behavior was reached.'] : []; + if (failureIndex < 0) return context; + const completed: string[] = []; + const lifecycle: string[] = []; + for (const { actor, evidence: action } of actions.slice(0, failureIndex)) { + if (action.status !== 'passed') continue; + const observation = action.observation && typeof action.observation === 'object' + ? action.observation as Record : {}; + if (action.action.id === 'click' && typeof observation.clicked === 'string') { + completed.push(`${actor ? `${sanitiseDiagnostic(actor, 120)}: ` : ''}${sanitiseDiagnostic(observation.clicked, 120)}`); + } + if (action.action.id === 'callAction' && typeof observation.action === 'string' + && Number.isInteger(observation.status) && Number(observation.status) >= 100 && Number(observation.status) <= 599) { + completed.push(`${actor ? `${sanitiseDiagnostic(actor, 120)}: ` : ''}${sanitiseDiagnostic(observation.action, 120)} returned HTTP ${observation.status}`); + } + const account = ({ signIn: 'sign-in completed', signUp: 'account creation completed', + freshClient: 'fresh client opened' } as Record)[action.action.id]; + if (account) completed.push(`${actor ? `${sanitiseDiagnostic(actor, 120)}: ` : ''}${account}`); + const operations: Record = { reload: 'page reloaded', stopAppServer: 'application server stopped', + startAppServer: 'application server started', restartBackend: 'database runtime restarted' }; + const operation = operations[action.action.id]; + if (operation) lifecycle.push(operation); + } + if (completed.length) context.push(`Recent completed actions: ${completed.slice(-8).join(' → ')}.`); + if (lifecycle.length) context.push(`Completed lifecycle actions: ${[...new Set(lifecycle)].join('; ')}.`); + if (evidence.finding && ['control-missing', 'control-not-ready', 'control-blocked', 'control-unreadable', + 'choice-missing', 'page-timeout'].includes(evidence.finding.kind)) { + context.push('The sequence stopped at this control; later behavior was not observed.'); + } + if (evidence.finding?.kind === 'page-error') { + context.push('The sequence stopped at this action; later behavior was not observed.'); + } + if (evidence.finding && ['value-mismatch', 'number-mismatch'].includes(evidence.finding.kind)) { + context.push('The sequence stopped at this value check; later behavior was not observed.'); + } + return context; +} + +// The public verb for the step that failed. Control and action names are the +// agent's own vocabulary; nothing else about the step is repeated. +function failedAction(action: string | undefined, finding: Finding | null): string | null { + if (action === 'fill') { + return finding?.kind === 'choice-missing' ? 'Select the requested choice' : 'Enter the requested value'; + } + if (action === 'click') return 'Use the requested control'; + if (action === 'signIn') return 'Sign in'; + if (action === 'signUp') return 'Create the account'; + if (action === 'reload') return 'Reload the page'; + return null; +} + +// What the application did, from the finding alone. A failure without a +// finding (the feature's setup failed before this behavior was reached) +// says so and nothing more. +function observed(finding: Finding | null, phase: string): string { + if (finding?.kind === 'page-error') { + // Raw browser diagnostics can contain URLs, credentials and private probe text. + // Report only a recognized transport code, never the surrounding diagnostic. + const code = finding.fields.detail?.match(/\b(?:net::)?(ERR_CONNECTION_REFUSED|ERR_CONNECTION_RESET|ERR_CONNECTION_CLOSED|ERR_CONNECTION_TIMED_OUT|ERR_NAME_NOT_RESOLVED|ERR_ADDRESS_UNREACHABLE|ERR_EMPTY_RESPONSE|ERR_TIMED_OUT)\b/)?.[1]; + if (code) return `the browser request failed (${code})`; + } + if (finding) return renderRepairFinding(finding); + return phase === 'setup' + ? 'the application did not reach this behavior; an earlier step of the same feature failed' + : 'a failure was recorded without a detailed observation'; +} + +export function parseReportBugsArgs(argv: string[]): ReportBugsArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + app: { type: 'string' }, results: { type: 'string' }, out: { type: 'string' }, + archive: { type: 'string' }, + 'history-json': { type: 'string' }, 'checks-json': { type: 'string' }, + 'controls-json': { type: 'string' }, + 'prior-regression': { type: 'string' }, + 'regression-context': { type: 'boolean' }, + } }); + const args: ParsedArgs = { app: values.app, results: values.results, out: values.out, + archive: values.archive, + history: values['history-json'] === undefined ? undefined : JSON.parse(values['history-json']), + checks: values['checks-json'] === undefined ? undefined : JSON.parse(values['checks-json']), + controls: values['controls-json'] === undefined ? undefined : JSON.parse(values['controls-json']) }; + if (!args.app) { + throw new Error('Usage: report-bugs --app [--out ]'); + } + args.results ??= privateGradingDirectory(args.app); + args.out ??= join(args.app, CODING_CONTAINER_BUG_REPORT_FILE); + args.history ??= []; + if (!Array.isArray(args.history)) throw new Error('--history-json must contain an array'); + args.checks ??= null; + if (args.checks !== null && (!Array.isArray(args.checks) + || args.checks.some(check => typeof check !== 'string' || !check) + || new Set(args.checks).size !== args.checks.length)) { + throw new Error('--checks-json must contain distinct non-empty strings'); + } + args.controls ??= null; + if (args.controls !== null && (!Array.isArray(args.controls) + || args.controls.some(control => typeof control !== 'string' || !control) + || new Set(args.controls).size !== args.controls.length)) { + throw new Error('--controls-json must contain distinct non-empty strings'); + } + return { app: args.app, results: args.results, out: args.out, archive: args.archive, + history: args.history as RepairHistoryEntry[], checks: args.checks as string[] | null, + controls: args.controls as string[] | null, + priorRegression: values['prior-regression'] ?? null, + regressionContext: values['regression-context'] ?? false }; +} + +function priorRegressionSection(path: string): string[] { + const details = assertAgentVisibleText(readFileSync(resolve(path), 'utf8')).trim() + .replace(/^### /gm, '#### ') + .replace(/^## /gm, '### '); + if (!details) throw new Error('prior regression report has no failure details'); + return [ + '## Previous repair regression', + '', + 'The previous repair was rolled back because it broke behavior that already worked.', + 'Keep this behavior working while you fix the current problems.', + '', + ...details.split(/\r?\n/), + '', + ]; +} + +export function createBugReport(args: ReportBugsArgs): number { + const resultsDir = resolve(args.results); + if (!existsSync(resultsDir)) throw new Error(`No grading results in ${resultsDir}`); + + const bugs: RepairBug[] = []; + const reportedSetups = new Set(); + const selectedChecks = args.checks === null ? null : new Set(args.checks); + const selectedControls = args.controls === null ? null : new Set(args.controls); + + for (const file of readdirSync(resultsDir).filter(name => /^grading-.*\.json$/.test(name))) { + const report = readArtifactPayload(join(resultsDir, file), { expectedKind: 'grade' }); + for (const feature of report.features ?? []) { + // Repairs receive only scored, typed application failures. + for (const criterion of feature.criteria ?? []) { + if (selectedChecks && (!criterion.stableKey + || !selectedChecks.has(criterion.stableKey))) continue; + if (!(Number(criterion.points) > 0)) continue; + const evidence = criterionEvidence(criterion); + if (!evidenceIsRepairable(evidence)) continue; + const failure = evidence.phase === 'setup' && feature.setupEvidence + ? validateCheckEvidence(feature.setupEvidence) : evidence; + if (!evidenceIsRepairable(failure) + || (failure.finding && findingStatus(failure.finding) !== 'failed')) continue; + if (evidence.phase === 'setup') { + // One failed setup is copied to each selected criterion it prevented. + // Use full evidence, not rendered prose, so distinct failures stay separate. + const key = JSON.stringify({ area: feature.name, failure }); + if (reportedSetups.has(key)) continue; + reportedSetups.add(key); + } + const actionEntry = failure.actions.findLast(entry => + entry.evidence !== null && typeof entry.evidence === 'object' + && (entry.evidence as { status?: string }).status === 'failed') ?? failure.actions.at(-1); + const actionId = actionEntry && typeof actionEntry.evidence === 'object' && actionEntry.evidence + ? String((actionEntry.evidence as { action?: { id?: string } }).action?.id ?? '') : undefined; + const expected = evidence.phase === 'setup' ? null + : (criterion.desc ?? criterion.statedBy ?? '').trim() || 'the requested behavior'; + bugs.push({ + area: sanitiseDiagnostic(feature.name, 120), + actor: sanitiseDiagnostic(actionEntry?.actor ?? failure.actor, 120) || null, + action: failedAction(actionId, failure.finding), + expected, + observed: observed(failure.finding, evidence.phase), + context: observationContext(failure), + consoleErrors: [...new Set((feature.consoleErrors ?? []) + .map(sanitiseConsoleError).filter(Boolean))].slice(0, 3), + contract: false, + }); + } + } + } + + // Contract failures are separate because the interface name is itself the public + // requirement here. Behavioral failures above must never expose one. + const lintPath = join(resultsDir, ARTIFACT_FILE.contractLint); + if (existsSync(lintPath)) { + const lint = readArtifactPayload(lintPath, { expectedKind: 'contract_lint' }); + for (const result of (lint.results ?? []).filter(item => item.status === 'FAIL' + && (!selectedControls || selectedControls.has(item.id)))) { + bugs.push({ + area: 'Application interface', + actor: null, + action: null, + expected: `A visible element for "${(result.detail ?? '').split('expected: ').pop()}" must use the "${result.id}" application interface`, + observed: sanitiseDiagnostic(result.detail + ?? `no visible element with id="${result.id}" was found after a clean reset`, 500), + consoleErrors: [], contract: true, + }); + } + } + + const bundlePath = join(resultsDir, ARTIFACT_FILE.gradeBundle); + if (existsSync(bundlePath)) { + const bundle = readArtifactPayload(bundlePath, { expectedKind: 'grade_bundle' }); + if (bundle.outcome?.kind === 'app_failure' && bundle.outcome.reason) { + const expectedByPhase: Record = { + 'database-provenance': `The app must use the ${bundle.backend} database and connection supplied for this run.`, + 'application-layout': 'The app must use a project layout that can be built, started, and reset repeatedly.', + 'application-restart': `The app must provide ${CODING_CONTAINER_START_SCRIPT}. From clean source, it must install dependencies, build, and start the complete application without changing source files.`, + }; + const expected = expectedByPhase[bundle.outcome.phase ?? ''] + ?? 'The app must start successfully in the supplied environment.'; + bugs.unshift({ + area: 'Application setup', + actor: null, + action: null, + expected, + observed: sanitiseDiagnostic(bundle.outcome.reason, 500), + consoleErrors: [], + contract: false, + }); + } + } + + if (bugs.length === 0) { + console.log('No failures — no bug report written.'); + return 3; + } + + const behavioral = bugs.filter(bug => !bug.contract); + const contractFailures = bugs.filter(bug => bug.contract); + const lines = args.regressionContext ? [] : [ + '# Bug Report', + '', + 'The application has these problems after a clean database reset and a fresh', + 'restart. Fix the behavior, then redeploy.', + 'Do not change behavior that is already correct. A result from existing local', + 'state does not replace the clean result below.', + '', + ]; + + if (!args.regressionContext && args.history.length) { + lines.push('## Earlier work', ''); + lines.push('Use the current source as the starting point. Preserve earlier fixes while', + 'addressing the remaining problems below.', ''); + } + + if (behavioral.length) { + lines.push('## Behavior', ''); + behavioral.forEach((bug, index) => { + lines.push(`### Bug ${index + 1}: ${bug.area}`, ''); + if (bug.actor) lines.push(`**Actor/session:** ${bug.actor}`, ''); + if (bug.action) lines.push(`**Failed action:** ${bug.action}`, ''); + if (bug.expected) lines.push(`**Expected:** ${bug.expected}`, ''); + lines.push(`**Actual:** ${bug.observed}`, ''); + if (bug.context?.length) lines.push(`**Observed context:** ${bug.context.join(' ')}`, ''); + if (bug.consoleErrors.length) { + lines.push('**Console or network errors:**', ''); + bug.consoleErrors.forEach(error => lines.push(`- \`${error}\``)); + lines.push(''); + } + }); + } + + if (contractFailures.length) { + lines.push('## Application interface', ''); + lines.push('These required elements were not available in the clean application state:', ''); + contractFailures.forEach(bug => { + lines.push(`- **Expected:** ${bug.expected}`); + lines.push(` **Actual:** ${bug.observed}`); + }); + lines.push(''); + } + + if (args.priorRegression) lines.push(...priorRegressionSection(args.priorRegression)); + + const reportText = assertAgentVisibleText(lines.join('\n')); + // The agent owns this directory: replace whatever is there rather than write through a link. + rmSync(args.out, { force: true }); + writeFileSync(args.out, reportText, { flag: 'wx' }); + if (args.archive) { + mkdirSync(dirname(args.archive), { recursive: true }); + writeFileSync(args.archive, reportText); + } + console.log(`Wrote ${bugs.length} bug(s) to ${args.out}`); + return 0; +} + +function main(): void { + try { + process.exitCode = createBugReport(parseReportBugsArgs(process.argv)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/reset-backend.ts b/tools/stack-bench/commands/reset-backend.ts new file mode 100644 index 00000000000..1e08c6da017 --- /dev/null +++ b/tools/stack-bench/commands/reset-backend.ts @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +import { GENERATED_APP_LAYOUT_EXIT_CODE, resetBackend } from '../src/stacks/backend-reset.js'; +import { GeneratedAppLayoutError } from '../src/runtime/spacetime-layout.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +const [backend, app] = process.argv.slice(2); +if (!backend || !app) throw new Error('usage: node dist/commands/reset-backend.js '); + +Promise.resolve().then(() => resetBackend({ backend, app })).then(result => { + console.log(result); +}).catch(error => { + if (error instanceof GeneratedAppLayoutError || error?.code === 'generated_app_layout') { + console.error(`GENERATED_APP_LAYOUT: ${error.message}`); + process.exitCode = GENERATED_APP_LAYOUT_EXIT_CODE; + return; + } + const childOutput = [error?.stderr, error?.stdout] + .filter(value => value !== undefined && value !== null && String(value).trim()) + .map(value => String(value).trim()).join('\n'); + if (childOutput) console.error(redactCredentials(childOutput).slice(-2000)); + console.error(redactCredentials(error.stack ?? error.message)); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/run-suite.ts b/tools/stack-bench/commands/run-suite.ts new file mode 100644 index 00000000000..6ead1d7e9db --- /dev/null +++ b/tools/stack-bench/commands/run-suite.ts @@ -0,0 +1,1286 @@ +#!/usr/bin/env node +import { privateGradingDirectory } from '../src/evidence/repair-evidence.js'; + +import { execFile, execFileSync } from 'node:child_process'; +import type { ExecFileException, ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { performance } from 'node:perf_hooks'; +import { measurePhase, type PhaseTiming } from '../src/evidence/phase-timing.js'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync, cpSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { parseArgs as parseNodeArgs, promisify } from 'node:util'; +import { chromium } from 'playwright'; +import { attemptBrowserLaunchOptions } from '../container/browser-pipe.js'; +import type { Browser, BrowserServer } from 'playwright'; +import { Actor } from '../grader/grade.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { ActionApplicationFailure, executeAction } from '../src/actions/action-contract.js'; +import { runApplicationNavigation } from '../src/actions/browser-navigation.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { dbName, loadTrack, suitesFor, DEFAULT_TRACK } from '../src/composition/tracks.js'; +import { controlAppServer, parseRuntimeControlSpec } + from '../src/runtime/backend-control.js'; +import type { RuntimeControlSpec } from '../src/runtime/backend-control.js'; +import { ARTIFACT_FILE, readArtifactPayload, recipeArtifactIdentities, writeArtifact } + from '../src/evidence/artifacts.js'; +import { bundleRecipeRelease, resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { contractInterfaceNames } from '../src/composition/agent-visible-contract.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { criterionEvidence, evidencePassed } from '../src/evidence/check-evidence.js'; +import { renderEvidenceConsoleLine } from '../src/evidence/evidence-presentation.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { requireLeasedDatabase, requireLeasedSpacetime } from '../src/stacks/backend-reset-guard.js'; +import { aggregatePackRuntime, exceededPackBudgets } from '../src/composition/pack-runtime.js'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import { GENERATED_APP_LAYOUT_EXIT_CODE } from '../src/stacks/backend-reset.js'; +import { readBackendLease } from '../src/runtime/backend-lease.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { sha256 } from '../src/evidence/provenance.js'; +import { GRADER_SOURCE_TIMEOUT_MS, gradingSourceTimeoutMs } from '../src/runtime/grading-timeout.js'; +import type { BackendLease, BackendLeaseExpectation } from '../src/runtime/backend-lease.js'; +import type { CheckEvidence } from '../src/evidence/check-evidence.js'; +import type { AggregatedPackRuntimeEvidence, PackRuntimeEvidence } from '../src/composition/pack-runtime.js'; +import { isModularRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import type { BoundRecipeTaskRequestResult, RecipeSelection } from '../src/composition/recipe-selection.js'; +import type { RecipeBinding, RecipeCheck } from '../src/composition/recipe-release.js'; +import type { Track, TrackSuite } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const RESET = compiledEntrypoint('commands', 'reset-backend.js'); + +type Observation = 'scored' | 'observed'; +type Selection = { + schemaVersion: number; + recipe: { id: string; contentSha256: string }; + requested: RecipeSelection['requested']; + sha256: string; + checks: RecipeCheck[]; + scoredPoints: number; + observedChecks?: Array; + observedPoints?: number; + evaluationSha256?: string; + regressionChecks?: Array; + regressionPoints?: number; + observation?: Observation; +}; +type DeclaredSuite = TrackSuite; +type Failure = Error & { stdout?: string; stderr?: string; status?: number | null; signal?: string | null; + code?: string }; +type FailureDetail = { message?: unknown; stderr?: unknown } | null; +type RecipeTaskArgument = { recipe: { id: string; contentSha256?: string } } & Record; +type RunArguments = { + app: string; + url: string; + backend: string; + label: string; + out: string; + level: string; + reset: boolean; + retryInconclusive: boolean; + media: boolean; + runIndex: number; + track: string; + packIds: string[]; + checkKeys: string[]; + observation: Observation; + recipe?: string; + recipeTask?: RecipeTaskArgument; + credentialAliases?: unknown; + regressionChecks: string[]; + sourceSha256?: string; + restartSpec?: RuntimeControlSpec; + applicationFailure?: ApplicationFailure; + parentAttemptId?: string; + databaseLease?: BackendLease | null; + browserWsEndpoint?: string; + selection?: Selection | null; + bundleArtifactId: string; +}; +type GradeCriterion = { id: string; stableKey?: string; serverCheck?: string; evidence?: CheckEvidence }; +type GradeFeature = { name: string; criteria: GradeCriterion[]; + cleanupEvidence?: { status?: string; failures: Array<{ stage: string }> } }; +type GradePayload = { total: number; max: number; features: GradeFeature[]; + cleanupEvidence?: { status?: string }; + selection?: { checks?: RecipeCheck[] }; packRuntime?: PackRuntimeEvidence }; +type LintPayload = { + pass: boolean; + counts: { pass: number; fail: number; blocked: number; scenario: number }; +}; +type ActionsPayload = { missing: string[]; results: unknown[] }; +type RuntimeProvenance = { ok: boolean | null; verified: boolean; reason: string }; +type ApplicationProbeResult = { ok: boolean; detail: string | null; timedOut?: true }; +type ResetOutcome = { kind: string; phase: string; appFailures?: string[] }; +type ApplicationFailure = ResetOutcome & { kind: 'app_failure'; reason: string }; +type DatabaseProvenance = { ok: boolean; reason: string; url?: string }; +type GradeLeaseReader = typeof readBackendLease; +type MutationDirectoryEntry = { name: string; isDirectory(): boolean; isFile(): boolean }; +type MutationDirectoryReader = (path: string, options: { withFileTypes: true }) => readonly MutationDirectoryEntry[]; +type ProbeResponse = { ok: boolean; status: number }; +type ApplicationFetch = (url: string, init: { signal: AbortSignal }) => Promise; +type DatabaseProvenanceDefinition = Track['databaseProvenance']; +type DatabaseNameLease = { resources: { database?: string | null } }; +type ProvenanceWrite = { ok: true; marker: string } | { ok: false; marker: null; reason: string }; +type ApplicationFailureSelection = { checks: Array<{ executionId: string; points?: number; stableKey?: string }> }; +type ContractLintArguments = Pick; +type BundleSelection = Selection & { attemptedChecks: string[]; reportedChecks: string[]; + notRun: Array<{ stableKey: string; reason: string }> }; +type Bundle = { + definitionSchemaVersion: number; + recipeRelease: ReturnType; + calibration: { id: string; contentSha256: string } | null; + label: string; track: string; backend: string; url: string; app: string; level: number; + observation: Observation; source?: { sha256: string }; + suites: Record; + suiteRetries?: Record; + totals: Record; + selection: BundleSelection | null; + code?: ReturnType; + error?: string; + outcome?: { kind: string; phase: string; reason?: string; appFailures?: string[] }; + provenance?: DatabaseProvenance & { runtime?: RuntimeProvenance }; + actions?: ActionsPayload | null; + packRuntime?: AggregatedPackRuntimeEvidence; + phaseTimings: PhaseTiming[]; +}; + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const parseObservation = (value: string): Observation => { + if (value === 'scored' || value === 'observed') return value; + throw new Error('--observation must be scored or observed'); +}; + +export function suitesForRecipe(track: Track, binding: RecipeBinding): DeclaredSuite[] { + if (!binding?.execution?.length) throw new Error('recipe has no typed execution plan'); + return binding.execution.map(entry => ({ + id: entry.id, + spec: resolve(track.dir, entry.source ?? ''), + ...(entry.ownership.kind === 'inherited' + ? { inherited: true, fromLevel: entry.ownership.fromLevel } + : {}), + })); +} + +export function childFailureDetail(failure: FailureDetail = null, stdout = '', limit = 600): string { + const processOutput = [failure?.stderr, stdout] + .filter(value => value !== undefined && value !== null && String(value).trim()) + .join('\n').trim(); + const diagnostic = processOutput || String(failure?.message ?? '').trim(); + const lines = diagnostic.split(/\r?\n/).map(line => line.trim()).filter(Boolean); + if (!lines.length) return ''; + const punctuationOnly = (line: string) => + [...line].every(character => '[]{},'.includes(character)); + const noise = (line: string) => line.startsWith('at ') || /^Node\.js v/.test(line) + || /^node:internal\//.test(line) || /^\^+$/.test(line) || punctuationOnly(line); + const cause = lines.find(line => !noise(line) && /(?:error|failed|timeout|closed|econn|killed)/i.test(line)) + ?? lines.find(line => !noise(line)) ?? lines[0]; + const selected = [cause, ...lines.slice(-4)].filter((line, index, all) => all.indexOf(line) === index); + return selected.join(' | ').slice(0, limit); +} + +export function resetFailureOutcome(error: unknown): ResetOutcome { + const failure = isRecord(error) ? error : {}; + return failure.status === GENERATED_APP_LAYOUT_EXIT_CODE + ? { kind: 'app_failure', phase: 'application-layout', + appFailures: ['application-layout'] } + : failure.code === 'generated_app_not_restartable' + ? { kind: 'app_failure', phase: 'application-restart', + appFailures: ['application-restart'] } + : { kind: 'harness_failure', phase: 'database-reset' }; +} + +// Passes measured before an app abort keep their points; inherited regression guards stay unscored. +export function applicationFailureTotals(selection: ApplicationFailureSelection | null | undefined, + declaredSuites: Array>, + passed: ReadonlySet): Record { + if (!selection?.checks?.length) return {}; + const inherited = new Set(declaredSuites.filter(suite => suite.inherited).map(suite => suite.id)); + const currentMax = selection.checks.filter(check => !inherited.has(check.executionId)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + const regressionMax = selection.checks.filter(check => inherited.has(check.executionId)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + const score = selection.checks.filter(check => !inherited.has(check.executionId) + && check.stableKey !== undefined && passed.has(check.stableKey)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + return { score, max: currentMax, dirty: false, contractPass: null, + regression: regressionMax ? { score: 0, max: regressionMax } : null }; +} + +export function clearPreviousGradeOutputs(output: string): void { + const generated = existsSync(output) ? readdirSync(output).filter(name => + /^grading-.+\.json$/.test(name) || /^grader-.+\.(?:stdout|stderr)\.log$/.test(name)) : []; + for (const name of [ARTIFACT_FILE.gradeBundle, ARTIFACT_FILE.contractLint, + ARTIFACT_FILE.actions, 'media', 'failure-media', + 'database-provenance', 'application-start.log', 'suite-retries', ...generated]) { + rmSync(join(output, name), { recursive: true, force: true }); + } +} + +function recordGraderChildResult(output: string, suiteId: string, + result: { stdout?: unknown; stderr?: unknown; failure?: Error | null }) { + const stdout = redactCredentials(String(result.stdout ?? '')); + const stderr = redactCredentials(String(result.stderr ?? '')); + const safeId = String(suiteId).replace(/[^A-Za-z0-9._-]/g, '_'); + const stdoutName = `grader-${safeId}.stdout.log`; + const stderrName = `grader-${safeId}.stderr.log`; + writeFileSync(join(output, stdoutName), stdout); + writeFileSync(join(output, stderrName), stderr); + const failure = result.failure ?? null; + if (failure) Object.assign(failure, { stdout, stderr }); + return { stdout, stderr, failure, stdoutName, stderrName }; +} + +const execFileAsync = promisify(execFile); + +export async function runGraderChild(argv: string[], output: string, suiteId: string, + timeout = COMMAND_TIMEOUT_MS) { + try { + const result = await execFileAsync(process.execPath, argv, { encoding: 'utf8', cwd: ROOT, + timeout, maxBuffer: 64 * 1024 * 1024 }); + return recordGraderChildResult(output, suiteId, result); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + const processFailure = error as ExecFileException & { stdout?: unknown; stderr?: unknown }; + return recordGraderChildResult(output, suiteId, { + stdout: processFailure.stdout, stderr: processFailure.stderr, failure, + }); + } +} + +function gradeLeaseInput(backend: string, env: NodeJS.ProcessEnv): { path: string; + expected: BackendLeaseExpectation } | null { + if (!['mongodb', 'postgres', 'spacetime', 'convex'].includes(backend)) return null; + const path = String(env.STACK_BENCH_LEASE ?? '').trim(); + const token = String(env.STACK_BENCH_LEASE_TOKEN ?? '').trim(); + if (!path && !token) return null; + if (!path || !token) throw new Error('database grading requires both lease path and lease token'); + return { path, expected: { token, backend, active: true } }; +} + +export function databaseLeaseForGrading(backend: string, env = process.env, { + readLease = readBackendLease, +}: { readLease?: GradeLeaseReader } = {}) { + const input = gradeLeaseInput(backend, env); + if (!input) return null; + const lease = readLease(input.path, input.expected); + if (backend === 'spacetime') { + if (!lease.resources.module || !lease.resources.serverUri) { + throw new Error('active spacetime lease has no complete module target'); + } + return lease; + } + const container = String(lease.resources?.container?.name ?? '').trim(); + const containerId = String(lease.resources?.container?.id ?? '').trim(); + if (!container || !containerId) { + throw new Error(`active ${backend} lease has no complete database container identity`); + } + return lease; +} + +export function databaseNameForGrading(track: Pick, runIndex: number, + lease: DatabaseNameLease | null = null): string { + if (!lease) return dbName(track, runIndex); + const database = String(lease.resources?.database ?? '').trim(); + if (!database) throw new Error('active database lease has no database name'); + return database; +} + +function parseArgs(argv: string[]): RunArguments { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + app: { type: 'string' }, url: { type: 'string' }, backend: { type: 'string' }, + label: { type: 'string' }, out: { type: 'string' }, level: { type: 'string' }, + recipe: { type: 'string' }, 'recipe-task-json': { type: 'string' }, + 'credential-aliases-json': { type: 'string' }, 'regression-checks-json': { type: 'string' }, + observation: { type: 'string' }, 'source-sha256': { type: 'string' }, + 'no-media': { type: 'boolean' }, track: { type: 'string' }, + pack: { type: 'string', multiple: true }, check: { type: 'string', multiple: true }, + 'restart-spec': { type: 'string' }, 'application-failure-json': { type: 'string' }, + 'run-index': { type: 'string' }, 'no-reset': { type: 'boolean' }, + 'retry-inconclusive': { type: 'boolean' }, + 'parent-attempt-id': { type: 'string' }, + } }); + const a: RunArguments = { app: values.app ?? '', url: values.url ?? '', + backend: values.backend ?? '', label: values.label ?? '', out: values.out ?? '', + level: values.level ?? '1', reset: !(values['no-reset'] ?? false), + retryInconclusive: values['retry-inconclusive'] ?? false, + media: !(values['no-media'] ?? false), runIndex: Number(values['run-index'] ?? 0), + track: values.track ?? DEFAULT_TRACK, + packIds: (values.pack ?? []).flatMap(value => value.split(',').filter(Boolean)), + checkKeys: (values.check ?? []).flatMap(value => value.split(',').filter(Boolean)), + observation: parseObservation(values.observation ?? 'scored'), + recipe: values.recipe, + recipeTask: values['recipe-task-json'] === undefined ? undefined : JSON.parse(values['recipe-task-json']), + credentialAliases: values['credential-aliases-json'] === undefined + ? undefined : JSON.parse(values['credential-aliases-json']), + regressionChecks: values['regression-checks-json'] === undefined + ? [] : JSON.parse(values['regression-checks-json']), + sourceSha256: values['source-sha256'], + restartSpec: values['restart-spec'] === undefined + ? undefined : parseRuntimeControlSpec(JSON.parse(values['restart-spec'])), + applicationFailure: values['application-failure-json'] === undefined + ? undefined : JSON.parse(values['application-failure-json']), + parentAttemptId: values['parent-attempt-id'], bundleArtifactId: '' }; + if (!a.app || !a.url || !a.backend || !a.label) { + console.error('Usage: node dist/commands/run-suite.js --app --url --backend --label [--out ] [--media] [--no-reset]'); + process.exit(2); + } + if (!['scored', 'observed'].includes(a.observation)) { + throw new Error('--observation must be scored or observed'); + } + if (a.observation === 'observed' && !/^[a-f0-9]{64}$/.test(a.sourceSha256 ?? '')) { + throw new Error('observed specifications require --source-sha256'); + } + if (a.sourceSha256 !== undefined && !/^[a-f0-9]{64}$/.test(a.sourceSha256)) { + throw new Error('--source-sha256 must be a SHA-256 digest'); + } + if (a.retryInconclusive && (!a.sourceSha256 + || (!a.reset && STACK_ADAPTER_REGISTRY.get(a.backend).runPolicy.resetEnabled))) { + throw new Error('suite recovery requires a frozen --source-sha256 and reset for stateful backends'); + } + if (a.applicationFailure && (a.applicationFailure.kind !== 'app_failure' + || typeof a.applicationFailure.phase !== 'string' || !a.applicationFailure.phase + || typeof a.applicationFailure.reason !== 'string' || !a.applicationFailure.reason)) { + throw new Error('--application-failure-json must describe an application failure'); + } + a.out = privateGradingDirectory(a.app, a.out); + if (!Array.isArray(a.regressionChecks) + || a.regressionChecks.some(key => typeof key !== 'string' || !key)) { + throw new Error('--regression-checks-json must contain stable check keys'); + } + return a; +} + +export function selectObservationScope(selectedTask: BoundRecipeTaskRequestResult | null, + observation: Observation = 'scored'): Selection | null { + if (observation === 'scored') return selectedTask?.selection ?? null; + if (observation !== 'observed') throw new Error(`unknown observation scope ${observation}`); + if (!selectedTask || !isModularRecipeTaskRequest(selectedTask)) { + throw new Error('observed specifications require a modular schema-3 task request'); + } + const selection = selectedTask.selection; + if (!selection.observedChecks.length) throw new Error('observed specification scope is empty'); + return { + ...selection, + observation: 'observed', + checks: selection.observedChecks, + scoredPoints: 0, + observedPoints: selection.observedChecks.reduce((total, check) => total + check.points, 0), + }; +} + +export function attachRegressionScope(selection: Selection | null, recipeBinding: RecipeBinding | null, + declaredSuites: DeclaredSuite[], stableKeys: string[] = []): Selection | null { + if (!stableKeys.length) return selection; + if (!selection || !recipeBinding?.release?.checkCatalog) { + throw new Error('regression checks require a recipe-bound scored selection'); + } + const uniqueKeys = [...new Set(stableKeys)]; + if (uniqueKeys.length !== stableKeys.length) throw new Error('regression checks contain duplicates'); + const currentKeys = new Set(selection.checks.map(check => check.stableKey)); + const catalog = new Map(recipeBinding.release.checkCatalog + .map(check => [check.stableKey, check])); + const inheritedSuites = new Set(declaredSuites.filter(suite => suite.inherited) + .map(suite => suite.id)); + const regressionChecks = uniqueKeys.map(key => { + if (currentKeys.has(key)) throw new Error(`regression check ${key} is already in the current score`); + const check = catalog.get(key); + if (!check) throw new Error(`regression check ${key} is absent from the cumulative recipe`); + if (!inheritedSuites.has(check.executionId)) { + throw new Error(`regression check ${key} does not belong to an inherited execution`); + } + return { ...check, treatment: check.treatment ?? 'regression' }; + }); + const evaluationDocument = { schemaVersion: 1, selectionSha256: selection.sha256, + regressionChecks: uniqueKeys.slice().sort() }; + return { + ...selection, + checks: [...selection.checks, ...regressionChecks], + regressionChecks: regressionChecks.map(check => ({ ...check, treatment: check.treatment ?? 'regression' })), + regressionPoints: regressionChecks.reduce((total, check) => total + check.points, 0), + evaluationSha256: sha256(Buffer.from(canonicalDefinitionJson(evaluationDocument))), + }; +} + +const COMMAND_TIMEOUT_MS = GRADER_SOURCE_TIMEOUT_MS; +const run = (cmd: string, args: readonly string[], opts: Omit = {}): string => + execFileSync(cmd, args, { + encoding: 'utf8', stdio: 'pipe', cwd: ROOT, timeout: COMMAND_TIMEOUT_MS, ...opts, + }); + +export async function verifyApplicationProbe(url: string, { + fetchImpl = fetch, timeoutMs = 5000, +}: { fetchImpl?: ApplicationFetch; timeoutMs?: number } = {}): Promise { + let response; + try { + response = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) }); + } catch (error) { + // A probe timeout measures nothing; only a refusal or HTTP error is app evidence. + return { ok: false, ...(error instanceof Error && error.name === 'TimeoutError' ? { timedOut: true } : {}), + detail: `application did not respond: ${error instanceof Error ? error.message : String(error)}` }; + } + if (!response.ok) { + return { ok: false, detail: `application returned HTTP ${response.status}` }; + } + return { ok: true, detail: null }; +} + +export async function waitForApplicationProbe(url: string, { + attempts = 9, intervalMs = 250, probeTimeoutMs = 1000, + probe = verifyApplicationProbe, sleepImpl = sleep, +}: { attempts?: number; intervalMs?: number; probeTimeoutMs?: number; + probe?: typeof verifyApplicationProbe; + sleepImpl?: (ms: number) => Promise } = {}): Promise { + if (!Number.isInteger(attempts) || attempts < 1) { + throw new Error('application probe attempts must be a positive integer'); + } + let result = null; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + result = await probe(url, { timeoutMs: probeTimeoutMs }); + if (result.ok || attempt === attempts) return result; + await sleepImpl(intervalMs); + } + return result ?? { ok: false, detail: 'application readiness probe did not run' }; +} + +// Confirm the app uses the database leased to this run. +export function checkDatabaseProvenance(args: Pick): DatabaseProvenance { + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + // Source text cannot prove which database the running app uses. Leased runs + // prove it with an application write, including container-local endpoints. + if (args.databaseLease && 'proveUse' in adapter.database) { + return { ok: true, reason: 'leased database requires runtime marker verification' }; + } + const expected = adapter.ports.allocations().db; + if (!expected) return { ok: true, reason: 'no external database for this backend' }; + // Neutral guidance does not prescribe project layout. Search the app for the + // connection string instead of assuming it is in server/.env. + const urls: string[] = []; + let usesLeasedEnvironment = false; + const walk = (dir: string): void => { + if (!existsSync(dir)) return; + for (const e of readdirSync(dir, { withFileTypes: true })) { + if (/^(node_modules|dist|\.vite|\.git|module_bindings)$/.test(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) { walk(p); continue; } + if (!/\.(env|ts|tsx|js|mjs|json|yaml|yml)$|^\.env/.test(e.name)) continue; + try { + const text = readFileSync(p, 'utf8'); + if (/process\.env(?:\.DATABASE_URL|\[['"]DATABASE_URL['"]\])/.test(text)) { + usesLeasedEnvironment = true; + } + urls.push(...adapter.agent.findDatabaseUrls({ text })); + } catch { /* unreadable file proves nothing */ } + } + }; + walk(args.app); + if (usesLeasedEnvironment) { + return { ok: true, url: 'process.env.DATABASE_URL', + reason: 'app reads the database URL supplied by its authenticated backend lease' }; + } + if (!urls.length) return { ok: false, + reason: 'app neither reads process.env.DATABASE_URL nor contains a database connection string' }; + const matchesExpectedPort = (value: string): boolean => { + try { return Number(new URL(value).port) === Number(expected); } + catch { return false; } + }; + const ok = urls.some(matchesExpectedPort); + return { ok, url: urls[0], + reason: ok ? 'ok' : `app targets ${urls[0]} but the benchmark database is on port ${expected}` }; +} + +export async function writeApplicationDatabaseMarker( + args: Pick, + definition: DatabaseProvenanceDefinition, + { browser: suppliedBrowser }: { browser?: Browser } = {}, +): Promise { + if (!definition) throw new Error('track does not define runtime database provenance'); + const marker = `sb${randomUUID().replaceAll('-', '').slice(0, 16)}`; + const browser = suppliedBrowser ?? (args.browserWsEndpoint + ? await chromium.connect(args.browserWsEndpoint) + : await chromium.launch({ headless: true, ...attemptBrowserLaunchOptions() })); + try { + const context = await browser.newContext(); + try { + const page = await context.newPage(); + page.setDefaultTimeout(8000); + const actor = new Actor('database-provenance', page, context); + await actor.ready; + await runApplicationNavigation(() => page.goto(args.url, { waitUntil: 'domcontentloaded', timeout: 20000 }), page); + const evidence = await executeAction(ACTION_REGISTRY, definition.browserAction, + { do: definition.browserAction, actor: actor.name, name: marker, exact: true }, { + capabilities: { + actors: new Map([[actor.name, actor]]), + 'browser-interaction': { defaultWithin: 8000, + roomName: (name: string) => name, scopedUser: (name: string) => name, + testId: stableElementSelector, + sleep: (ms: number, signal: AbortSignal) => sleep(ms, undefined, { signal }), + }, + }, + onAbort: () => context.close(), + }); + if (evidence.status === 'passed') return { ok: true, marker }; + if (evidence.status === 'failed') return { ok: false, marker: null, + reason: evidence.summary ?? 'application signup failed during database provenance' }; + throw new Error(`database provenance signup ${evidence.status}: ${evidence.summary}`); + } finally { await context.close(); } + } catch (error) { + if (error instanceof ActionApplicationFailure) return { ok: false, marker: null, reason: error.message }; + throw error; + } finally { if (!suppliedBrowser) await browser.close(); } +} + +export function databaseProvenanceFailure(error: unknown): { kind: string; phase: string; reason: string } { + return { kind: 'harness_failure', phase: 'database-provenance', + reason: `runtime database provenance failed: ${error instanceof Error ? error.message : String(error)}` }; +} + +// A successful browser/provider flow is not evidence of an application database write. +export async function verifyApplicationDatabaseMarker( + args: Pick, + definition: DatabaseProvenanceDefinition, + { write = writeApplicationDatabaseMarker, read = checkRuntimeDatabaseProvenance } = {}, +): Promise<{ write: ProvenanceWrite; runtime: RuntimeProvenance | null }> { + const result = await write(args, definition); + return { write: result, runtime: result.ok ? read(args, result.marker) : null }; +} + +export function checkRuntimeDatabaseProvenance(args: Pick, + marker: string | null = null): RuntimeProvenance { + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + if (!('proveUse' in adapter.database)) { + return { ok: null, verified: false, + reason: 'exact runtime database marker proof is not implemented for this stack' }; + } + if (!args.databaseLease) { + return { ok: null, verified: false, + reason: 'standalone grading has no authenticated database lease' }; + } + if (typeof marker !== 'string' || !marker) { + return { ok: null, verified: false, + reason: 'the application action did not produce a database marker' }; + } + if (args.backend === 'spacetime') { + return STACK_ADAPTER_REGISTRY.get('spacetime').database.proveUse( + { lease: requireLeasedSpacetime(args.databaseLease), marker }); + } + if (args.backend === 'convex') { + return STACK_ADAPTER_REGISTRY.get('convex').database.proveUse({ lease: args.databaseLease, marker }); + } + const lease = requireLeasedDatabase(args.databaseLease); + return args.backend === 'mongodb' + ? STACK_ADAPTER_REGISTRY.get('mongodb').database.proveUse({ lease, marker }) + : STACK_ADAPTER_REGISTRY.get('postgres').database.proveUse({ lease, marker }); +} + +function isGradePayload(value: GradePayload | LintPayload | null | undefined): value is GradePayload { + return value !== null && value !== undefined && 'total' in value && 'max' in value; +} + +// Report the application size and direct runtime dependency count. +export function codeMetrics(args: Pick): { serverLoc: number; serverFiles: number; + totalLoc: number; totalFiles: number; runtimeDeps: number } { + // Minimal-guidance apps may place server code outside the conventional directory. + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const conventional = adapter.agent.serverDirectory; + const SERVER_DIR = existsSync(join(args.app, conventional)) ? conventional : '.'; + const walk = (dir: string, out: string[] = []): string[] => { + if (!existsSync(dir)) return out; + for (const e of readdirSync(dir, { withFileTypes: true })) { + if (/^(node_modules|dist|\.vite|module_bindings|drizzle)$/.test(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) walk(p, out); + // Count supported source files; agent-planted links and FIFOs are not read. + else if (e.isFile() && /\.(ts|tsx|js|jsx|mjs|cjs)$/.test(e.name)) out.push(p); + } + return out; + }; + const count = (files: string[]): number => files.reduce((n, f) => n + readFileSync(f, 'utf8').split('\n').length, 0); + // With no conventional server directory, "server" is everything that is not + // the client — otherwise the fallback counts the client twice and serverLoc + // equals totalLoc, which reads as a much larger backend than was written. + const allFiles = walk(args.app); + const serverFiles = SERVER_DIR === '.' + ? allFiles.filter(f => !/[\\/]client[\\/]/.test(f)) + : walk(join(args.app, SERVER_DIR)); + + let deps = 0; + const packageFiles = new Set([ + resolve(args.app, 'package.json'), + resolve(args.app, SERVER_DIR, 'package.json'), + resolve(args.app, 'client/package.json'), + ]); + for (const p of packageFiles) { + if (!existsSync(p)) continue; + try { deps += Object.keys(JSON.parse(readFileSync(p, 'utf8')).dependencies ?? {}).length; } catch { /* ignore */ } + } + + return { + serverLoc: count(serverFiles), serverFiles: serverFiles.length, + totalLoc: count(allFiles), totalFiles: allFiles.length, + runtimeDeps: deps, + }; +} + +export function findMutationBackups(app: string, { readDir = readdirSync }: + { readDir?: MutationDirectoryReader } = {}): string[] { + const backups: string[] = []; + const walk = (dir: string): void => { + let entries; + try { + entries = readDir(dir, { withFileTypes: true }); + } catch (error) { + // Vite atomically replaces transient dependency directories while the + // app runs. They are not source and may vanish between parent and child + // reads; a missing directory cannot contain a mutation backup. + if (isRecord(error) && error.code === 'ENOENT') return; + throw error; + } + for (const entry of entries) { + if (/^(node_modules|dist|\.vite|\.git|module_bindings)$/.test(entry.name)) continue; + const path = join(dir, entry.name); + if (entry.isDirectory()) walk(path); + else if (entry.isFile() && entry.name.endsWith('.mutation-backup')) backups.push(path); + } + }; + walk(app); + return backups; +} + +function resetDatabase(args: RunArguments): { ok: boolean; detail: string | null; + outcome: { kind: string; phase: string; appFailures?: string[] } | null } { + process.stdout.write(' reset database ... '); + try { + run(process.execPath, [RESET, args.backend, args.app]); + console.log('ok'); + } catch (err) { + console.log('FAILED'); + const failure: Failure = err instanceof Error ? err : new Error(String(err)); + const detail = childFailureDetail(failure, failure.stdout); + console.log(` ${detail}`); + return { ok: false, detail, outcome: resetFailureOutcome(failure) }; + } + return { ok: true, detail: null, outcome: null }; +} + +export function contractLintArgv(args: ContractLintArguments, + selectedTask: BoundRecipeTaskRequestResult | null = null): string[] { + const interfaces = selectedTask ? contractInterfaceNames(selectedTask.task.contractText) : []; + const out = join(args.out, ARTIFACT_FILE.contractLint); + return [compiledEntrypoint('linter', 'lint.js'), '--url', args.url, '--level', args.level, + '--track', args.track, '--label', args.label, '--out', out, + '--parent-attempt-id', args.bundleArtifactId, + ...(args.credentialAliases + ? ['--credential-aliases-json', JSON.stringify(args.credentialAliases)] : []), + ...(selectedTask ? ['--selected-hooks'] : []), + ...interfaces.flatMap(id => ['--hook', id])]; +} + +function lint(args: RunArguments, selectedTask: BoundRecipeTaskRequestResult | null = null): LintPayload | null { + process.stdout.write(' contract lint ... '); + const out = join(args.out, ARTIFACT_FILE.contractLint); + rmSync(out, { force: true }); + let failure: unknown = null; + try { + run('node', contractLintArgv(args, selectedTask)); + } catch (error) { failure = error; /* hook failures still write a report */ } + if (!existsSync(out)) { + const output = failure && typeof failure === 'object' && 'stdout' in failure + ? String(failure.stdout ?? '') : undefined; + const detail = failure instanceof Error + ? childFailureDetail(failure, output) : null; + throw new Error(`contract lint produced no report${detail ? `: ${detail}` : ''}`); + } + const r = readArtifactPayload(out, { expectedKind: 'contract_lint' }); + console.log(r.pass + ? r.counts.pass > 0 + ? `PASS (${r.counts.pass} interfaces)` + : r.counts.scenario > 0 + ? `DEFERRED (${r.counts.scenario} interfaces checked during feature grading)` + : 'NO STANDALONE INTERFACES SELECTED' + : `FAIL (${r.counts.fail} failed, ${r.counts.blocked} blocked)`); + return r; +} + +// Named write actions let concurrency checks issue authenticated operations +// without prescribing one transport. Missing actions are reported explicitly. +function checkActions(args: RunArguments): ActionsPayload | null { + process.stdout.write(` ${'actions'.padEnd(10)} ... `); + const out = join(args.out, ARTIFACT_FILE.actions); + rmSync(out, { force: true }); + try { + run('node', [compiledEntrypoint('commands', 'check-actions.js'), '--backend', args.backend, + '--url', args.url, '--app', args.app ?? '.', '--track', args.track, '--out', out, '--quiet', + '--parent-attempt-id', args.bundleArtifactId]); + } catch { /* non-zero exit means something is missing; the report still lands */ } + if (!existsSync(out)) { console.log('NO REPORT'); return null; } + const r = readArtifactPayload(out, { expectedKind: 'action_check' }); + if (!r.missing.length) { console.log(`all ${r.results.length} present`); return r; } + console.log(`${r.missing.length} MISSING — ${r.missing.join(', ')}`); + return r; +} + +export function suiteMayRetry(grade: GradePayload): boolean { + if (grade.cleanupEvidence?.status === 'harness_failure') return false; + if (grade.features.some(feature => feature.cleanupEvidence?.status === 'harness_failure' + || feature.cleanupEvidence?.failures.length)) return false; + const evidence = grade.features.flatMap(feature => feature.criteria.map(criterion => criterion.evidence)); + return evidence.some(value => value?.status === 'inconclusive' && value.retryable) + && evidence.every(value => value?.status === 'passed' + || (value?.status === 'inconclusive' && value.retryable)); +} + +async function gradeSuite(args: RunArguments, suite: DeclaredSuite, track: Track, + recipeBinding: RecipeBinding | null, selectedTask: BoundRecipeTaskRequestResult | null, + bundleArtifactId: string, selectedChecks: RecipeCheck[] = [], + { recordSelection = true, captureMedia = true, outputDirectory = args.out }: { + recordSelection?: boolean; captureMedia?: boolean; outputDirectory?: string; + } = {}): Promise { + process.stdout.write(` ${suite.id.padEnd(10)} ... `); + mkdirSync(outputDirectory, { recursive: true }); + const out = join(outputDirectory, `grading-${suite.id}.json`); + rmSync(out, { force: true }); + const argv = [compiledEntrypoint('grader', 'grade.js'), '--url', args.url, '--level', args.level, + '--label', `${args.label}-${suite.id}${outputDirectory === args.out ? '' : '-retry'}`, '--out', out]; + if (suite.spec) argv.push('--spec', suite.spec); + argv.push('--backend', args.backend, '--track', args.track); + if (recipeBinding) argv.push('--expected-recipe-sha256', recipeBinding.release.contentSha256); + const requestedRecipe = args.recipe ?? (args.recipeTask + ? args.recipeTask.recipe.id : null); + if (requestedRecipe) argv.push('--recipe', requestedRecipe); + if (selectedTask) argv.push('--recipe-task-json', JSON.stringify(selectedTask.request)); + for (const check of selectedChecks) argv.push('--selected-check', check.stableKey); + if (args.credentialAliases) { + argv.push('--credential-aliases-json', JSON.stringify(args.credentialAliases)); + } + if (recordSelection && args.selection?.sha256) { + argv.push('--selection-sha256', args.selection.evaluationSha256 ?? args.selection.sha256); + } + argv.push('--parent-attempt-id', bundleArtifactId); + // The out-of-band write goes straight to this run's database, with no + // app code in the loop; only the harness knows which one that is. + argv.push('--db-name', databaseNameForGrading(track, args.runIndex ?? 0, + args.databaseLease?.resources.database ? args.databaseLease : null)); + if (args.restartSpec) argv.push('--restart-spec', JSON.stringify(args.restartSpec)); + // The systems criteria run scripts the app itself ships (back-office writes), + // so the grader has to know where the app lives. + if (args.app) argv.push('--app', args.app); + if (captureMedia && args.media) argv.push('--media', join(outputDirectory, 'media'), '--trace'); + else if (captureMedia) argv.push('--failure-media', join(outputDirectory, 'failure-media')); + if (args.browserWsEndpoint) argv.push('--browser-ws-endpoint', args.browserWsEndpoint); + const child = await runGraderChild(argv, outputDirectory, suite.id, + gradingSourceTimeoutMs(recipeBinding?.plan.packs ?? [], selectedChecks)); + const { stdout, failure } = child; + if (!existsSync(out)) { + console.log('NO REPORT'); + const detail = childFailureDetail(failure, stdout); + throw new Error(`grader produced no report for ${suite.id}${detail ? `: ${detail}` : ''}; ` + + `full diagnostics: ${child.stdoutName}, ${child.stderrName}`); + } + const r = readArtifactPayload(out, { expectedKind: 'grade' }); + if (failure) { + throw new Error(`grader process did not complete for ${suite.id}: ${childFailureDetail(failure, stdout)}; ` + + `full diagnostics: ${child.stdoutName}, ${child.stderrName}`); + } + if (selectedChecks.length) { + const expected = selectedChecks.map(check => check.stableKey).sort(); + const reported = (r.selection?.checks ?? []).map(check => check.stableKey).sort(); + if (JSON.stringify(reported) !== JSON.stringify(expected)) { + throw new Error(`grader report scope differs from requested suite scope for ${suite.id}`); + } + } + console.log(`${r.total}/${r.max}`); + for (const f of r.features) { + for (const c of f.criteria.filter(c => !evidencePassed(criterionEvidence(c)))) { + console.log(` ${renderEvidenceConsoleLine(criterionEvidence(c), `${f.name} / ${c.id}`, { + includeSummary: false, + })}`); + } + } + // Disclose passes that lack server-side confirmation. + const uiOnly = r.features.flatMap(f => + f.criteria.filter(c => evidencePassed(criterionEvidence(c)) && c.serverCheck === 'unverified') + .map(c => `${f.name}/${c.id}`)); + if (uiOnly.length) { + console.log(` note: ${uiOnly.length} criterion/criteria passed on interface behaviour only`); + for (const u of uiOnly) console.log(` ${u} — server-side check not runnable on this backend`); + } + return r; +} + +export async function closeSuiteBrowser(browser: Pick | null, + bundle: Pick, persist: () => unknown): Promise { + try { await browser?.close(); } + catch (error) { + bundle.error = `grader browser shutdown failed: ${error instanceof Error ? error.message : String(error)}`; + bundle.outcome = { kind: 'harness_failure', phase: 'grading-cleanup', reason: bundle.error }; + persist(); + throw error; + } +} + +export function preserveStartFailure(error: unknown, out: string): void { + if (isRecord(error) && typeof error.startLog === 'string' && error.startLog) { + writeFileSync(join(out, 'application-start.log'), redactCredentials(error.startLog) + '\n'); + } +} + +async function main() { + const startedAt = new Date().toISOString(); + const args = parseArgs(process.argv); + args.databaseLease = databaseLeaseForGrading(args.backend); + const track = loadTrack(args.track); + const recipeBinding = resolveRecipeRelease(track, Number(args.level), args.recipeTask?.recipe ?? args.recipe); + if (!recipeBinding && (args.packIds.length || args.checkKeys.length)) { + throw new Error('--pack and --check require a recipe-bound level'); + } + const selectedTask = recipeBinding + ? (args.recipeTask + ? resolveBoundRecipeTaskRequest(recipeBinding, args.recipeTask) + : createBoundRecipeTaskRequest(recipeBinding, args)) + : null; + let selection = selectObservationScope(selectedTask, args.observation); + if (args.sourceSha256) { + const source = hashAppSource(args.app); + if (source.sha256 !== args.sourceSha256) { + throw new Error('live application source differs from the source selected for grading'); + } + } + const declaredSuites = recipeBinding + ? suitesForRecipe(track, recipeBinding) + : suitesFor(track, Number(args.level)); + if (args.observation === 'scored') { + selection = attachRegressionScope(selection, recipeBinding, declaredSuites, + args.regressionChecks); + } else if (args.regressionChecks.length) { + throw new Error('observed grading cannot include regression checks'); + } + args.selection = selection; + if (selection) { + const suiteIds = new Set(declaredSuites.map(suite => suite.id)); + const unmapped = selection.checks.filter(check => !suiteIds.has(check.executionId)); + if (unmapped.length) { + throw new Error(`selected recipe checks do not map to a declared suite: ${ + unmapped.map(check => check.stableKey).join(', ')}`); + } + } + const calibration = resolveCalibrationForRelease(recipeBinding?.release ?? null, { + trackRoot: track.dir, + stackBenchRoot: ROOT, + alias: `L${args.level}`, + }); + const observationSuffix = args.observation === 'observed' ? '-observed' : ''; + const bundleArtifactId = `${args.parentAttemptId ?? args.label}-grade-bundle-l${args.level}${observationSuffix}`; + args.bundleArtifactId = bundleArtifactId; + mkdirSync(args.out, { recursive: true }); + // Remove all prior grade output before writing cumulative evidence. + clearPreviousGradeOutputs(args.out); + + console.log(`\n=== ${args.label} (${args.backend}) ===`); + console.log(` app: ${args.app}`); + console.log(` url: ${args.url}`); + if (recipeBinding && selection) { + console.log(` recipe: ${recipeBinding.alias} -> ${recipeBinding.release.id} ` + + `(${recipeBinding.release.contentSha256.slice(0, 12)})`); + console.log(args.observation === 'observed' + ? ` scope: ${selection.checks.length} observed check(s), ${selection.observedPoints} observed point(s), 0 score contribution` + : ` scope: ${selection.checks.length} check(s), ${selection.scoredPoints} point(s)`); + if (selection.requested.packs?.length) console.log(` packs: ${selection.requested.packs.join(', ')}`); + if (selection.requested.features?.length) { + console.log(` features: ${selection.requested.features.join(', ')}`); + } + if (selection.requested.checks.length) console.log(` extra checks: ${selection.requested.checks.join(', ')}`); + } + + const bundle: Bundle = { + definitionSchemaVersion: track.schemaVersion, + recipeRelease: bundleRecipeRelease(recipeBinding), + calibration: calibration ? { id: calibration.id, + contentSha256: calibration.contentSha256 } : null, + label: args.label, track: args.track, backend: args.backend, url: args.url, app: args.app, + level: Number(args.level), observation: args.observation, + ...(args.sourceSha256 ? { source: { sha256: args.sourceSha256 } } : {}), + suites: {}, totals: {}, phaseTimings: [], + selection: selection ? { ...selection, attemptedChecks: [], reportedChecks: [], notRun: [] } : null, + }; + const selectedPackIds = new Set(selection?.checks.map(check => check.packId) ?? []); + const selectedPackDefinitions = recipeBinding?.plan.packs + .filter(pack => selectedPackIds.has(pack.id)) ?? []; + const writeBundle = () => { + const writeStarted = performance.now(); + if (args.sourceSha256) { + const current = hashAppSource(args.app); + if (current.sha256 !== args.sourceSha256) { + bundle.error = 'application source changed while grading was in progress'; + bundle.outcome = { kind: 'harness_failure', phase: 'source-provenance', + reason: bundle.error }; + } + } + const result = writeArtifact(join(args.out, ARTIFACT_FILE.gradeBundle), { + kind: 'grade_bundle', + id: bundleArtifactId, + attempt: { id: bundleArtifactId, parentId: args.parentAttemptId ?? null }, + timestamps: { startedAt, completedAt: new Date().toISOString() }, + identities: recipeArtifactIdentities(recipeBinding?.release ?? null, { + calibration: calibration ? { id: calibration.id, + sha256: calibration.contentSha256 } : null, + stackAdapter: { id: args.backend }, + }), + payload: bundle, + }); + console.log(` evidence write and source verification ... ${(performance.now() - writeStarted).toFixed(1)}ms`); + return result; + }; + const recordApplicationAbort = () => { + const passed = new Set(Object.values(bundle.suites ?? {}).flatMap(suite => + (isGradePayload(suite) ? suite.features : []).flatMap(feature => feature.criteria ?? []) + .flatMap(criterion => criterion.evidence?.status === 'passed' && criterion.stableKey ? [criterion.stableKey] : []))); + bundle.totals = applicationFailureTotals(selection, declaredSuites, passed); + }; + const freshenFailureMessage = () => { + const detail = lastResetFailure ? `: ${lastResetFailure}` : ''; + if (lastResetOutcome?.phase !== 'application-readiness') { + return `database reset failed — scores would not be comparable${detail}`; + } + return lastResetOutcome.kind === 'harness_failure' + ? `application server stopped by the grader was not restored${detail}` + : `application did not become ready after database reset${detail}`; + }; + const markRemainingNotRun = (reason: string): void => { + if (!bundle.selection) return; + const accounted = new Set([ + ...bundle.selection.attemptedChecks, + ...bundle.selection.notRun.map(check => check.stableKey), + ]); + bundle.selection.notRun.push(...bundle.selection.checks + .filter(check => !accounted.has(check.stableKey)) + .map(check => ({ stableKey: check.stableKey, reason }))); + }; + + if (args.applicationFailure) { + bundle.error = args.applicationFailure.reason; + bundle.outcome = args.applicationFailure; + recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + // Reset before each stateful check. + let lastResetFailure: string | null = null; + let lastResetOutcome: ResetOutcome = { kind: 'harness_failure', phase: 'database-reset' }; + // Set when a grader stopped the application server and could not start it + // again. Until the harness starts it, the app cannot be blamed for being + // unreachable. + let applicationLeftStopped = false; + let timingSuite: string | null = null; + const measure = (phase: string, work: () => T | Promise) => + measurePhase(bundle.phaseTimings, phase, timingSuite, work); + const freshen = async () => { + if (!args.reset) return true; + const requiresReseed = STACK_ADAPTER_REGISTRY.get(args.backend).reset.requiresReseed; + const restartSpec = args.restartSpec; + if (track.reseedOnReset && requiresReseed && !restartSpec) { + lastResetFailure = `track ${args.track} requires --restart-spec to initialize the app after reset`; + lastResetOutcome = { kind: 'harness_failure', phase: 'application-reset-control' }; + return false; + } + if (track.reseedOnReset && restartSpec && requiresReseed) { + process.stdout.write(' stop application ... '); + try { + await measure('stop', () => controlAppServer(restartSpec, 'stop')); + applicationLeftStopped = true; + console.log('ok'); + } catch (error) { + const failure: Failure = error instanceof Error ? error : new Error(String(error)); + lastResetFailure = childFailureDetail(failure); + lastResetOutcome = { kind: 'harness_failure', phase: 'application-reset-control' }; + console.log(`FAILED (${lastResetFailure})`); + return false; + } + } + const reset = await measure('reset', () => resetDatabase(args)); + lastResetFailure = reset.detail; + lastResetOutcome = reset.outcome ?? { kind: 'harness_failure', phase: 'database-reset' }; + if (!reset.ok) return false; + // Do not grade until the reset application is reachable. + const waitUntilReady = async () => { + const ready = await measure('readiness', () => waitForApplicationProbe(args.url)); + if (!ready.ok) { + lastResetFailure = ready.detail; + lastResetOutcome = applicationLeftStopped + ? { kind: 'harness_failure', phase: 'application-readiness' } + : ready.timedOut ? { kind: 'inconclusive', phase: 'application-readiness' } + : { kind: 'app_failure', phase: 'application-readiness', + appFailures: ['application-readiness'] }; + console.log(`FAILED (${ready.detail})`); + return false; + } + console.log('ok'); + return true; + }; + if (track.reseedOnReset && restartSpec && requiresReseed) { + process.stdout.write(' restart ... '); + // Judge restart success with the readiness probe. The restart command can + // leave a long-running server process behind, so the command also needs a deadline. + try { + // Do not give a background server an inherited pipe that keeps the + // synchronous restart command open. + await measure('start', () => controlAppServer(restartSpec, 'start')); + applicationLeftStopped = false; + } catch (err) { + preserveStartFailure(err, args.out); + const failure: Failure = err instanceof Error ? err : new Error(String(err)); + lastResetOutcome = resetFailureOutcome(failure); + const detail = ((failure.stderr || '') + (failure.stdout || '') + (failure.message || '')) + .toString().trim().split('\n').slice(-3).join(' | ').slice(0, 300); + lastResetFailure = detail || null; + console.log('FAILED (application did not restart)'); + console.log(` control: ${JSON.stringify(restartSpec)}`); + console.log(` ${detail}`); + return false; + } + return await waitUntilReady(); + } + process.stdout.write(' ready ... '); + return await waitUntilReady(); + }; + + bundle.code = codeMetrics(args); + console.log(` code ... ${bundle.code.serverLoc} server LOC in ${bundle.code.serverFiles} files, ` + + `${bundle.code.totalLoc} total LOC, ${bundle.code.runtimeDeps} runtime deps`); + + // Refuse source left modified by an interrupted mutation run. + const mutated = findMutationBackups(args.app); + if (mutated.length) { + bundle.error = `app still carries mutation backups (${mutated.join(', ')}) — its source is mutated, not the build under test`; + bundle.outcome = { kind: 'harness_failure', phase: 'mutation-cleanup', reason: bundle.error }; + markRemainingNotRun('run aborted because application source is still mutated'); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + const prov = checkDatabaseProvenance(args); + bundle.provenance = prov; + console.log(` database ... ${prov.ok ? prov.reason : `WRONG DATABASE — ${prov.reason}`}`); + if (!prov.ok) { + bundle.error = `app is not using the benchmark database: ${prov.reason}`; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance', reason: bundle.error, + appFailures: ['database-provenance'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because database provenance was invalid'); + writeBundle(); + console.log('\nABORTED: results would not describe the benchmark environment.'); + process.exit(1); + } + + if (args.observation === 'scored') { + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + let runtime = checkRuntimeDatabaseProvenance(args); + let proofError = null; + let actionFailure: string | null = null; + const proof = track.databaseProvenance; + const supportsRuntimeProof = 'proveUse' in STACK_ADAPTER_REGISTRY.get(args.backend).database; + const requiresRuntimeProof = supportsRuntimeProof && args.databaseLease && args.reset; + if (requiresRuntimeProof && !proof) { + proofError = new Error(`${args.track} does not define a runtime database provenance check`); + } else if (requiresRuntimeProof && proof) { + try { + const result = await verifyApplicationDatabaseMarker(args, proof); + if (!result.write.ok) actionFailure = result.write.reason; + else if (result.runtime) runtime = result.runtime; + } catch (error) { + proofError = error; + } + + // The proof writes unique data through the application. Remove it before + // linting and scored grading so the proof cannot change the result. + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + } else if (supportsRuntimeProof && !args.reset) { + runtime = { ok: null, verified: false, + reason: 'runtime marker proof requires database reset to isolate its write' }; + } + + if (!proofError && !actionFailure && supportsRuntimeProof && args.databaseLease && !runtime.verified) { + proofError = new Error(`leased database identity was not verified: ${runtime.reason}`); + } + if (proofError) { + bundle.outcome = databaseProvenanceFailure(proofError); + bundle.error = bundle.outcome.reason; + markRemainingNotRun('run aborted because runtime database provenance could not be verified'); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + if (actionFailure) { + bundle.error = actionFailure; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance-action', + reason: actionFailure, appFailures: ['database-provenance-action'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because the application database write failed'); + writeBundle(); + console.log(`\nABORTED: ${actionFailure}`); + process.exit(1); + } + + bundle.provenance.runtime = runtime; + console.log(` db runtime ... ${runtime.verified + ? runtime.ok ? runtime.reason : `WRONG DATABASE — ${runtime.reason}` + : runtime.reason}`); + if (runtime.ok === false) { + bundle.error = `app did not write its marker to the benchmark database: ${runtime.reason}`; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance', reason: bundle.error, + appFailures: ['database-provenance'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because runtime database provenance failed'); + writeBundle(); + console.log('\nABORTED: application data came from outside the benchmark database.'); + process.exit(1); + } + try { + bundle.suites.lint = lint(args, selectedTask); + } catch (error) { + markRemainingNotRun('run aborted after contract lint failed to produce evidence'); + bundle.error = error instanceof Error ? error.message : String(error); + bundle.outcome = { kind: 'harness_failure', phase: 'contract-lint', reason: bundle.error }; + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + bundle.actions = checkActions(args); + } + + // Keep current-level score separate from earlier guarantee regressions. + let total = 0, max = 0, regTotal = 0, regMax = 0; + const dirty = false; + let browserServer: BrowserServer | null = null; + try { + if (declaredSuites.some(suite => !selection + || selection.checks.some(check => check.executionId === suite.id))) { + browserServer = await chromium.launchServer({ headless: true, ...attemptBrowserLaunchOptions() }); + args.browserWsEndpoint = browserServer.wsEndpoint(); + } + for (const suite of declaredSuites) { + timingSuite = suite.id; + const selectedChecks = selection?.checks.filter(check => check.executionId === suite.id) ?? []; + if (selection && selectedChecks.length === 0) { + console.log(` ${suite.id.padEnd(10)} ... not selected`); + continue; + } + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + console.log(` ${suite.id}: SKIPPED (${bundle.error})`); + markRemainingNotRun(`run aborted: ${bundle.error}`); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + throw new Error(bundle.error); + } + if (bundle.selection) { + bundle.selection.attemptedChecks.push(...selectedChecks.map(check => check.stableKey)); + } + let r; + try { + r = await measure('grader', () => gradeSuite(args, suite, track, recipeBinding, selectedTask, bundleArtifactId, selectedChecks)); + bundle.suites[suite.id] = r; + if (args.retryInconclusive && suiteMayRetry(r)) { + const recoveryDirectory = join(args.out, 'suite-retries', suite.id); + const initialDirectory = join(recoveryDirectory, 'initial'); + mkdirSync(initialDirectory, { recursive: true }); + for (const name of [`grading-${suite.id}.json`, `grader-${suite.id}.stdout.log`, `grader-${suite.id}.stderr.log`]) { + cpSync(join(args.out, name), join(initialDirectory, name)); + } + (bundle.suiteRetries ??= {})[suite.id] = r; + writeBundle(); + if (bundle.outcome?.kind === 'harness_failure') throw new Error(bundle.error); + console.log(` ${suite.id}: retrying this inconclusive suite once; completed suites are retained`); + if (!(await freshen())) throw new Error(`suite recovery could not restore fresh state: ${freshenFailureMessage()}`); + // Reset/start must not modify the source selected for either execution. + if (hashAppSource(args.app).sha256 !== args.sourceSha256) { + throw new Error('application source changed before suite recovery'); + } + r = await measure('grader', () => gradeSuite(args, suite, track, recipeBinding, selectedTask, + bundleArtifactId, selectedChecks, { outputDirectory: recoveryDirectory })); + // Keep the established final-grade paths for raw evidence consumers. + for (const name of [`grading-${suite.id}.json`, `grader-${suite.id}.stdout.log`, `grader-${suite.id}.stderr.log`, 'media', 'failure-media']) { + if (existsSync(join(recoveryDirectory, name))) cpSync(join(recoveryDirectory, name), join(args.out, name), { recursive: true }); + } + } + } catch (error) { + markRemainingNotRun(`run aborted after ${suite.id} grader failure`); + bundle.error = error instanceof Error ? error.message : String(error); + bundle.outcome = { kind: 'harness_failure', phase: `grade:${suite.id}`, reason: bundle.error }; + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + throw error; + } + bundle.suites[suite.id] = r; + if (isGradePayload(r) && r.features.some(feature => feature.cleanupEvidence?.failures + .some(failure => failure.stage === 'application-restore'))) { + applicationLeftStopped = true; + } + if (bundle.selection) { + bundle.selection.reportedChecks.push(...selectedChecks.map(check => check.stableKey)); + } + if (selection) { + bundle.packRuntime = aggregatePackRuntime( + [...Object.values(bundle.suites).filter(isGradePayload), ...Object.values(bundle.suiteRetries ?? {})], + selectedPackDefinitions); + const exceeded = exceededPackBudgets(bundle.packRuntime); + if (exceeded.length) { + // Runtime budgets qualify references; generated apps still receive a complete grade. + console.log(` runtime ... ${exceeded.map(pack => + `${pack.id} ${pack.measuredRuntimeMs}ms > ${pack.budget.maxRuntimeMs}ms`) + .join(', ')} [recorded; grading continues]`); + } + } + if (suite.inherited) { regTotal += r.total; regMax += r.max; } + else { total += r.total; max += r.max; } + } + } finally { + args.browserWsEndpoint = undefined; + await closeSuiteBrowser(browserServer, bundle, writeBundle); + } + + bundle.totals = { + score: total, max, dirty, contractPass: isGradePayload(bundle.suites.lint) + ? null : bundle.suites.lint?.pass ?? null, + // null rather than 0/0 at L1, where there is nothing earlier to regress. + regression: regMax ? { score: regTotal, max: regMax } : null, + }; + writeBundle(); + + console.log(` ${'TOTAL'.padEnd(10)} ... ${total}/${max}${dirty ? ' [DIRTY]' : ''}`); + if (regMax) { + const kept = regTotal === regMax ? 'all earlier guarantees still hold' : `${regMax - regTotal} EARLIER GUARANTEE(S) LOST`; + console.log(` ${'REGRESSION'.padEnd(10)} ... ${regTotal}/${regMax} — ${kept}`); + } + console.log(` bundle: ${join(args.out, ARTIFACT_FILE.gradeBundle)}`); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/test-loop.ts b/tools/stack-bench/commands/test-loop.ts new file mode 100644 index 00000000000..3192f2ce3ef --- /dev/null +++ b/tools/stack-bench/commands/test-loop.ts @@ -0,0 +1,310 @@ +#!/usr/bin/env node +import { privateGradingDirectory } from '../src/evidence/repair-evidence.js'; + +import { execFileSync, spawnSync } from 'node:child_process'; +import { readFileSync, existsSync, rmSync, mkdirSync, mkdtempSync, readdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { ARTIFACT_FILE, readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js'; +import type { ArtifactIdentities } from '../src/evidence/artifacts.js'; +import type { CostRun, CostSession } from '../src/evidence/cost-proof.js'; +import type { PublicBackendLease } from '../src/runtime/backend-lease.js'; +import type { RepairLevel, RepairOutcome } from '../src/runtime/repair-grant.js'; +import type { LevelCheckpoint } from '../src/runtime/source-checkpoint.js'; +import { CODING_CONTAINER_BUG_REPORT_FILE } from '../src/runtime/coding-container-policy.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const WORK = mkdtempSync(join(tmpdir(), 'stack-bench-loop-')); +const APP = join(WORK, 'app'); +// A cold Playwright start plus two grades can exceed three minutes on Windows +// Docker hosts. The timeout is a deadlock guard, not a performance assertion. +const BENCH_TIMEOUT_MS = 300_000; + +interface LoopSession extends CostSession { + sessionId?: string; + tokens?: number; + turns?: number; + durationMs?: number; +} + +interface LoopLevel extends RepairLevel { + buildSessions?: LoopSession[]; + repairSessions?: LoopSession[]; + resumeSession?: LoopSession; + contractPass?: boolean; + stalled?: boolean; + code?: { totalLoc?: number }; + sessionTotals?: { sessions?: number; tokens?: number; turns?: number; durationMs?: number }; +} + +interface LoopRun extends CostRun { + id?: string; + levels?: LoopLevel[]; + outcome?: RepairOutcome; + artifactEnvelope?: { identities?: ArtifactIdentities }; + backendLease?: PublicBackendLease; + totals?: CostRun['totals'] & { max?: number; sessions?: number; tokens?: number; turns?: number; + modelDurationMs?: number; durationSec?: number }; +} + +interface GradeFeature { + id?: string; + setupEvidence?: { schemaVersion?: number; status?: string }; + criteria?: { evidence?: { schemaVersion?: number; status?: string; actions?: unknown[] } }[]; +} + +interface GradePayload { features?: GradeFeature[]; } +interface SourceCheckpointPayload { source: LevelCheckpoint; } +interface RepairContinuation { + baseline?: { reproduced?: boolean; score?: number; sourceSha256?: string }; + cumulativeRepairsBefore?: number; + cumulativeRepairsAfter?: number; + resumeSetup?: { sourceVerified?: boolean }; +} +interface RepairContinuationPayload extends LoopRun { continuation?: RepairContinuation; } + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function processOutput(error: unknown): string { + if (!isRecord(error)) return String(error); + return `${String(error.stdout ?? '')}${String(error.stderr ?? '')}`; +} + +// A failed assertion or interrupted CI job must not leave a fixture app that a +// later loop can mistake for its own output. +process.on('exit', () => rmSync(WORK, { recursive: true, force: true })); + +let failures = 0; +const check = (name: string, ok: boolean, detail = ''): void => { + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${ok || !detail ? '' : ` — ${detail}`}`); + if (!ok) failures += 1; +}; + +function runBench(extra: string[] = []): string { + const argv = [compiledEntrypoint('commands', 'bench.js'), '--backend', 'stub', '--levels', '1', + '--agent-adapter', 'deterministic', + '--app', APP, '--out', WORK, + '--track', 'loop', + '--url', `file:///${join(APP, 'index.html').replace(/\\/g, '/')}`, ...extra]; + try { + return execFileSync('node', argv, { + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + timeout: BENCH_TIMEOUT_MS, + killSignal: 'SIGTERM', + }); + } catch (error: unknown) { return processOutput(error); } +} + +const invalidRounds = spawnSync('node', [compiledEntrypoint('commands', 'bench.js'), '--backend', 'stub', + '--repairs', '1.5'], + { encoding: 'utf8' }); +check('fractional correction budgets are rejected before a run starts', + invalidRounds.status !== 0 + && /--repairs must be a non-negative safe integer/.test(invalidRounds.stderr)); + +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); + +console.log('\nLoop test — one repair available'); +const out = runBench(['--repairs', '1']); +const runPath = join(WORK, ARTIFACT_FILE.run); + +check(`the benchmark run produced ${ARTIFACT_FILE.run}`, existsSync(runPath)); +if (!existsSync(runPath)) { + console.log(`\ncannot continue without ${ARTIFACT_FILE.run}`); + process.exit(1); +} + +const run = readArtifactPayload(runPath); +const level = run.levels?.[0]; +const evidenceDir = privateGradingDirectory(APP); +const bundleArtifact = readArtifact(join(evidenceDir, ARTIFACT_FILE.gradeBundle), + { expectedKind: 'grade_bundle' }); +const lintArtifact = readArtifact(join(evidenceDir, ARTIFACT_FILE.contractLint), + { expectedKind: 'contract_lint' }); +const actionArtifact = readArtifact(join(evidenceDir, ARTIFACT_FILE.actions), + { expectedKind: 'action_check' }); +const gradeArtifact = readArtifact(join(evidenceDir, 'grading-features.json'), { expectedKind: 'grade' }); +const leaseArtifact = readArtifact(join(WORK, ARTIFACT_FILE.backendLease), + { expectedKind: 'backend_lease_evidence' }); +const checkpointArtifact = readArtifact(join(WORK, 'level-l1-checkpoint.json'), + { expectedKind: 'source_checkpoint' }); + +check('recorded exactly one level', run.levels?.length === 1); +check('run and level carry structured outcomes', + typeof run.outcome?.kind === 'string' && run.outcome.kind === level?.outcome?.kind, + `run=${run.outcome?.kind} level=${level?.outcome?.kind}`); +check('artifacts carry the producing run id', typeof run.id === 'string' && run.id.length > 10); +check('run envelope identifies engine, agent adapter, and stack adapter', + /^[a-f0-9]{64}$/.test(run.artifactEnvelope?.identities?.engine?.sha256 ?? '') + && /^[a-f0-9]{64}$/.test(run.artifactEnvelope?.identities?.agentAdapter?.sha256 ?? '') + && run.artifactEnvelope?.identities?.stackAdapter?.id === 'stub'); +check('bundle is a child of the run', bundleArtifact.attempt.parentId === run.id, + JSON.stringify(bundleArtifact.attempt)); +check('public lease evidence is a child of the run', leaseArtifact.attempt.parentId === run.id, + JSON.stringify(leaseArtifact.attempt)); +check('level source checkpoint is hash-bound and linked to the run', + checkpointArtifact.attempt.parentId === run.id + && level?.checkpoint?.artifact === 'level-l1-checkpoint.json' + && level.checkpoint.sha256 === checkpointArtifact.payload.source.sha256 + && /^[a-f0-9]{64}$/.test(level.checkpoint.sha256) + && existsSync(join(WORK, level.checkpoint.directory)), + JSON.stringify(level?.checkpoint)); +check('lint, action, and grade evidence are children of the bundle', + [lintArtifact, actionArtifact, gradeArtifact] + .every(artifact => artifact.attempt.parentId === bundleArtifact.attempt.id)); +const gradedFeatures = gradeArtifact.payload?.features ?? []; +check('grade artifacts retain typed setup, criterion, and action evidence', + gradedFeatures.length > 0 + && gradedFeatures.every(feature => feature.setupEvidence?.schemaVersion === 1 + && (feature.criteria ?? []).every(criterion => criterion.evidence?.schemaVersion === 1 + && Array.isArray(criterion.evidence.actions))), + JSON.stringify(gradedFeatures.map(feature => ({ id: feature.id, + setup: feature.setupEvidence?.status, + criteria: feature.criteria?.map(criterion => criterion.evidence?.status) })))); +const publicJson = [runPath, join(WORK, ARTIFACT_FILE.backendLease), + join(evidenceDir, ARTIFACT_FILE.gradeBundle), join(evidenceDir, ARTIFACT_FILE.contractLint), + join(evidenceDir, ARTIFACT_FILE.actions), join(evidenceDir, 'grading-features.json')] + .map(path => readFileSync(path, 'utf8')).join('\n'); +check('public envelopes contain no secret or lease-token fields', + !/"(?:apiKey|leaseToken|ownershipToken|password|secret)"\s*:/i.test(publicJson)); +check('backend lease was released', + ['released', 'stopped'].includes(run.backendLease?.state ?? '') + && (run.backendLease?.resources?.locks?.every(lock => lock.releasedAt) ?? false), + JSON.stringify(run.backendLease?.state)); +check('a repair ran', level?.repairs === 1, `repairs=${level?.repairs}`); +check('successful repair is explicit', level?.repair?.status === 'corrected' + && level.repair.limit === 1 && level.repair.used === 1 + && level.repair.stopReason === 'passed', + JSON.stringify(level?.repair)); +const reportPath = join(APP, CODING_CONTAINER_BUG_REPORT_FILE); +const reportExists = existsSync(reportPath); +check('the bug report was written', reportExists); +// Behavioural findings must never reveal how they were detected, or a fix can +// target the check instead of the app. Missing-control findings are exempt: +// there the element id is the requirement. +const report = reportExists ? readFileSync(reportPath, 'utf8') : ''; +const behaviourSection = report.split('## Application interface')[0] ?? ''; +check('behavioural findings do not leak selectors or timings', + !/data-(?:role|testid)|locator|within \d+ms/.test(behaviourSection)); +check('missing interfaces are reported separately', /## Application interface/.test(report)); +check('build and fix costs are both recorded', + (level?.buildCostUsd ?? 0) > 0 && (level?.repairCostUsd ?? 0) > 0, + `build=${level?.buildCostUsd} fix=${level?.repairCostUsd}`); +check('build and fix sessions remain individually auditable', + level?.buildSessions?.length === 1 + && level.buildSessions[0]?.sessionId === 'stub-build' + && level?.repairSessions?.length === 1 + && level.repairSessions[0]?.sessionId === 'stub-fix', + JSON.stringify({ builds: level?.buildSessions, fixes: level?.repairSessions })); +check('level session totals include the build and fix', + level?.sessionTotals?.sessions === 2 + && level.sessionTotals.tokens === 2000 + && level.sessionTotals.turns === 5 + && level.sessionTotals.durationMs === 100, + JSON.stringify(level?.sessionTotals)); +check('grading produced a score out of a maximum', Number.isInteger(level?.score) && (level?.max ?? 0) > 0, + `${level?.score}/${level?.max}`); +check('code metrics captured', Boolean(level?.code) && typeof level?.code?.totalLoc === 'number', + JSON.stringify(level?.code)); +check('totals aggregate the levels', run.totals?.max === level?.max); +check('run totals aggregate every model session', + run.totals?.sessions === 2 && run.totals.tokens === 2000 + && run.totals.turns === 5 && run.totals.modelDurationMs === 100, + JSON.stringify(run.totals)); +check('wall time recorded', (run.totals?.durationSec ?? -1) >= 0); +check('the fix improved the contract lint', + /APPLICATION CONTRACT FAIL[\s\S]*APPLICATION CONTRACT PASS/.test(out) + || level?.contractPass === true, + 'expected the broken fixture to fail the lint and the fixed one to pass'); + +console.log('\nLoop test — zero repairs allowed'); +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); +runBench(['--repairs', '0']); +const capped = readArtifactPayload(runPath); +check('no fix ran when the cap is zero', capped.levels?.[0]?.repairs === 0); +check('no bug report was written when no fix is allowed', + !existsSync(join(APP, CODING_CONTAINER_BUG_REPORT_FILE))); + +console.log('\nLoop test - flat corrections exhaust their declared budget'); +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); +runBench(['--repairs', '2', '--model', 'deterministic-stall']); +const exhausted = readArtifactPayload(runPath); +const exhaustedLevel = exhausted.levels?.[0]; +check('both correction rounds ran after the first flat result', exhaustedLevel?.repairs === 2, + `repairs=${exhaustedLevel?.repairs}`); +check('an unresolved app records budget exhaustion', exhaustedLevel?.repair?.status === 'budget-exhausted' + && exhaustedLevel.repair.limit === 2 && exhaustedLevel.repair.used === 2 + && exhaustedLevel.repair.stopReason === 'budget-exhausted' + && exhaustedLevel.stalled === true && exhausted.outcome?.kind === 'app_failure', + JSON.stringify({ repair: exhaustedLevel?.repair, outcome: exhausted.outcome })); + +console.log('\nLoop test - a later finite grant continues the exact exhausted source'); +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); +runBench(['--repairs', '2', '--model', 'deterministic-deferred']); +const parentBefore = readFileSync(runPath, 'utf8'); +const deferred = readArtifactPayload(runPath); +check('the deferred parent exhausted its original two-round budget', + deferred.levels?.[0]?.repair?.status === 'budget-exhausted' + && deferred.levels[0].repair.used === 2, + JSON.stringify(deferred.levels?.[0]?.repair)); +let continuationOutput = ''; +try { + continuationOutput = execFileSync('node', [join(ROOT, 'dist', 'commands', 'repair-cli.js'), 'grant', WORK, + '--level', '1', '--repairs', '2', '--timeout-minutes', '10'], { + encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: BENCH_TIMEOUT_MS, + }); +} catch (error: unknown) { + continuationOutput = processOutput(error); +} +const continuationRoot = join(WORK, 'continuations'); +const continuationDirectories = existsSync(continuationRoot) + ? readdirSync(continuationRoot, { withFileTypes: true }).filter(entry => entry.isDirectory()) : []; +const continuationDirectory = continuationDirectories.length === 1 + ? join(continuationRoot, continuationDirectories[0]?.name ?? '') : null; +const continuationPath = continuationDirectory + ? join(continuationDirectory, ARTIFACT_FILE.run) : null; +check('repair grant produced one linked continuation', + continuationPath !== null && existsSync(continuationPath), continuationOutput.slice(-2000)); +if (continuationDirectory && continuationPath && existsSync(continuationPath)) { + const continuationArtifact = readArtifact(continuationPath, { expectedKind: 'repair_continuation' }); + const continuation = continuationArtifact.payload; + const continuedLevel = continuation.levels?.[0]; + const continuationDetails = continuation.continuation; + const deferredLevel = deferred.levels?.[0]; + check('continuation reproduced the exact failed baseline before spending a repair', + continuationDetails?.baseline?.reproduced === true + && continuationDetails.baseline.score === deferredLevel?.score + && continuationDetails.baseline.sourceSha256 === deferredLevel?.checkpoint?.sha256, + JSON.stringify(continuationDetails?.baseline)); + check('continuation reached correctness inside its finite added budget', + continuation.outcome?.kind === 'passed' + && continuedLevel?.repair?.status === 'corrected' + && continuedLevel.repair.used === 1 + && continuationDetails?.cumulativeRepairsBefore === 2 + && continuationDetails?.cumulativeRepairsAfter === 3, + JSON.stringify({ repair: continuedLevel?.repair, continuation: continuationDetails })); + check('resume setup is visible, separately costed, and does not consume a repair', + continuationDetails?.resumeSetup?.sourceVerified === true + && continuedLevel?.resumeSession?.sessionId === 'stub-resume' + && (continuedLevel?.resumeCostUsd ?? 0) > 0 + && continuedLevel.repairs === 1, + JSON.stringify({ setup: continuationDetails?.resumeSetup, + resume: continuedLevel?.resumeSession, fixes: continuedLevel?.repairs })); + check('continuation process outcome is retained as a typed child artifact', + readArtifact(join(continuationDirectory, ARTIFACT_FILE.process), + { expectedKind: 'repair_process' }).attempt.parentId === deferred.id); +} +check('grant left the original run artifact byte-for-byte unchanged', + readFileSync(runPath, 'utf8') === parentBefore); + +rmSync(WORK, { recursive: true, force: true }); +console.log(`\n${failures === 0 ? 'loop OK' : `${failures} check(s) failed`}`); +process.exit(failures === 0 ? 0 : 1); diff --git a/tools/stack-bench/conditions/catalog.json b/tools/stack-bench/conditions/catalog.json new file mode 100644 index 00000000000..c88e06cf670 --- /dev/null +++ b/tools/stack-bench/conditions/catalog.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "kind": "study-condition-catalog", + "guidanceProfiles": { + "model-free-stub": "guidance/model-free-stub.json", + "prescribed": "guidance/prescribed.json", + "neutral": "guidance/neutral.json", + "neutral-dev": "guidance/neutral-dev.json", + "neutral-dev-no-sdk": "guidance/neutral-dev-no-sdk.json", + "neutral-managed-dev": "guidance/neutral-managed-dev.json", + "neutral-no-sdk": "guidance/neutral-no-sdk.json" + }, + "repairPolicies": { + "scored-only": "repairs/scored-only.json" + } +} diff --git a/tools/stack-bench/conditions/guidance/model-free-stub.json b/tools/stack-bench/conditions/guidance/model-free-stub.json new file mode 100644 index 00000000000..ffd536b8c02 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/model-free-stub.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "model-free-stub", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": false, + "designAdvice": false + }, + "documents": { + "stub": "backends/model-free-stub.md" + }, + "applicationInterfaces": { + "stub": "http" + }, + "skills": { + "stub": [] + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral-dev-no-sdk.json b/tools/stack-bench/conditions/guidance/neutral-dev-no-sdk.json new file mode 100644 index 00000000000..0533576765a --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral-dev-no-sdk.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral-dev-no-sdk", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "spacetime-dev" + ], + "convex": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral-dev.json b/tools/stack-bench/conditions/guidance/neutral-dev.json new file mode 100644 index 00000000000..0c7ccd8f137 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral-dev.json @@ -0,0 +1,39 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral-dev", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "typescript-server", + "typescript-client", + "cli", + "spacetime-dev" + ], + "convex": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral-managed-dev.json b/tools/stack-bench/conditions/guidance/neutral-managed-dev.json new file mode 100644 index 00000000000..d2c02221c36 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral-managed-dev.json @@ -0,0 +1,39 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral-managed-dev", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "typescript-server", + "typescript-client", + "cli", + "spacetime-managed-dev" + ], + "convex": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral-no-sdk.json b/tools/stack-bench/conditions/guidance/neutral-no-sdk.json new file mode 100644 index 00000000000..26deb8bee1c --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral-no-sdk.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral-no-sdk", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [], + "convex": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral.json b/tools/stack-bench/conditions/guidance/neutral.json new file mode 100644 index 00000000000..3f559d8cc36 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "typescript-server", + "typescript-client", + "cli" + ], + "convex": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/prescribed.json b/tools/stack-bench/conditions/guidance/prescribed.json new file mode 100644 index 00000000000..dafb4f90f80 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/prescribed.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "prescribed", + "mode": "prescribed", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/mongodb.md", + "postgres": "backends/postgres.md", + "spacetime": "backends/spacetime.md", + "convex": "backends/convex.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "typescript-server", + "typescript-client", + "cli" + ], + "convex": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/repairs/scored-only.json b/tools/stack-bench/conditions/repairs/scored-only.json new file mode 100644 index 00000000000..e3fc69b24af --- /dev/null +++ b/tools/stack-bench/conditions/repairs/scored-only.json @@ -0,0 +1,8 @@ +{ + "schemaVersion": 1, + "kind": "repair-policy", + "id": "scored-only", + "scoredEvidence": true, + "observedEvidence": false, + "scenarioValues": "failed-observations" +} diff --git a/tools/stack-bench/container/Dockerfile b/tools/stack-bench/container/Dockerfile new file mode 100644 index 00000000000..eb169e54d3a --- /dev/null +++ b/tools/stack-bench/container/Dockerfile @@ -0,0 +1,50 @@ +# The image a generated app is built in. +# +# The generated app must not see the harness, grader, or test definitions. +# +# No harness, grader, or test definition is copied in or mounted at run time. +# An adapter can mount only its selected stack artifacts, read-only. +# Keep the readable tag, but bind the base to an exact manifest. Campaigns use +# the digest of the completed image, so every attempt runs the same artifact. +FROM node:22-slim@sha256:f86be15afa9a8277608e141ce2a8aa55d3d9c40845921b8511f4fb7897be2554 + +# git: builds initialise repositories and some tooling shells out to it. +# curl: readiness probes against the app's own dev server. +# ca-certificates: TLS for npm and the API. +# procps: the build starts and stops its own dev servers. +# lsof: `kill-port` locates a listener with lsof on Linux, and finds nothing +# without it — it then prints "Process on port N killed" and exits 0 while the +# server keeps running. Every durability and deploy-window test would pass +# without restarting anything, which is worse than failing. None of lsof, fuser, +# ss or netstat is present in node:22-slim. +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates procps lsof chromium util-linux \ + && rm -rf /var/lib/apt/lists/* + +# Browser testing uses the system browser; no downloads into the agent home are needed. +ENV CHROME_BIN=/usr/bin/chromium +COPY browser-tools/package*.json /opt/browser-tools/ +RUN npm ci --prefix /opt/browser-tools --omit=dev --ignore-scripts \ + && node -e "require('/opt/browser-tools/node_modules/puppeteer-core')" + +# Pinned, and the auto-updater disabled: a CLI that updates itself mid-series +# changes the thing under test between one backend and the next. Override at +# build time to move deliberately rather than by drift. +ARG CLAUDE_VERSION=2.1.226 +ARG CODEX_VERSION=0.153.4 +ENV DISABLE_AUTOUPDATER=1 +RUN npm install -g @anthropic-ai/claude-code@${CLAUDE_VERSION} \ + && claude --version +RUN npm install -g @openai/codex@${CODEX_VERSION} && codex --version + +# The coding session can change the app and its own temporary state. It does +# not run as root, so it cannot change the system or harness control files. +RUN useradd --uid 10001 --create-home --shell /bin/bash developer \ + && chmod 0700 /home/developer + +# The app under construction. Everything the build writes lives here, and the +# host mounts its own work directory over it. +WORKDIR /app + +# No ENTRYPOINT: the run-build command supplies the whole command so the prompt can go +# in on stdin exactly as it does on the host. diff --git a/tools/stack-bench/container/binary-provenance.ts b/tools/stack-bench/container/binary-provenance.ts new file mode 100644 index 00000000000..9686ca2db4c --- /dev/null +++ b/tools/stack-bench/container/binary-provenance.ts @@ -0,0 +1,207 @@ +#!/usr/bin/env node + +import { createHash, randomBytes } from 'node:crypto'; +import { existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { binarySourceIdentity, SOURCE_IDENTITY_SCHEME } + from '../src/releases/release-source.js'; +import { STACK_BENCH_RUNNER_PLATFORM } from '../src/runtime/runner-environment.js'; + +export const RUST_BUILDER_IMAGE = + 'rust:1.93-slim-bookworm@sha256:8f8609d448e821fbc0e44241bc5ca4ce49663cc6306ff1a17f655a0e2a7cd084'; +export const BINARY_NAMES = Object.freeze(['spacetimedb-cli', 'spacetimedb-standalone']); +const PROVENANCE_NAME = 'spacetimedb-binaries.json'; + +interface BinarySourceIdentity { + identityScheme: typeof SOURCE_IDENTITY_SCHEME; + revision: string; + sha256: string; + files: number; +} + +interface BinaryRecord { + sha256: string; + size: number; +} + +interface BinaryProvenance { + schemaVersion: 2; + platform: typeof STACK_BENCH_RUNNER_PLATFORM; + builderImage: string; + source: BinarySourceIdentity; + binaries: Record; +} + +function sha256File(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function binaryPath(stackBenchRoot: string, name: string): string { + return join(stackBenchRoot, 'container', 'bin', name); +} + +function provenancePath(stackBenchRoot: string): string { + return join(stackBenchRoot, 'container', PROVENANCE_NAME); +} + +function assertSha256(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || !/^[a-f0-9]{64}$/.test(value)) { + throw new Error(`${label} must be a SHA-256 digest`); + } +} + +function inspectBinary(path: string, name: string): BinaryRecord { + if (!existsSync(path)) { + throw new Error(`${name} is absent; run tools/stack-bench/container/build-linux-cli.sh`); + } + const stat = statSync(path); + if (!stat.isFile() || stat.size < 4) throw new Error(`${name} is not a non-empty file`); + const magic = readFileSync(path).subarray(0, 4); + if (!magic.equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) { + throw new Error(`${name} is not a Linux ELF binary`); + } + return { sha256: sha256File(path), size: stat.size }; +} + +export function createBinaryProvenance(stackBenchRoot: string, + source: BinarySourceIdentity): BinaryProvenance { + if (source?.identityScheme !== SOURCE_IDENTITY_SCHEME) { + throw new Error('binary source identity scheme is unsupported'); + } + assertSha256(source?.sha256, 'binary source identity'); + if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(source?.revision ?? '')) { + throw new Error('binary source revision must be an exact commit id'); + } + if (!Number.isSafeInteger(source?.files) || source.files < 1) { + throw new Error('binary source file count must be a positive integer'); + } + const binaries: Record = {}; + for (const name of BINARY_NAMES) binaries[name] = inspectBinary(binaryPath(stackBenchRoot, name), name); + return { + schemaVersion: 2, + platform: STACK_BENCH_RUNNER_PLATFORM, + builderImage: RUST_BUILDER_IMAGE, + source: { identityScheme: source.identityScheme, + revision: source.revision, sha256: source.sha256, files: source.files }, + binaries, + }; +} + +export function assertBinarySourceUnchanged(before: BinarySourceIdentity, + after: BinarySourceIdentity): void { + if (before?.identityScheme !== after?.identityScheme + || before?.revision !== after?.revision || before?.sha256 !== after?.sha256 + || before?.files !== after?.files) { + throw new Error('binary source changed during the build'); + } +} + +function readProvenance(stackBenchRoot: string): BinaryProvenance { + const path = provenancePath(stackBenchRoot); + if (!existsSync(path)) { + throw new Error(`${PROVENANCE_NAME} is absent; run tools/stack-bench/container/build-linux-cli.sh`); + } + let manifest: BinaryProvenance & { status?: string }; + try { manifest = JSON.parse(readFileSync(path, 'utf8')); } + catch (error) { + throw new Error(`${PROVENANCE_NAME} is not valid JSON: ${error instanceof Error + ? error.message : String(error)}`); + } + if (manifest.status === 'unbuilt') { + throw new Error(`${PROVENANCE_NAME} has no verified binaries; run tools/stack-bench/container/build-linux-cli.sh`); + } + return manifest; +} + +export function verifyBinaryProvenance(stackBenchRoot: string, + { sourceSha256 }: { sourceSha256: string }): BinaryProvenance { + assertSha256(sourceSha256, 'expected binary source identity'); + const manifest = readProvenance(stackBenchRoot); + if (manifest.schemaVersion !== 2) throw new Error('unsupported binary provenance schema'); + if (manifest.platform !== STACK_BENCH_RUNNER_PLATFORM) { + throw new Error(`binary provenance platform must be ${STACK_BENCH_RUNNER_PLATFORM}`); + } + if (manifest.builderImage !== RUST_BUILDER_IMAGE) { + throw new Error('binary provenance does not use the pinned Rust builder image'); + } + assertSha256(manifest.source?.sha256, 'recorded binary source identity'); + if (manifest.source.identityScheme !== SOURCE_IDENTITY_SCHEME) { + throw new Error('recorded binary source identity scheme is unsupported'); + } + if (manifest.source.sha256 !== sourceSha256) { + throw new Error('SpacetimeDB binaries do not match the selected release source'); + } + if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(manifest.source?.revision ?? '')) { + throw new Error('recorded binary source revision is invalid'); + } + if (!Number.isSafeInteger(manifest.source?.files) || manifest.source.files < 1) { + throw new Error('recorded binary source file count is invalid'); + } + for (const name of BINARY_NAMES) { + const expected = manifest.binaries?.[name]; + assertSha256(expected?.sha256, `${name} recorded checksum`); + if (!Number.isSafeInteger(expected.size) || expected.size < 4) { + throw new Error(`${name} recorded size is invalid`); + } + const actual = inspectBinary(binaryPath(stackBenchRoot, name), name); + if (actual.size !== expected.size) throw new Error(`${name} size does not match provenance`); + if (actual.sha256 !== expected.sha256) throw new Error(`${name} checksum does not match provenance`); + } + return manifest; +} + +function option(args: string[], name: string): string { + const index = args.indexOf(name); + const value = index === -1 ? undefined : args[index + 1]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function main(): void { + const [command, ...args] = process.argv.slice(2); + if (command === 'source') { + const repo = resolve(option(args, '--repo')); + console.log(JSON.stringify(binarySourceIdentity(repo), null, 2)); + return; + } + if (command === 'record') { + const repo = resolve(option(args, '--repo')); + const stackBenchRoot = join(repo, 'tools', 'stack-bench'); + const source = JSON.parse(readFileSync(resolve(option(args, '--source-file')), 'utf8')); + const current = binarySourceIdentity(repo); + assertBinarySourceUnchanged(source, current); + const manifest = createBinaryProvenance(stackBenchRoot, source); + const path = provenancePath(stackBenchRoot); + const temporary = `${path}.${process.pid}.${randomBytes(8).toString('hex')}.tmp`; + writeFileSync(temporary, `${JSON.stringify(manifest, null, 2)}\n`, { flag: 'wx' }); + try { renameSync(temporary, path); } + catch (error) { rmSync(temporary, { force: true }); throw error; } + console.log(`recorded ${path}`); + return; + } + if (command === 'record-snapshot') { + // The Docker source stage records this identity before compiling the same + // immutable Git archive. This path needs no host-built binary or Git metadata. + const root = resolve(option(args, '--root')); + const source = JSON.parse(readFileSync(resolve(option(args, '--source-file')), 'utf8')); + writeFileSync(provenancePath(root), `${JSON.stringify(createBinaryProvenance(root, source), null, 2)}\n`); + return; + } + if (command === 'verify') { + const stackBenchRoot = resolve(option(args, '--root')); + verifyBinaryProvenance(stackBenchRoot, { sourceSha256: option(args, '--source-sha256') }); + console.log('verified SpacetimeDB CLI and standalone binary provenance'); + return; + } + throw new Error('Usage: binary-provenance source --repo PATH | record --repo PATH --source-file PATH | verify --root PATH --source-sha256 SHA256'); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(`binary provenance failed: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } +} diff --git a/tools/stack-bench/container/broker-protocols.ts b/tools/stack-bench/container/broker-protocols.ts new file mode 100644 index 00000000000..9cab18b3a02 --- /dev/null +++ b/tools/stack-bench/container/broker-protocols.ts @@ -0,0 +1,285 @@ +import type { IncomingMessage, OutgoingHttpHeaders } from 'node:http'; +import { brotliDecompressSync, gunzipSync, inflateSync } from 'node:zlib'; +import { createParser } from 'eventsource-parser'; +import type { BrokerConfig } from './credential-broker-accounting.js'; + +const MAX_REQUEST_BYTES = 32 * 1024 * 1024; +type JsonRecord = Record; +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} +function isNumber(value: unknown): value is number { return typeof value === 'number'; } +function fail(message: string): never { throw new Error(`credential broker: ${message}`); } + +function upstreamHeaders(request: IncomingMessage, config: BrokerConfig): OutgoingHttpHeaders { + const headers: OutgoingHttpHeaders = { ...request.headers }; + delete headers.host; + // Request identity encoding so accounting and the client read the same bytes. + delete headers['accept-encoding']; + delete headers.authorization; + delete headers['proxy-authorization']; + delete headers['x-api-key']; + if (config.mode === 'api-key') headers['x-api-key'] = config.credential; + else headers.authorization = `Bearer ${config.credential}`; + return headers; +} + +function parseProviderRequest(body: Buffer, path: string, config: BrokerConfig): JsonRecord { + let payload: unknown; + try { payload = JSON.parse(body.toString('utf8')); } + catch { fail('request body must be valid JSON'); } + if (!isRecord(payload)) { + fail('request body must be an object'); + } + if (payload.model !== config.model) fail('request model does not match the selected model'); + if (path === '/v1/messages' + && (!isNumber(payload.max_tokens) || !Number.isInteger(payload.max_tokens) || payload.max_tokens < 1 + || payload.max_tokens > config.maxOutputTokens)) { + fail(`max_tokens must be from 1 through ${config.maxOutputTokens}`); + } + return payload; +} + +function decodedResponseBody(body: Buffer, contentEncoding: string | string[] | undefined): Buffer { + const encodings = String(contentEncoding ?? '') + .split(',').map(value => value.trim().toLowerCase()).filter(Boolean); + let decoded = body; + for (const encoding of encodings.reverse()) { + if (encoding === 'identity') continue; + const options = { maxOutputLength: MAX_REQUEST_BYTES }; + if (encoding === 'gzip' || encoding === 'x-gzip') decoded = gunzipSync(decoded, options); + else if (encoding === 'deflate') decoded = inflateSync(decoded, options); + else if (encoding === 'br') decoded = brotliDecompressSync(decoded, options); + else throw new Error(`unsupported response encoding ${encoding}`); + } + return decoded; +} + +function responseUsage(body: Buffer, contentEncoding: string | string[] | undefined = undefined): JsonRecord | null { + const values: JsonRecord[] = []; + const add = (value: unknown): void => { + if (!isRecord(value)) return; + if (isRecord(value.usage)) values.push(value.usage); + if (isRecord(value.message) && isRecord(value.message.usage)) values.push(value.message.usage); + }; + let text: string; + try { text = decodedResponseBody(body, contentEncoding).toString('utf8'); } + catch { return null; } + try { + add(JSON.parse(text)); + } catch { + let sawError = false; + let sawFinalUsage = false; + let sawMessageStop = false; + let parseError = false; + const parser = createParser({ + maxBufferSize: MAX_REQUEST_BYTES, + onError: () => { parseError = true; }, + onEvent: ({ data }) => { + if (!data || data === '[DONE]') return; + try { + const event = JSON.parse(data); + if (isRecord(event) && event.type === 'error') sawError = true; + if (isRecord(event) && event.type === 'message_delta' && isRecord(event.usage)) { + sawFinalUsage = true; + } + if (isRecord(event) && event.type === 'message_stop') sawMessageStop = true; + add(event); + } catch { /* Ignore non-JSON event data. */ } + }, + }); + try { parser.feed(`${text}\n\n`); } + catch { parseError = true; } + if (parseError || sawError || !sawFinalUsage || !sawMessageStop) return null; + } + if (values.length === 0) return null; + const number = (field: string): number => Math.max(0, ...values.map(value => Number(value[field]) || 0)); + const cacheWrite = (field: string): number => Math.max(0, ...values.map(value => + isRecord(value.cache_creation) ? Number(value.cache_creation[field]) || 0 : 0)); + const cacheWrite5m = cacheWrite('ephemeral_5m_input_tokens'); + const cacheWrite1h = cacheWrite('ephemeral_1h_input_tokens'); + const flatCacheWrite = number('cache_creation_input_tokens'); + return { + input_tokens: number('input_tokens'), + output_tokens: number('output_tokens'), + cache_read_input_tokens: number('cache_read_input_tokens'), + cache_creation: { + ephemeral_5m_input_tokens: cacheWrite5m + cacheWrite1h > 0 ? cacheWrite5m : flatCacheWrite, + ephemeral_1h_input_tokens: cacheWrite1h, + }, + }; +} + + +interface BrokerProtocol { + hostname: string; + allowedPaths: Set; + upstreamPath(path: string): string; + billable(path: string): boolean; + headers(request: IncomingMessage): OutgoingHttpHeaders; + parseRequest(body: Buffer, path: string): JsonRecord; + inputTokenAdjustment?(payload: JsonRecord): number; + outputLimit(payload: JsonRecord): number; + responseUsage(body: Buffer, encoding?: string | string[]): JsonRecord | null; +} + +function responsesUsage(body: Buffer, encoding?: string | string[], config?: BrokerConfig): JsonRecord | null { + let response: JsonRecord | null = null; + let failed = false; + let metadata: JsonRecord | null = null; + const accept = (value: unknown): void => { + if (!isRecord(value)) return; + if (isRecord(value.openrouter_metadata)) metadata = value.openrouter_metadata; + if (value.type === 'error' || value.type === 'response.failed') failed = true; + if ((value.type === 'response.completed' || value.type === 'response.incomplete' + || (config?.provider === 'openrouter' && value.type === 'response.done')) && isRecord(value.response)) { + response = value.response; + } else if (value.object === 'response' && (value.status === 'completed' || value.status === 'incomplete')) { + response = value; + } + }; + try { + const text = decodedResponseBody(body, encoding).toString('utf8'); + try { accept(JSON.parse(text)); } + catch { + const parser = createParser({ maxBufferSize: MAX_REQUEST_BYTES, + onError: () => { failed = true; }, + onEvent: ({ data }) => { + if (data === '[DONE]') return; + try { accept(JSON.parse(data)); } catch { failed = true; } + }, + }); + parser.feed(`${text}\n\n`); + } + } catch { return null; } + const usage = (response as JsonRecord | null)?.usage; + if (failed || !isRecord(usage)) return null; + const input = usage.input_tokens; + const output = usage.output_tokens; + const cached = isRecord(usage.input_tokens_details) ? usage.input_tokens_details.cached_tokens : 0; + if (![input, output, cached].every(value => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) + || (cached as number) > (input as number)) return null; + const normalized = { input_tokens: (input as number) - (cached as number), output_tokens: output, + cache_read_input_tokens: cached, cache_creation_input_tokens: 0 }; + if (config?.provider !== 'openrouter') return normalized; + const final = response as JsonRecord | null; + const route = (final?.openrouter_metadata ?? metadata) as JsonRecord | null; + const selected = isRecord(route?.endpoints) && Array.isArray(route.endpoints.available) + ? route.endpoints.available.filter(endpoint => isRecord(endpoint) && endpoint.selected === true) : []; + if (typeof usage.cost !== 'number' || !Number.isFinite(usage.cost) || usage.cost < 0 + || final?.model !== config.model || !route || route.requested !== config.model + || route.strategy !== 'direct' || route.is_byok !== false || route.attempt !== 1 + || (route.pipeline !== undefined && (!Array.isArray(route.pipeline) || route.pipeline.length !== 0)) + || selected.length !== 1 || !isRecord(selected[0]) || selected[0].model !== config.model + || typeof selected[0].provider !== 'string' || !selected[0].provider || selected[0].provider.length > 128) return null; + return { ...normalized, provider_reported_cost_usd: usage.cost, upstream_provider: selected[0].provider }; +} + + +function hasUnpricedInput(value: unknown): boolean { + if (Array.isArray(value)) return value.some(hasUnpricedInput); + if (!isRecord(value)) return false; + if (['input_file', 'item_reference'].includes(String(value.type))) return true; + return Object.values(value).some(hasUnpricedInput); +} + +// Only inline images have bounded input here. Provider receipts price actual tokens. +export function imageTokenAdjustment(value: unknown, model: string): number { + if (Array.isArray(value)) return value.reduce((sum, item) => sum + imageTokenAdjustment(item, model), 0); + if (!isRecord(value)) return 0; + if (value.type === 'input_image') { + if (!['gpt-6-astra', 'gpt-5.6-sol', 'gpt-5.6-terra', 'gpt-5.6-luna', 'gpt-5.4', 'gpt-5.4-2026-03-05'].includes(model) + || typeof value.image_url !== 'string' + || !/^data:image\/(png|jpeg|webp|gif);base64,[A-Za-z0-9+/]+={0,2}$/.test(value.image_url) + || value.file_id !== undefined) fail('image input requires inline data and a verified token bound'); + // OpenAI vision: 30,000 patches maximum x 1.2 tokens, plus rounding. + // https://developers.openai.com/api/docs/guides/images-vision + // Replace base64 text bytes rather than charging for both representations. + return 36_001 - Buffer.byteLength(value.image_url, 'utf8'); + } + return Object.values(value).reduce((sum, item) => sum + imageTokenAdjustment(item, model), 0); +} + +export function brokerProtocol(config: BrokerConfig): BrokerProtocol { + if (!config.provider || config.provider === 'anthropic') return { + hostname: 'api.anthropic.com', + allowedPaths: new Set(['/v1/messages', '/v1/messages/count_tokens']), + upstreamPath: path => path, + billable: path => path === '/v1/messages', + headers: request => upstreamHeaders(request, config), + parseRequest: (body, path) => parseProviderRequest(body, path, config), + outputLimit: payload => payload.max_tokens as number, + responseUsage, + }; + const router = config.provider === 'openrouter'; + if (router && (!/^[a-z0-9._-]+\/[a-zA-Z0-9._-]+$/.test(config.model) + || config.model.startsWith('openrouter/') || /(?:^|[-/])latest$/.test(config.model))) { + fail('OpenRouter requires one explicit model without routing variants'); + } + const account = config.mode === 'subscription-token'; + if (account && !config.accountId) fail('OpenAI account identity is required'); + // Account Responses does not promise max_output_tokens. Use a documented model + // bound, never assume an unknown model shares it. API requests enforce the cap. + const accountOutputLimits: Record = { 'gpt-5.3-codex': 128_000, 'gpt-5.4': 128_000, 'gpt-5.4-2026-03-05': 128_000, + 'gpt-5.6-sol': 128_000, 'gpt-6-astra': 128_000 }; + const outputLimit = account + ? Object.hasOwn(accountOutputLimits, config.model) ? accountOutputLimits[config.model] : undefined + : config.maxOutputTokens; + if (!outputLimit) fail('OpenAI account model has no verified output-token bound'); + return { + hostname: router ? 'openrouter.ai' : account ? 'chatgpt.com' : 'api.openai.com', + allowedPaths: new Set(['/v1/responses']), + upstreamPath: () => router ? '/api/v1/responses' : account ? '/backend-api/codex/responses' : '/v1/responses', + billable: () => true, + headers: request => { + // OpenRouter routing/auth headers are trusted configuration, not agent input. + const headers = router ? { 'content-type': 'application/json', accept: 'text/event-stream', + 'x-openrouter-metadata': 'enabled' } as OutgoingHttpHeaders : upstreamHeaders(request, config); + delete headers['x-api-key']; + for (const name of ['chatgpt-account-id', 'openai-organization', 'openai-project']) delete headers[name]; + headers.authorization = `Bearer ${config.credential}`; + if (account) headers['chatgpt-account-id'] = config.accountId; + return headers; + }, + parseRequest: body => { + const payload = JSON.parse(body.toString('utf8')); + if (!isRecord(payload) || payload.model !== config.model) fail('request model does not match'); + imageTokenAdjustment(payload.input, config.model); + // Token-only receipts cannot price hosted tools or hidden server-side input. + if (payload.prompt || payload.previous_response_id || payload.conversation || hasUnpricedInput(payload.input) + || payload.image_config !== undefined || payload.audio !== undefined + || (payload.modalities !== undefined && (!Array.isArray(payload.modalities) + || payload.modalities.some(modality => modality !== 'text'))) + || (payload.truncation !== undefined && payload.truncation !== 'disabled') + || payload.background === true + || (payload.service_tier !== undefined && payload.service_tier !== 'default' && payload.service_tier !== 'auto') + || (payload.tools !== undefined && (!Array.isArray(payload.tools) + || payload.tools.some(tool => !isRecord(tool) || !['function', 'custom'].includes(String(tool.type)))))) { + fail('request requires unsupported pricing or server-side state'); + } + if (router && ['provider', 'models', 'route', 'plugins', 'transforms', 'preset', 'user', 'session_id', 'trace', 'debug'].some(key => key in payload)) { + fail('OpenRouter routing and transforms are owned by the broker'); + } + const requested = payload.max_output_tokens; + if (requested !== undefined && (!Number.isSafeInteger(requested) || (requested as number) < 1 + || (requested as number) > outputLimit)) fail('invalid max_output_tokens'); + if (!account) { + payload.max_output_tokens = requested ?? outputLimit; + payload.service_tier = 'default'; + } + if (router) { + const rates = config.pricingRates!; + payload.provider = { only: [config.providerRoute], order: [config.providerRoute], + allow_fallbacks: false, require_parameters: true, + max_price: { prompt: rates.input, completion: rates.output, request: 0 } }; + payload.plugins = []; + payload.transforms = []; + payload.store = false; + } + return payload; + }, + inputTokenAdjustment: payload => imageTokenAdjustment(payload.input, config.model), + outputLimit: payload => account ? outputLimit : payload.max_output_tokens as number, + responseUsage: (body, encoding) => responsesUsage(body, encoding, config), + }; +} diff --git a/tools/stack-bench/container/browser-pipe.ts b/tools/stack-bench/container/browser-pipe.ts new file mode 100644 index 00000000000..15bc88d56f5 --- /dev/null +++ b/tools/stack-bench/container/browser-pipe.ts @@ -0,0 +1,63 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { Socket } from 'node:net'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { chromium } from 'playwright'; +import type { LaunchOptions } from 'playwright'; +import { compiledEntrypoint } from '../src/package-root.js'; +import { readBackendLease } from '../src/runtime/backend-lease.js'; + +function browserContainer(): string | null { + const path = process.env.STACK_BENCH_LEASE; + if (!path) { + if (process.env.STACK_BENCH_APPLIANCE === '1') throw new Error('browser requires a private attempt lease'); + return null; + } + const token = process.env.STACK_BENCH_LEASE_TOKEN; + if (!token) throw new Error('browser requires the attempt ownership token'); + const lease = readBackendLease(path, { token }); + if (lease.backend === 'stub') return null; + const container = lease.resources.browserContainer; + if (!container?.owned || container.running === false || !/^[a-f0-9]{64}$/.test(container.id)) { + throw new Error('browser requires the exact running attempt browser container'); + } + return container.id; +} + +export function attemptBrowserLaunchOptions(): LaunchOptions { + return browserContainer() ? { + executablePath: compiledEntrypoint('container', 'browser-pipe.js'), + // The owned browser has private shared memory; do not fill /tmp with IPC buffers. + ignoreDefaultArgs: ['--disable-dev-shm-usage'], + } : {}; +} + +// Playwright uses fd 3/4. Docker carries these bytes over stdin/stdout, so no +// browser control socket is reachable from the generated app's network. +function main(): void { + const id = browserContainer(); + if (!id) throw new Error('browser pipe requires an isolated attempt'); + const child = spawn('docker', ['exec', '-i', id, 'sh', '-c', + 'exec 3<&0 4>&1 1>&2; exec "$@"', 'sh', chromium.executablePath(), ...process.argv.slice(2)], + { stdio: ['pipe', 'pipe', 'inherit'] }); + const input = new Socket({ fd: 3, readable: true, writable: false }); + const output = new Socket({ fd: 4, readable: false, writable: true }); + input.pipe(child.stdin); + child.stdout.pipe(output); + const close = () => { input.destroy(); child.stdin.end(); }; + process.on('SIGTERM', close); + process.on('SIGINT', close); + child.stdin.on('error', error => { + if ('code' in error && error.code === 'EPIPE') close(); + else throw error; + }); + child.on('error', error => { console.error(error.message); process.exit(1); }); + child.on('exit', () => { input.destroy(); child.stdin.destroy(); }); + // fd 3 can still have a pending read while Playwright waits for this process + // to exit. Docker's close event means all browser output has been forwarded. + child.on('close', code => { process.exit(code ?? 1); }); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/container/browser-tools/package-lock.json b/tools/stack-bench/container/browser-tools/package-lock.json new file mode 100644 index 00000000000..cd5ab8d8f7e --- /dev/null +++ b/tools/stack-bench/container/browser-tools/package-lock.json @@ -0,0 +1,328 @@ +{ + "name": "stack-bench-browser-tools", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "stack-bench-browser-tools", + "dependencies": { + "puppeteer-core": "25.10.0" + } + }, + "node_modules/@puppeteer/browsers": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.2.2.tgz", + "integrity": "sha512-q2BU4YfO9h/Wt7IcWPcggpOOqLk2Tbs1hDwolvKZrweRjy751OJBKMN9zO5bfD0pzU7X/tvKw/exQds4pM/LOg==", + "license": "Apache-2.0", + "dependencies": { + "modern-tar": "^0.8.4", + "yargs": "^18.0.0" + }, + "bin": { + "browsers": "lib/main-cli.js" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "proxy-agent": ">=8.0.1", + "yauzl": "^2.10.0 || ^3.4.0" + }, + "peerDependenciesMeta": { + "proxy-agent": { + "optional": true + }, + "yauzl": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", + "integrity": "sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chromium-bidi": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-17.0.2.tgz", + "integrity": "sha512-5v9GQFhTktFvotn/OFNJBmKLKRAb6n9r0bVCwf7sHgWc3/JryK0bj1nn93L3pHFrfgcsu6Be6EWsDi+1XHTGDg==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "engines": { + "node": ">=20.19.0 <22.0.0 || >=22.12.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/cliui/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1666840", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1666840.tgz", + "integrity": "sha512-gCcO42XCHKEs7Ag0S7aGYsnJ7hlgrO3qderYqeiY0Eqk+0GFfuvT13IA0hHreJTa2KCdDVyGMeOhdMNmrrTjVg==", + "license": "BSD-3-Clause" + }, + "node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, + "node_modules/modern-tar": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.5.tgz", + "integrity": "sha512-snEhs+6G5Tjd4I7tLCDOaoln2RgE0bD19RzEKgvgK2hZ5VKy3MpLhLTZ2fWpXSTg4K2cyPwp+VHATFJhxfnOeA==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/puppeteer-core": { + "version": "25.10.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.10.0.tgz", + "integrity": "sha512-Hy5eMQshOEMil4JUUx03h5pw1HYkYCso1RG/gcpPlFSd4cYPOcopxcXEAxpLPOkOPJb9LIJtwxuj66bSdvknFg==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "3.2.2", + "chromium-bidi": "17.0.2", + "devtools-protocol": "0.0.1666840", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.3", + "ws": "^8.21.3" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/string-width": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.2.tgz", + "integrity": "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/typed-query-selector": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", + "license": "MIT" + }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.3.tgz", + "integrity": "sha512-uuN0goWfxP22B7J/uAgBpOYNPttC+XVseYE+rSY5+rQ+YBeVz/VORw8WbmLVcqW78zNg5A4qnjNXYUWR3il2ig==", + "license": "Apache-2.0" + }, + "node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "18.1.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.1.0.tgz", + "integrity": "sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==", + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^8.2.1", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/tools/stack-bench/container/browser-tools/package.json b/tools/stack-bench/container/browser-tools/package.json new file mode 100644 index 00000000000..5d7eeecaa9c --- /dev/null +++ b/tools/stack-bench/container/browser-tools/package.json @@ -0,0 +1 @@ +{"name":"stack-bench-browser-tools","private":true,"dependencies":{"puppeteer-core":"25.10.0"}} diff --git a/tools/stack-bench/container/build-container-inspection.ts b/tools/stack-bench/container/build-container-inspection.ts new file mode 100644 index 00000000000..22605d0a3f5 --- /dev/null +++ b/tools/stack-bench/container/build-container-inspection.ts @@ -0,0 +1,228 @@ +import { spawnSync } from 'node:child_process'; +import type { SpawnSyncOptionsWithStringEncoding, SpawnSyncReturns } from 'node:child_process'; +import { resolve } from 'node:path'; + +import { LEGACY_SUBSCRIPTION_TOKEN_TARGET } from './container-auth.js'; +import type { ContainerMount } from '../src/runtime/container-mount.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +type InspectedMount = { + type: string; + source: string; + name: string | null; + destination: string; + readOnly: boolean; +}; + +export type InspectedBuildContainer = { + id: string; + image: string; + running: boolean; + networkMode: string | null; + readonlyRootfs: boolean; + tmpfs: Record; + capAdd: string[]; + capDrop: string[]; + securityOpt: string[]; + pidsLimit: number | null; + nanoCpus: number | null; + memoryBytes: number | null; + memorySwapBytes: number | null; + mounts: InspectedMount[]; + unsafeCredentialExposure: boolean; +}; + +type DockerExecute = (command: string, args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding) => SpawnSyncReturns; + +type JsonRecord = Record; + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : []; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === 'number' ? value : null; +} + +function dockerDetail(result: SpawnSyncReturns): string { + return String(result.stderr || result.stdout || result.error?.message || `exit ${result.status}`).trim(); +} + +export function waitForBuildContainerReady(id: string, readyFile: string, description: string, { + env = process.env, + execute = spawnSync as DockerExecute, +}: { env?: NodeJS.ProcessEnv; execute?: DockerExecute } = {}): void { + const options = { encoding: 'utf8' as const, env, timeout: 10_000 }; + try { + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + const probe = execute('docker', ['exec', id, 'test', '-f', readyFile], options); + if (probe.status === 0) return; + const inspection = execute('docker', ['inspect', '--format', '{{json .State}}', id], options); + if (inspection.status !== 0) throw new Error(`cannot inspect build container ${id}: ${dockerDetail(inspection)}`); + const state = JSON.parse(inspection.stdout) as { Status: string; ExitCode: number; OOMKilled: boolean }; + if (state.Status === 'exited' || state.Status === 'dead') { + throw new Error(`build container ${id} ${state.Status} before ${description}; ` + + `exit code ${state.ExitCode}; OOMKilled=${state.OOMKilled}`); + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500); + } + throw new Error(`timed out waiting for ${description}`); + } catch (error) { + const logs = execute('docker', ['logs', '--tail', '100', id], options); + throw new Error(redactCredentials(`${error instanceof Error ? error.message : String(error)}\n` + + `${logs.stdout || ''}${logs.stderr || ''}`)); + } +} + +export function parseCgroupResources(value: string) { + const count = (name: string, key?: string): number | null => { + const section = (value.split(`[${name}]`)[1]?.split('[')[0] ?? '').replaceAll('\r', ''); + const match = section.match(new RegExp(key ? `^${key} (\\d+)$` : '^(\\d+)$', 'm')); + if (!match) return null; + const parsed = Number(match[1]); + return Number.isSafeInteger(parsed) ? parsed : null; + }; + return { + buildContainerMemory: { + currentBytes: count('memory.current'), + peakBytes: count('memory.peak'), + limitBytes: count('memory.max'), + oomEvents: count('memory.events', 'oom'), + oomKillEvents: count('memory.events', 'oom_kill'), + }, + buildContainerPids: { + current: count('pids.current'), + peak: count('pids.peak'), + limit: count('pids.max'), + limitEvents: count('pids.events', 'max'), + }, + }; +} + +export function inspectBuildContainer(name: string, { + env = process.env, + timeoutMs = 120_000, + execute = spawnSync as DockerExecute, +}: { env?: NodeJS.ProcessEnv; timeoutMs?: number; execute?: DockerExecute } = {}): InspectedBuildContainer | null { + const result = execute('docker', ['inspect', name], { encoding: 'utf8', env, timeout: timeoutMs }); + if (result.status !== 0) { + const detail = dockerDetail(result); + if (/no such (?:object|container)/i.test(detail)) return null; + throw new Error(`cannot inspect build container ${name}: ${detail}`); + } + + let parsed: unknown; + try { parsed = JSON.parse(result.stdout); } + catch (error) { + throw new Error(`Docker returned invalid inspection JSON for ${name}: ${error instanceof Error + ? error.message : String(error)}`); + } + if (!Array.isArray(parsed) || !isRecord(parsed[0])) { + throw new Error(`Docker returned an invalid container inspection for ${name}`); + } + + const inspected = parsed[0]; + const mounts = Array.isArray(inspected.Mounts) ? inspected.Mounts.filter(isRecord) : []; + const config = isRecord(inspected.Config) ? inspected.Config : {}; + const hostConfig = isRecord(inspected.HostConfig) ? inspected.HostConfig : {}; + const state = isRecord(inspected.State) ? inspected.State : {}; + const sensitiveTargets = new Set([LEGACY_SUBSCRIPTION_TOKEN_TARGET, '/root/.claude/.credentials.json', + '/root/.codex/auth.json', '/home/developer/.codex/auth.json']); + const capabilities = (values: unknown): string[] => stringArray(values).map(value => value.replace(/^CAP_/, '')); + const tmpfs = isRecord(hostConfig.Tmpfs) + ? Object.fromEntries(Object.entries(hostConfig.Tmpfs) + .filter((entry): entry is [string, string] => typeof entry[1] === 'string')) : {}; + + return { + id: String(inspected.Id), + image: String(inspected.Image), + running: state.Running === true, + networkMode: typeof hostConfig.NetworkMode === 'string' ? hostConfig.NetworkMode : null, + readonlyRootfs: hostConfig.ReadonlyRootfs === true, + tmpfs, + capAdd: capabilities(hostConfig.CapAdd), + capDrop: capabilities(hostConfig.CapDrop), + securityOpt: stringArray(hostConfig.SecurityOpt).map(option => option.replace(/:true$/, '')), + pidsLimit: numberOrNull(hostConfig.PidsLimit), + nanoCpus: numberOrNull(hostConfig.NanoCpus), + memoryBytes: numberOrNull(hostConfig.Memory), + memorySwapBytes: numberOrNull(hostConfig.MemorySwap), + mounts: mounts.map(mount => ({ + type: String(mount.Type), + source: String(mount.Source), + name: typeof mount.Name === 'string' ? mount.Name : null, + destination: String(mount.Destination), + readOnly: mount.RW !== true, + })), + unsafeCredentialExposure: mounts.some(mount => sensitiveTargets.has(String(mount.Destination))) + || stringArray(config.Env).some(value => /^(?:ANTHROPIC_API_KEY|CLAUDE_CODE_OAUTH_TOKEN|OPENAI_API_KEY|OPENROUTER_API_KEY|CODEX_AUTH_FILE)=/.test(value)), + }; +} + +export function sameHostPath(left: string, right: string, + platform: NodeJS.Platform = process.platform): boolean { + const normalize = (value: string): string => resolve(value).replaceAll('\\', '/'); + const normalizedLeft = normalize(left); + const normalizedRight = normalize(right); + return platform === 'win32' + ? normalizedLeft.toLowerCase() === normalizedRight.toLowerCase() + : normalizedLeft === normalizedRight; +} + +export function parsePublishedPorts(value: string | undefined): string[] { + if (!value) return []; + const ports = value.split(',').map(port => port.trim()).filter(Boolean); + if (ports.some(port => !/^\d+$/.test(port) || Number(port) < 1 || Number(port) > 65_535)) { + throw new Error('--ports must contain integers from 1 through 65535'); + } + if (new Set(ports).size !== ports.length) throw new Error('--ports must not contain duplicates'); + return ports; +} + +export function hasRequiredBuildContainerIsolation(container: InspectedBuildContainer, { + expectedMounts, + requiredTmpfs, + requiredCapabilities, + pidsLimit, + cpuCount, + memoryBytes, + memorySwapBytes, + image, +}: { + expectedMounts: ContainerMount[]; + requiredTmpfs: Readonly>; + requiredCapabilities: readonly string[]; + pidsLimit: number; + cpuCount: number; + memoryBytes: number; + memorySwapBytes: number; + image: string; +}): boolean { + const mountsMatch = container.mounts.length === expectedMounts.length + && expectedMounts.every(expected => container.mounts.some(actual => + actual.type === (expected.kind ?? 'bind') + && actual.destination === expected.target + && actual.readOnly === expected.readOnly + && (expected.kind === 'volume' + ? actual.name === expected.source + : sameHostPath(actual.source, expected.source)))); + return container.readonlyRootfs + && Object.entries(requiredTmpfs).every(([path, options]) => container.tmpfs[path] === options) + && Object.keys(container.tmpfs).length === Object.keys(requiredTmpfs).length + && requiredCapabilities.every(capability => container.capAdd.includes(capability)) + && container.capAdd.length === requiredCapabilities.length + && container.capDrop.includes('ALL') + && container.securityOpt.includes('no-new-privileges') + && container.pidsLimit === pidsLimit + && container.nanoCpus === cpuCount * 1_000_000_000 + && container.memoryBytes === memoryBytes + && container.memorySwapBytes === memorySwapBytes + && container.image === image + && mountsMatch; +} diff --git a/tools/stack-bench/container/build-linux-cli.sh b/tools/stack-bench/container/build-linux-cli.sh new file mode 100644 index 00000000000..8ce82bb007f --- /dev/null +++ b/tools/stack-bench/container/build-linux-cli.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Export native binaries through the same Docker build used by the appliance. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../../.." && pwd)" +MSYS_NO_PATHCONV=1 docker build --platform linux/amd64 \ + --file "$REPO/tools/stack-bench/appliance/Controller.Dockerfile" \ + --target binary-export --output "type=local,dest=$HERE" "$REPO" diff --git a/tools/stack-bench/container/claude-transcript-reader.ts b/tools/stack-bench/container/claude-transcript-reader.ts new file mode 100644 index 00000000000..604c3c3d610 --- /dev/null +++ b/tools/stack-bench/container/claude-transcript-reader.ts @@ -0,0 +1,54 @@ +import { execFileSync } from 'node:child_process'; +import { isAbsolute, join, relative, sep } from 'node:path'; +import type { ClaudeTranscriptReader } from '../src/agents/claude-terminal-recovery.js'; +import { CODING_CONTAINER_AGENT, codingContainerAgentExecOptions } + from '../src/runtime/coding-container-policy.js'; + +// Read as the transcript owner. Claude creates private files while the controller +// has no DAC override; the final transcript handback cannot serve a live reader. +export const CONTAINER_CLAUDE_TRANSCRIPT_READ = ` +const fs = require('node:fs'), path = require('node:path'); +const [root, name, offset, count] = process.argv.slice(1); +if (name === '') { + const files = fs.readdirSync(root, { recursive: true, withFileTypes: true }) + .filter(entry => entry.isFile() && entry.name.endsWith('.jsonl')) + .map(entry => path.join(entry.parentPath, entry.name)) + .map(file => [path.relative(root, file), fs.statSync(file).size, fs.statSync(file).mtimeMs]); + process.stdout.write(JSON.stringify(files)); +} else { + const file = path.resolve(root, name), resolvedRoot = fs.realpathSync(root); + if (!file.endsWith('.jsonl') || !fs.realpathSync(file).startsWith(resolvedRoot + path.sep)) { + throw new Error('transcript is outside the attempt directory'); + } + const start = Number(offset), length = Number(count); + if (!Number.isSafeInteger(start) || start < 0 || !Number.isSafeInteger(length) || length < 0) { + throw new Error('invalid transcript range'); + } + const buffer = Buffer.alloc(length), fd = fs.openSync(file, 'r'); + try { process.stdout.write(buffer.subarray(0, fs.readSync(fd, buffer, 0, length, start))); } + finally { fs.closeSync(fd); } +} +`; + +export function containerClaudeTranscriptReader(containerId: string, directory: string, + env: NodeJS.ProcessEnv): ClaudeTranscriptReader { + if (!/^[a-f0-9]{64}$/.test(containerId)) throw new Error('transcript reader requires an exact container ID'); + const root = `${CODING_CONTAINER_AGENT.home}/.claude/projects/-app`; + const read = (name: string, start = 0, length = 0): Buffer => execFileSync('docker', [ + 'exec', ...codingContainerAgentExecOptions(), containerId, 'node', '-e', + CONTAINER_CLAUDE_TRANSCRIPT_READ, root, name, String(start), String(length), + ], { env, timeout: 5_000, maxBuffer: 256 * 1024 * 1024 }); + return { + snapshot() { + const entries: [string, number][] = JSON.parse(read('').toString('utf8')); + return new Map(entries.map(([name, size]) => [join(directory, name), size])); + }, + read(path, start, length) { + const name = relative(directory, path); + if (!name || isAbsolute(name) || name.split(sep).includes('..')) { + throw new Error('transcript is outside the attempt directory'); + } + return read(name.split(sep).join('/'), start, length); + }, + }; +} diff --git a/tools/stack-bench/container/coding-providers.ts b/tools/stack-bench/container/coding-providers.ts new file mode 100644 index 00000000000..4ebc7a24313 --- /dev/null +++ b/tools/stack-bench/container/coding-providers.ts @@ -0,0 +1,112 @@ +import { appendFileSync, existsSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { codexArguments, codexTranscriptDirectory, parseCodexResult, runCodexProcess } + from '../src/agents/codex-protocol.js'; +import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js'; +import { runTranscriptAwareProcess } from '../src/agents/claude-terminal-recovery.js'; +import type { PricingRates } from '../src/evidence/pricing-authority.js'; +import { containerClaudeTranscriptReader } from './claude-transcript-reader.js'; +import { CODING_CONTAINER_AGENT, CODING_CONTAINER_APP_ROOT } from '../src/runtime/coding-container-policy.js'; +import { validateClaudeNativeSession, validateCodexNativeSession } + from '../src/agents/native-session-validation.js'; + +type Invocation = { model: string; effort: string; baseUrl: string; resumeSession: string | null; + maxBudgetUsd: string | null }; +type ProcessOptions = Parameters[0] & { + projects: string; containerId: string; marker: string; model: string; + pricingRates: PricingRates | null; resumeSession: string | null; +}; +interface CodingProvider { + requiresBudget: boolean; + executable: string; + apiKeyEnvironment: string; + credentialPath?: string; + containerTranscripts: string; + tokenEnvironment: string; + environment(baseUrl: string): string[]; + projects(appDir: string): string; + rates(model: string): PricingRates | null; + args(options: Invocation): string[]; + run(options: ProcessOptions): ReturnType; + result(stdout: string, appDir: string, invocationToken: string): Record | null; + validateContinuation(directory: string, sessionId: string, model: string): void; +} + +const codexProvider: CodingProvider = { + requiresBudget: true, + executable: 'codex', apiKeyEnvironment: 'OPENAI_API_KEY', + containerTranscripts: `${CODING_CONTAINER_AGENT.home}/.codex/sessions`, + tokenEnvironment: 'MODEL_PROXY_TOKEN', + environment: () => [`CODEX_HOME=${CODING_CONTAINER_AGENT.home}/.codex`], + projects: appDir => join(codexTranscriptDirectory(appDir), 'sessions'), + rates: () => null, + args: codexArguments, + run: runCodexProcess, + validateContinuation: validateCodexNativeSession, + result: (stdout, appDir, invocationToken) => { + const result = parseCodexResult(stdout); + const sessionId = result.session_id; + const eventFile = typeof sessionId === 'string' && /^[0-9a-f-]{36}$/i.test(sessionId) + ? `${sessionId}.events.jsonl` : `interrupted-${invocationToken}.events.jsonl`; + const path = join(codexTranscriptDirectory(appDir), eventFile); + const header = existsSync(path) ? '' : `${JSON.stringify({ type: 'stack_bench_context', cwd: '/app' })}\n`; + appendFileSync(path, `${header}${stdout}\n`, { mode: 0o600 }); + return result; + }, +}; + +export const CODING_PROVIDERS = { + anthropic: { + requiresBudget: false, + executable: 'claude', apiKeyEnvironment: 'ANTHROPIC_API_KEY', + credentialPath: join(homedir(), '.claude', '.credentials.json'), + containerTranscripts: `${CODING_CONTAINER_AGENT.home}/.claude/projects/-app`, + tokenEnvironment: 'ANTHROPIC_AUTH_TOKEN', + environment: baseUrl => [`ANTHROPIC_BASE_URL=${baseUrl}`, 'DISABLE_AUTOUPDATER=1', 'FORCE_PROMPT_CACHING_5M=1'], + projects: appDir => join(homedir(), '.claude', 'projects', + resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase()), + rates: claudeRatesForModel, + validateContinuation: validateClaudeNativeSession, + args: ({ model, effort, maxBudgetUsd, resumeSession }) => { + return [ + '--print', '--output-format', 'json', + // Isolate the session from project memory, plugins, and integrations. + '--bare', + '--permission-mode', 'acceptEdits', + '--settings', JSON.stringify({ permissions: { allow: ['Bash'] } }), + '--effort', effort, + '--model', model, + ...(maxBudgetUsd !== null ? ['--max-budget-usd', maxBudgetUsd] : []), + // The app is the only directory a session may reach; inside the container + // that is all there is, but the flag is kept so host and container runs are + // configured identically. + '--add-dir', CODING_CONTAINER_APP_ROOT, + ...(resumeSession !== null ? ['--resume', resumeSession] : []), + ]; + }, + run: options => { + const transcriptReader = containerClaudeTranscriptReader(options.containerId, options.projects, options.env); + return runTranscriptAwareProcess({ ...options, transcriptDirectory: options.projects, + transcriptReader, transcriptSnapshot: transcriptReader.snapshot(), pollMs: 1_000 }); + }, + result: stdout => { + try { return JSON.parse(stdout); } + catch { + for (const line of stdout.split(/\r?\n/).reverse()) { + try { return JSON.parse(line); } catch { /* Keep looking. */ } + } + } + return null; + }, + }, + openai: codexProvider, + openrouter: { ...codexProvider, apiKeyEnvironment: 'OPENROUTER_API_KEY' }, +} satisfies Record; + +export type CodingProviderId = keyof typeof CODING_PROVIDERS; + +export function parseCodingProvider(value: string): CodingProviderId { + if (!Object.hasOwn(CODING_PROVIDERS, value)) throw new Error(`unsupported coding provider: ${value}`); + return value as CodingProviderId; +} diff --git a/tools/stack-bench/container/container-auth.ts b/tools/stack-bench/container/container-auth.ts new file mode 100644 index 00000000000..d52ff6c457b --- /dev/null +++ b/tools/stack-bench/container/container-auth.ts @@ -0,0 +1,99 @@ +import { readPinnedExecutionCredential } from '../src/agents/credential-profiles.js'; +import { existsSync, readFileSync } from 'node:fs'; +import type { PathLike } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; + +export const SUBSCRIPTION_TOKEN_ENVIRONMENT = 'CLAUDE_CODE_OAUTH_TOKEN'; +export const LEGACY_SUBSCRIPTION_TOKEN_TARGET = '/run/secrets/claude-code-oauth-token'; + +export type ContainerAuth = { + provider?: 'anthropic' | 'openai' | 'openrouter'; + accountId?: string; + mode: 'api-key' | 'subscription-token'; + credential: string; +}; + +type ReadTextFile = (path: PathLike | number, encoding: BufferEncoding) => string; + +export interface ResolveContainerAuthOptions { + provider?: 'anthropic' | 'openai' | 'openrouter'; + apiKey?: string; + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + exists?: (path: PathLike) => boolean; + read?: ReadTextFile; +} + +export function resolveContainerAuth({ provider = 'anthropic', apiKey = '', env = process.env, credentialsPath, + exists = existsSync, read = readFileSync as ReadTextFile }: ResolveContainerAuthOptions = {}): ContainerAuth { + const pinned = readPinnedExecutionCredential(env); + if (pinned) { + if (pinned.assignment.provider !== provider) throw new Error('Pinned credential provider does not match invocation'); + apiKey = pinned.assignment.mode === 'api-key' ? pinned.secret : ''; + // Resolve the broker credential from the same bytes that passed its pin check. + // Never reopen a file that an operator can replace between validation and use. + read = path => { + if (String(path) !== pinned.secretFile) throw new Error('Pinned credential file does not match invocation'); + return pinned.secret; + }; + } + if (provider === 'openrouter') { + if (!apiKey) throw new Error('OpenRouter requires an API key'); + return { provider, mode: 'api-key', credential: apiKey }; + } + if (provider === 'openai') { + const authFile = env.CODEX_AUTH_FILE?.trim(); + if (apiKey && authFile) throw new Error('use only one of OpenAI API-key and account authentication'); + if (apiKey) return { provider, mode: 'api-key', credential: apiKey }; + if (!authFile) throw new Error('OpenAI requires an API key or an explicit CODEX_AUTH_FILE'); + if (!isAbsolute(authFile)) throw new Error('CODEX_AUTH_FILE must be an absolute path'); + if (!exists(authFile)) throw new Error('CODEX_AUTH_FILE does not exist'); + let auth: { auth_mode?: string; OPENAI_API_KEY?: unknown; + tokens?: { access_token?: unknown; account_id?: unknown } }; + try { auth = JSON.parse(read(authFile, 'utf8')); } + catch { throw new Error('CODEX_AUTH_FILE must contain valid Codex login JSON'); } + if (!auth || auth.OPENAI_API_KEY || auth.auth_mode !== 'chatgpt' + || typeof auth.tokens?.access_token !== 'string' || !auth.tokens.access_token.trim() + || typeof auth.tokens.account_id !== 'string' || !auth.tokens.account_id.trim()) { + throw new Error('CODEX_AUTH_FILE must contain ChatGPT account login tokens, not an API key'); + } + let expiry: unknown; + try { expiry = JSON.parse(Buffer.from(auth.tokens.access_token.split('.')[1]!, 'base64url').toString()).exp; } + catch { throw new Error('Codex account access token has no valid expiry; log in again'); } + if (typeof expiry !== 'number' || !Number.isFinite(expiry) || expiry * 1000 <= Date.now()) { + throw new Error('Codex account access token is expired; log in again and replace CODEX_AUTH_FILE'); + } + // Each broker uses an access-token snapshot. It never rotates shared refresh tokens. + return { provider, mode: 'subscription-token', credential: auth.tokens.access_token, + accountId: auth.tokens.account_id }; + } + const token = String(env[SUBSCRIPTION_TOKEN_ENVIRONMENT] ?? '').trim(); + const tokenFileValue = String(env[`${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE`] ?? '').trim(); + if (token && tokenFileValue) { + throw new Error(`use only one of ${SUBSCRIPTION_TOKEN_ENVIRONMENT} and ` + + `${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE`); + } + if (apiKey && (token || tokenFileValue)) { + throw new Error('use only one of API-key and subscription-token authentication'); + } + if (apiKey) return { mode: 'api-key', credential: apiKey }; + if (token) return { mode: 'subscription-token', credential: token }; + if (tokenFileValue) { + if (!isAbsolute(tokenFileValue)) { + throw new Error(`${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE must be an absolute path`); + } + const source = resolve(tokenFileValue); + if (!exists(source)) throw new Error(`subscription token file does not exist: ${source}`); + const credential = String(read(source, 'utf8')).trim(); + if (!credential) { + throw new Error(`subscription token file is empty: ${source}`); + } + return { mode: 'subscription-token', credential }; + } + if (credentialsPath && exists(credentialsPath)) { + throw new Error('rotating Claude credential files cannot be isolated from generated shell commands; ' + + 'select an API key or CLAUDE_CODE_OAUTH_TOKEN_FILE'); + } + throw new Error(`no API key, ${SUBSCRIPTION_TOKEN_ENVIRONMENT}, ` + + `${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE, or credentials file is available`); +} diff --git a/tools/stack-bench/container/credential-broker-accounting.ts b/tools/stack-bench/container/credential-broker-accounting.ts new file mode 100644 index 00000000000..b8f404c0715 --- /dev/null +++ b/tools/stack-bench/container/credential-broker-accounting.ts @@ -0,0 +1,378 @@ +import type { ProviderFailure } from '../src/agents/provider-failure.js'; +import { randomBytes } from 'node:crypto'; +import { readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { z } from 'zod'; + +import { normalizeClaudeUsage, priceClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import type { ClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import { validatePricingRates as validateSharedPricingRates } from '../src/evidence/pricing-authority.js'; +import { formatZodError } from '../src/zod-error.js'; + +export const BROKER_LEDGER_SCHEMA_VERSION = 4; +// Why a billable request was charged its cost ceiling instead of priced from +// the provider's usage: a 2xx response without complete usage (an aborted or +// errored stream, an oversized body), a response that broke off, or an +// upstream connection that failed. The ceiling makes spend an upper bound. +export const ESTIMATE_REASONS = ['no-usage', 'response-aborted', 'upstream-error'] as const; +export type EstimateReason = typeof ESTIMATE_REASONS[number]; +export type EstimateCounts = Record; +export const noEstimates = (): EstimateCounts => ({ 'no-usage': 0, 'response-aborted': 0, 'upstream-error': 0 }); +export const MAX_BROKER_OUTPUT_TOKENS = 128_000; +export const CLAUDE_USAGE_FIELDS = ['input', 'output', 'cacheRead', 'cacheWrite5m', 'cacheWrite1h'] as const; +const COST_TOLERANCE_USD = 0.0001; + +type JsonRecord = Record; +export type BrokerMode = 'api-key' | 'subscription-token'; +export type PricingRates = ReturnType; + +export type BrokerConfig = { + provider?: 'anthropic' | 'openai' | 'openrouter'; + accountId?: string; + providerRoute?: string; + mode: BrokerMode; + credential: string; + sessionToken: string; + readyPath?: string; + parentPid?: number; + heartbeatPath?: string; + expiresAt?: number; + listenHost?: '127.0.0.1' | '0.0.0.0'; + ledgerPath?: string; + model: string; + maxOutputTokens: number; + maxBudgetUsd?: number | null; + pricingRates?: PricingRates; +}; + +export type BrokerLedger = { + provider?: 'openrouter'; + providerRoute?: string; + providerReportedCostUsd?: number; + upstreamProviders?: string[]; + providerIntegrityError?: string; + providerFailure?: ProviderFailure | null; + schemaVersion: number; + model: string; + maxBudgetUsd: number | null; + acceptedRequests: number; + billableRequests: number; + completedBillableRequests: number; + estimatedBillableRequests: number; + estimatedByReason: EstimateCounts; + spentUsd: number; + reservedUsd: number; + usage: ClaudeUsage; + complete: boolean; + updatedAt: string; +}; + +// `costUsd` is what the broker charged: exact provider usage priced at the +// receipt's rates, plus the cost ceiling of every estimated request. With +// `exact` false it is an upper bound and `calculatedCostUsd`, priced from the +// exact usage alone, a lower bound. +export interface CredentialBrokerReceipt { + costSource?: 'provider-reported'; + provider?: 'openrouter'; + providerRoute?: string; + providerReportedCostUsd?: number; + upstreamProviders?: string[]; + schemaVersion: 3; + source: 'credential-broker'; + model: string; + maxBudgetUsd: number; + costUsd: number; + cliCostUsd: number | null; + calculatedCostUsd: number | null; + usage: ClaudeUsage | null; + pricingRates: PricingRates | null; + exact: boolean; + estimatedRequests: number; + estimatedByReason: EstimateCounts; + complete: boolean; + reconciled: boolean; + error: string | null; +} + +export interface CredentialBrokerResult extends JsonRecord { + total_cost_usd: number; + usage?: ReturnType; + stack_bench_cost_receipt: CredentialBrokerReceipt; +} + +const positiveFinite = z.number().finite().positive(); +const nonNegativeFinite = z.number().finite().nonnegative(); +const nonNegativeSafeInteger = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +const usageSchema = z.strictObject({ + input: nonNegativeSafeInteger, + output: nonNegativeSafeInteger, + cacheRead: nonNegativeSafeInteger, + cacheWrite5m: nonNegativeSafeInteger, + cacheWrite1h: nonNegativeSafeInteger, +}); +const brokerConfigSchema = z.strictObject({ + provider: z.enum(['anthropic', 'openai', 'openrouter']).optional(), + accountId: z.string().min(1).optional(), + providerRoute: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9._/-]{0,127}$/).optional(), + mode: z.enum(['api-key', 'subscription-token']), + credential: z.string().min(16), + sessionToken: z.string().min(16), + readyPath: z.string().min(1).optional(), + parentPid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(), + heartbeatPath: z.string().min(1).optional(), + expiresAt: positiveFinite.optional(), + listenHost: z.enum(['127.0.0.1', '0.0.0.0']).optional(), + ledgerPath: z.string().min(1).optional(), + model: z.string().min(1), + maxOutputTokens: z.number().int().min(1).max(MAX_BROKER_OUTPUT_TOKENS), + maxBudgetUsd: positiveFinite.nullable().optional(), + pricingRates: z.unknown().optional(), +}).superRefine((value, context) => { + if (value.provider === 'openrouter' && (value.mode !== 'api-key' || !value.providerRoute || value.maxBudgetUsd == null)) { + context.addIssue({ code: 'custom', message: 'OpenRouter requires API-key auth, providerRoute, and a spend budget' }); + } + if (value.provider !== 'openrouter' && value.providerRoute !== undefined) { + context.addIssue({ code: 'custom', path: ['providerRoute'], message: 'only OpenRouter uses providerRoute' }); + } + if (value.expiresAt !== undefined && value.expiresAt <= Date.now()) { + context.addIssue({ code: 'custom', path: ['expiresAt'], message: 'must be in the future' }); + } +}); +const brokerLedgerSchema = z.strictObject({ + provider: z.literal('openrouter').optional(), + providerRoute: z.string().min(1).optional(), + providerReportedCostUsd: nonNegativeFinite.optional(), + upstreamProviders: z.array(z.string().min(1).max(128)).optional(), + providerIntegrityError: z.string().max(200).optional(), + providerFailure: z.strictObject({ + category: z.enum(['rate-limit', 'quota', 'authentication', 'transport', 'request', 'broker-budget']), + status: z.number().int().min(100).max(599).nullable(), + code: z.string().regex(/^[a-zA-Z0-9_.-]{1,100}$/).nullable(), + budget: z.strictObject({ + maxBudgetUsd: positiveFinite, + spentUsd: nonNegativeFinite, + estimatedSpendUsd: nonNegativeFinite, + reservedUsd: nonNegativeFinite, + requestCeilingUsd: nonNegativeFinite, + }).optional(), + }).nullable().optional(), + schemaVersion: z.literal(BROKER_LEDGER_SCHEMA_VERSION), + model: z.string().min(1), + maxBudgetUsd: positiveFinite.nullable(), + acceptedRequests: nonNegativeSafeInteger, + billableRequests: nonNegativeSafeInteger, + completedBillableRequests: nonNegativeSafeInteger, + estimatedBillableRequests: nonNegativeSafeInteger, + estimatedByReason: z.strictObject({ + 'no-usage': nonNegativeSafeInteger, + 'response-aborted': nonNegativeSafeInteger, + 'upstream-error': nonNegativeSafeInteger, + }), + spentUsd: nonNegativeFinite, + reservedUsd: nonNegativeFinite, + usage: usageSchema, + complete: z.boolean(), + updatedAt: z.string().refine(value => !Number.isNaN(Date.parse(value)), 'must be a timestamp'), +}); + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function isNumber(value: unknown): value is number { + return typeof value === 'number'; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function fail(message: string): never { + throw new Error(`credential broker: ${message}`); +} + +export function validatePricingRates(value: unknown): PricingRates { + try { return validateSharedPricingRates(value, { at: 'pricingRates' }); } + catch (error) { return fail(errorMessage(error)); } +} + +export function priceNormalizedClaudeUsage(usage: ClaudeUsage, rates: PricingRates): number { + return priceClaudeUsage({ + input_tokens: usage.input, + output_tokens: usage.output, + cache_read_input_tokens: usage.cacheRead, + cache_creation: { + ephemeral_5m_input_tokens: usage.cacheWrite5m, + ephemeral_1h_input_tokens: usage.cacheWrite1h, + }, + }, rates); +} + +function rawUsage(usage: ClaudeUsage): JsonRecord { + return { + input_tokens: usage.input, + output_tokens: usage.output, + cache_read_input_tokens: usage.cacheRead, + cache_creation_input_tokens: usage.cacheWrite5m + usage.cacheWrite1h, + cache_creation: { + ephemeral_5m_input_tokens: usage.cacheWrite5m, + ephemeral_1h_input_tokens: usage.cacheWrite1h, + }, + }; +} + +function brokerCoversCliUsage(broker: ClaudeUsage, cli: ClaudeUsage): boolean { + return broker.input >= cli.input + && broker.output >= cli.output + && broker.cacheRead >= cli.cacheRead + && broker.cacheWrite5m + broker.cacheWrite1h >= cli.cacheWrite5m + cli.cacheWrite1h; +} + +export function validateBrokerConfig(value: unknown): BrokerConfig { + const parsed = brokerConfigSchema.safeParse(value); + if (!parsed.success) fail(formatZodError(parsed.error, 'configuration')); + const { pricingRates, ...config } = parsed.data; + return config.maxBudgetUsd === null || config.maxBudgetUsd === undefined + ? config + : { ...config, pricingRates: validatePricingRates(pricingRates) }; +} + +function validateLedger(value: unknown, + { model = null, maxBudgetUsd = undefined }: { model?: string | null; maxBudgetUsd?: number | null } = {}): BrokerLedger { + const parsed = brokerLedgerSchema.safeParse(value); + if (!parsed.success) fail(formatZodError(parsed.error, 'spend ledger')); + const ledger = parsed.data; + if (model !== null && ledger.model !== model) fail('spend ledger model does not match'); + if (maxBudgetUsd !== undefined && ledger.maxBudgetUsd !== maxBudgetUsd) { + fail('spend ledger budget does not match'); + } + if (ledger.completedBillableRequests > ledger.billableRequests) { + fail('spend ledger completed request count is invalid'); + } + if (ledger.estimatedBillableRequests > ledger.completedBillableRequests) { + fail('spend ledger estimated request count is invalid'); + } + const reasons = ESTIMATE_REASONS.reduce((sum, reason) => sum + ledger.estimatedByReason[reason], 0); + if (reasons !== ledger.estimatedBillableRequests) fail('spend ledger estimate reasons do not add up'); + const complete = ledger.reservedUsd === 0 + && ledger.completedBillableRequests === ledger.billableRequests; + if (ledger.complete !== complete) fail('spend ledger completion state is invalid'); + if (ledger.provider === 'openrouter' && (!ledger.providerRoute || ledger.providerReportedCostUsd === undefined + || !ledger.upstreamProviders || ledger.providerReportedCostUsd > ledger.spentUsd + COST_TOLERANCE_USD)) { + fail('OpenRouter spend ledger lacks valid cost and routing provenance'); + } + return ledger; +} + +export function writeCredentialBrokerLedger(path: string | undefined, value: unknown): void { + if (!path) return; + const ledger = validateLedger(value); + const temporary = `${path}.${process.pid}.${randomBytes(8).toString('hex')}.tmp`; + writeFileSync(temporary, `${JSON.stringify(ledger)}\n`, { flag: 'wx', mode: 0o600 }); + try { renameSync(temporary, path); } + catch (error) { rmSync(temporary, { force: true }); throw error; } +} + +export function readCredentialBrokerLedger(path: string, + expected: { model?: string | null; maxBudgetUsd?: number | null } = {}): BrokerLedger { + return validateLedger(JSON.parse(readFileSync(path, 'utf8')), expected); +} + +export function reconcileCredentialBrokerReceipt({ ledger, cliResult, model, maxBudgetUsd, + pricingRates, provider = 'anthropic', brokerDiagnostics = null, toleranceUsd = COST_TOLERANCE_USD }: { + provider?: 'anthropic' | 'openai' | 'openrouter'; + ledger: unknown; cliResult: unknown; model: unknown; maxBudgetUsd: unknown; pricingRates: unknown; + brokerDiagnostics?: unknown; toleranceUsd?: number; +}): { ok: boolean; result: CredentialBrokerResult; receipt: CredentialBrokerReceipt } { + if (typeof model !== 'string' || !model) fail('receipt model is invalid'); + if (!isNumber(maxBudgetUsd) || !Number.isFinite(maxBudgetUsd) || maxBudgetUsd <= 0) fail('receipt budget is invalid'); + if (!Number.isFinite(toleranceUsd) || toleranceUsd < 0) fail('receipt tolerance is invalid'); + const receiptBudget = maxBudgetUsd; + let verifiedLedger: BrokerLedger | null = null; + let verifiedRates: PricingRates | null = null; + let usage: ClaudeUsage | null = null; + let cliUsage: ClaudeUsage | null = null; + let calculatedCostUsd: number | null = null; + let issue: string | null = null; + try { verifiedLedger = validateLedger(ledger, { model, maxBudgetUsd: receiptBudget }); } + catch (error) { issue = errorMessage(error); } + if (!issue && verifiedLedger?.complete !== true) issue = 'credential broker spend ledger is incomplete'; + const estimatedRequests = verifiedLedger?.estimatedBillableRequests ?? 0; + const exact = verifiedLedger !== null && estimatedRequests === 0; + try { verifiedRates = validatePricingRates(pricingRates); } + catch (error) { if (!issue) issue = errorMessage(error); } + try { cliUsage = normalizeClaudeUsage(isRecord(cliResult) ? cliResult.usage : undefined); } + catch (error) { if (!issue) issue = errorMessage(error); } + if (verifiedLedger) usage = structuredClone(verifiedLedger.usage); + if (!issue && exact && cliUsage && usage && !brokerCoversCliUsage(usage, cliUsage)) { + issue = 'credential broker usage is lower than CLI usage totals'; + } + if (!issue && provider === 'openrouter' && verifiedLedger?.provider !== 'openrouter') issue = 'OpenRouter spend ledger lacks provenance'; + if (!issue && verifiedLedger?.providerIntegrityError) issue = verifiedLedger.providerIntegrityError; + try { if (provider !== 'openrouter' && verifiedRates && usage) calculatedCostUsd = priceNormalizedClaudeUsage(usage, verifiedRates); } + catch (error) { if (!issue) issue = errorMessage(error); } + const brokerCost = verifiedLedger + ? provider === 'openrouter' ? verifiedLedger.spentUsd + verifiedLedger.reservedUsd + : Math.min(receiptBudget, verifiedLedger.spentUsd + verifiedLedger.reservedUsd) : receiptBudget; + // Estimated requests contribute ceilings, not observed tokens. Require those + // ceilings to cover every usage component seen by either recorder. + if (!issue && provider !== 'openrouter' && !exact && verifiedRates && usage && cliUsage) { + const observedUsage = { ...usage }; + for (const field of CLAUDE_USAGE_FIELDS) { + observedUsage[field] = Math.max(usage[field], cliUsage[field]); + } + if (priceNormalizedClaudeUsage(observedUsage, verifiedRates) - brokerCost > toleranceUsd) { + issue = 'observed usage-priced spend exceeds credential broker spend ceiling'; + } + } + const cliCost = Number(isRecord(cliResult) ? cliResult.total_cost_usd : undefined); + if (!issue && (provider === 'anthropic' || (isRecord(cliResult) && cliResult.total_cost_usd !== undefined)) + && (!Number.isFinite(cliCost) || cliCost < 0)) { + issue = 'coding session did not return a usable cost receipt'; + } + // Exact spend must price back to the broker's figure. Estimated requests + // add their ceilings on top of the priced usage, so the priced usage can + // only fall below the broker's figure, never above it. + if (!issue && calculatedCostUsd !== null && exact && Math.abs(calculatedCostUsd - brokerCost) > toleranceUsd) { + issue = `usage-priced spend $${calculatedCostUsd.toFixed(6)} does not match credential broker spend $${brokerCost.toFixed(6)}`; + } + if (!issue && provider === 'openrouter' && brokerCost > receiptBudget + toleranceUsd) { + issue = 'OpenRouter reported spend exceeds the session budget'; + } + const receipt: CredentialBrokerReceipt = { + ...(provider === 'openrouter' ? { costSource: 'provider-reported' as const, provider, + providerRoute: verifiedLedger?.providerRoute, providerReportedCostUsd: verifiedLedger?.providerReportedCostUsd, + upstreamProviders: verifiedLedger?.upstreamProviders } : {}), + schemaVersion: 3, + source: 'credential-broker', + model, + maxBudgetUsd: receiptBudget, + costUsd: Number(brokerCost.toFixed(6)), + cliCostUsd: Number.isFinite(cliCost) && cliCost >= 0 ? Number(cliCost.toFixed(6)) : null, + calculatedCostUsd: calculatedCostUsd === null ? null : Number(calculatedCostUsd.toFixed(6)), + usage, + pricingRates: verifiedRates, + exact, + estimatedRequests, + estimatedByReason: verifiedLedger ? structuredClone(verifiedLedger.estimatedByReason) : noEstimates(), + complete: verifiedLedger?.complete === true, + reconciled: issue === null, + error: issue, + }; + const result: CredentialBrokerResult = { + ...(isRecord(cliResult) + ? structuredClone(cliResult) : { type: 'result', is_error: true, result: '' }), + total_cost_usd: receipt.costUsd, + stack_bench_cost_receipt: receipt, + }; + delete result.stack_bench_provider_failure; + if (verifiedLedger?.providerFailure) result.stack_bench_provider_failure = verifiedLedger.providerFailure; + if (usage) result.usage = rawUsage(usage); + if (brokerDiagnostics) result.stack_bench_credential_broker = structuredClone(brokerDiagnostics); + if (issue) { + result.is_error = true; + result.terminal_reason = 'cost_receipt_error'; + result.result = [typeof result.result === 'string' ? result.result.trim() : '', issue] + .filter(Boolean).join('\n'); + } + return { ok: issue === null, result, receipt }; +} diff --git a/tools/stack-bench/container/credential-broker-process.ts b/tools/stack-bench/container/credential-broker-process.ts new file mode 100644 index 00000000000..800443fb276 --- /dev/null +++ b/tools/stack-bench/container/credential-broker-process.ts @@ -0,0 +1,408 @@ +import { spawn, spawnSync } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { isAbsolute, join } from 'node:path'; +import { setTimeout as wait } from 'node:timers/promises'; + +import type { ContainerAuth } from './container-auth.js'; +import { MAX_BROKER_OUTPUT_TOKENS, readCredentialBrokerLedger, validateBrokerConfig } + from './credential-broker-accounting.js'; +import type { BrokerLedger, PricingRates } from './credential-broker-accounting.js'; +import { compiledEntrypoint } from '../src/package-root.js'; +import { killTree } from '../src/runtime/platform.js'; +import { ATTEMPT_CREATION_LABEL } from '../src/runtime/container-identity.js'; +import { BROKER_CONTAINER_RESOURCE_LIMITS } from '../src/composition/product-config.js'; + +const BROKER_DRAIN_TIMEOUT_MS = 30_000; +const BROKER_DRAIN_POLL_MS = 100; +const BROKER_STDERR_LIMIT_BYTES = 16 * 1024; +const BROKER_STOP_GRACE_MS = 2_000; +const BROKER_STOP_FORCE_MS = 2_000; + +type JsonRecord = Record; +type Alive = (pid: number | undefined) => boolean; + +export type BrokerProcessState = { + exitCode: number | null; + signal: NodeJS.Signals | null; + exitedAt: string | null; + stderrTail: string; + stderrPending: string; + stderrTruncated: boolean; +}; + +export type BrokerError = { type: string; phase: string; message: string }; + +export type BrokerDiagnostics = { + schemaVersion: number; + endpointKind: string; + child: { pid: number | undefined | null; exitCode: number | null; signal: NodeJS.Signals | null; + exitedAt: string | null; stderrTail: string | null; stderrTruncated: boolean }; + drain: { timeoutMs: number; elapsedMs: number; timedOut: boolean; reason: string | null; + terminationRequested: boolean } | null; + termination: { gracefulRequested: boolean; forceRequested: boolean; exited: boolean; + gracefulTimeoutMs: number; forceTimeoutMs: number } | null; + ledger: BrokerLedger | null; + errors: BrokerError[]; +}; + +export interface CredentialBrokerChild { + pid?: number; + exitCode?: number | null; + signalCode?: NodeJS.Signals | null; + kill?: (signal?: NodeJS.Signals | number) => boolean; +} + +export interface CredentialBrokerHandle { + child: CredentialBrokerChild; + root: string; + ledgerPath: string; + model: string; + maxBudgetUsd: number | null; + sessionToken?: string; + baseUrl?: string; + listenHost?: string; + endpointKind?: string; + processState?: Partial; + diagnosticSecrets?: string[]; + finalDiagnostics?: BrokerDiagnostics | null; + finalLedger?: BrokerLedger | null; + container?: CredentialBrokerContainer; + heartbeat?: NodeJS.Timeout; +} + +export type CredentialBrokerContainer = { + name: string; id: string; image: string; owned: true; networkMode: string; +}; + +export type CredentialBrokerDockerOptions = { + imageId: string; + networkContainerId: string; + // The caller persists this intent before creating any resource. + name: string; + creationToken: string; + // This private path must be mounted at the identical path on the Docker host. + privateDirectory: string; + onCreated: (container: CredentialBrokerContainer) => void; +}; + +function brokerDocker(args: string[]): string { + const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 10_000, windowsHide: true }); + if (result.status !== 0) throw new Error(`credential broker Docker ${args[0]} failed: ` + + (result.stderr || result.error?.message || `exit ${result.status}`).trim()); + return result.stdout.trim(); +} + +function signalBrokerContainer(container: CredentialBrokerContainer, signal: 'TERM' | 'KILL'): void { + brokerDocker(['kill', '--signal', signal, container.id]); +} + +export interface CredentialBroker extends CredentialBrokerHandle { + child: ChildProcess; + sessionToken: string; + baseUrl: string; + listenHost: string; + endpointKind: string; + processState: BrokerProcessState; + diagnosticSecrets: string[]; + finalDiagnostics: BrokerDiagnostics | null; + finalLedger: BrokerLedger | null; +} + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function fail(message: string): never { + throw new Error(`credential broker: ${message}`); +} + +function redactDiagnosticText(value: unknown, + broker: CredentialBrokerHandle | string[] | null | undefined): string { + let result = String(value ?? ''); + const secrets = Array.isArray(broker) ? broker : broker?.diagnosticSecrets ?? []; + for (const secret of secrets) { + if (typeof secret === 'string' && secret) result = result.replaceAll(secret, '[REDACTED]'); + } + return result; +} + +function appendDiagnosticStderr(state: BrokerProcessState, chunk: string, secrets: string[], flush = false): void { + const raw = state.stderrPending + chunk; + let redacted = redactDiagnosticText(raw, secrets); + let pendingLength = 0; + if (!flush) { + for (const secret of secrets) { + for (let length = Math.min(secret.length - 1, redacted.length); length > pendingLength; length -= 1) { + if (redacted.endsWith(secret.slice(0, length))) { pendingLength = length; break; } + } + } + } else if (raw && secrets.some(secret => secret.startsWith(raw))) redacted = '[REDACTED]'; + state.stderrPending = pendingLength ? redacted.slice(-pendingLength) : ''; + const safe = pendingLength ? redacted.slice(0, -pendingLength) : redacted; + const next = state.stderrTail + safe; + if (Buffer.byteLength(next) > BROKER_STDERR_LIMIT_BYTES) { + state.stderrTruncated = true; + state.stderrTail = Buffer.from(next).subarray(-BROKER_STDERR_LIMIT_BYTES).toString('utf8'); + } else state.stderrTail = next; +} + +export async function startCredentialBroker(selectedAuth: ContainerAuth, { networkMode, deadlineMs, + model, providerRoute, maxOutputTokens = MAX_BROKER_OUTPUT_TOKENS, maxBudgetUsd = null, pricingRates = null, + env = process.env, docker }: { networkMode: string; deadlineMs: number; model: string; + maxOutputTokens?: number; maxBudgetUsd?: number | null; pricingRates?: PricingRates | null; + providerRoute?: string; + env?: NodeJS.ProcessEnv; docker?: CredentialBrokerDockerOptions }): Promise { + if (!['bridge', 'host'].includes(networkMode) + && (!docker || networkMode !== `container:${docker.networkContainerId}`)) fail('network mode is invalid'); + if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) fail('deadline is invalid'); + const credential = selectedAuth.credential.trim(); + if (!credential) fail('selected authentication has no broker credential'); + if (docker) { + if (process.platform !== 'linux' || !isAbsolute(docker.privateDirectory)) { + fail('Docker broker requires a Linux controller and an absolute shared private directory'); + } + if (!/^sha256:[a-f0-9]{64}$/.test(docker.imageId) + || !/^[a-f0-9]{64}$/.test(docker.networkContainerId) + || !/^[a-z0-9][a-z0-9_.-]{0,127}$/.test(docker.name) + || !/^[a-f0-9]{32,64}$/.test(docker.creationToken)) fail('Docker broker identity is invalid'); + } + const root = mkdtempSync(join(docker?.privateDirectory ?? tmpdir(), 'stack-bench-credential-broker-')); + let child: ChildProcess | null = null; + let container: CredentialBrokerContainer | undefined; + const processState: BrokerProcessState = { exitCode: null, signal: null, exitedAt: null, + stderrTail: '', stderrPending: '', stderrTruncated: false }; + try { + chmodSync(root, 0o700); + const configPath = join(root, 'config.json'); + const readyPath = join(root, 'ready.json'); + const ledgerPath = join(root, 'spend-ledger.json'); + const heartbeatPath = join(root, 'heartbeat'); + if (docker) writeFileSync(heartbeatPath, '', { mode: 0o600 }); + const sessionToken = randomBytes(32).toString('hex'); + const listenHost = docker || networkMode === 'host' ? '127.0.0.1' : '0.0.0.0'; + const config = validateBrokerConfig({ provider: selectedAuth.provider, providerRoute, accountId: selectedAuth.accountId, + mode: selectedAuth.mode, credential, sessionToken, readyPath, + ...(docker ? { heartbeatPath } : { parentPid: process.pid }), + expiresAt: Date.now() + deadlineMs + 60_000, listenHost, ledgerPath, + model, maxOutputTokens, maxBudgetUsd, pricingRates }); + writeFileSync(configPath, `${JSON.stringify(config)}\n`, { flag: 'wx', mode: 0o600 }); + if (docker) { + const network = `container:${docker.networkContainerId}`; + const id = brokerDocker(['create', '--name', docker.name, + '--label', `${ATTEMPT_CREATION_LABEL}=${docker.creationToken}`, + '--network', network, '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges:true', + '--read-only', '--pids-limit', String(BROKER_CONTAINER_RESOURCE_LIMITS.pids), + '--memory', String(BROKER_CONTAINER_RESOURCE_LIMITS.memoryBytes), + '--memory-swap', String(BROKER_CONTAINER_RESOURCE_LIMITS.memoryBytes), + '--mount', `type=bind,src=${root},dst=${root}`, + '--entrypoint', 'node', docker.imageId, + '/opt/stack-bench/dist/container/credential-broker.js', '--config', configPath]); + if (!/^[a-f0-9]{64}$/.test(id)) fail('Docker broker did not return a container ID'); + container = { id, name: docker.name, image: docker.imageId, owned: true, networkMode: network }; + docker.onCreated(container); + } + child = spawn(container ? 'docker' : process.execPath, + container ? ['start', '--attach', container.id] + : [compiledEntrypoint('container', 'credential-broker.js'), '--config', configPath], { + stdio: ['ignore', 'ignore', 'pipe'], + windowsHide: true, + env: Object.fromEntries(['PATH', 'Path', 'SystemRoot', 'WINDIR', 'SSL_CERT_FILE', + 'NODE_EXTRA_CA_CERTS', 'HTTPS_PROXY', 'HTTP_PROXY'] + .filter(name => env[name] !== undefined).map(name => [name, env[name]])), + }); + const diagnosticSecrets = [credential, sessionToken]; + child.stderr?.on('data', (chunk: Buffer) => appendDiagnosticStderr( + processState, chunk.toString('utf8'), diagnosticSecrets)); + child.once('exit', (code: number | null, signal: NodeJS.Signals | null) => { + appendDiagnosticStderr(processState, '', diagnosticSecrets, true); + processState.exitCode = code; + processState.signal = signal; + processState.exitedAt = new Date().toISOString(); + }); + let spawnError: Error | null = null; + child.once('error', (error: Error) => { spawnError = error; }); + const readyDeadline = Date.now() + 10_000; + while (!spawnError && child.exitCode === null && !existsSync(readyPath) && Date.now() < readyDeadline) { + await wait(100); + } + if (spawnError) throw spawnError; + if (!existsSync(readyPath)) throw new Error(`credential broker did not become ready${processState.stderrTail ? `: ${processState.stderrTail}` : ''}`); + const ready: unknown = JSON.parse(readFileSync(readyPath, 'utf8')); + if (!isRecord(ready) || typeof ready.port !== 'number' || !Number.isInteger(ready.port) + || ready.port < 1 || ready.port > 65_535) throw new Error('credential broker returned an invalid port'); + if (ready.host !== listenHost) throw new Error('credential broker returned an invalid host'); + const host = docker || networkMode === 'host' ? '127.0.0.1' : 'host.docker.internal'; + const heartbeat = docker ? setInterval(() => { + try { const time = new Date(); utimesSync(heartbeatPath, time, time); } catch { /* the broker stops itself */ } + }, 5_000).unref() : undefined; + return { child, root, ledgerPath, model, maxBudgetUsd: maxBudgetUsd ?? null, ...(heartbeat ? { heartbeat } : {}), + sessionToken, baseUrl: `http://${host}:${ready.port}`, listenHost, + endpointKind: docker ? 'container-credential-broker' : 'local-credential-broker', processState, + ...(container ? { container } : {}), + diagnosticSecrets, finalDiagnostics: null, finalLedger: null }; + } catch (error) { + // Keep private authority when Docker cleanup fails. Recovery uses the saved exact ID. + if (container) brokerDocker(['rm', '-f', container.id]); + if (child?.pid) killTree(child.pid); + rmSync(root, { recursive: true, force: true }); + throw error; + } +} + +function processAlive(pid: number | undefined): boolean { + if (typeof pid !== 'number' || !Number.isInteger(pid) || pid < 1) return false; + try { process.kill(pid, 0); return true; } + catch (error) { return !isRecord(error) || error.code !== 'ESRCH'; } +} + +function brokerExited(broker: CredentialBrokerHandle | null | undefined, alive: Alive): boolean { + if (broker?.container) { + try { + return brokerDocker(['inspect', '--format', '{{.State.Running}}', broker.container.id]) === 'false'; + } catch { + // Docker unavailability is not proof that a provider-capable process stopped. + return false; + } + } + const state = broker?.processState; + if (!state) return !alive(broker?.child?.pid); + if (state.exitedAt || state.exitCode !== null && state.exitCode !== undefined + || state.signal !== null && state.signal !== undefined + || broker?.child?.exitCode !== null && broker?.child?.exitCode !== undefined + || broker?.child?.signalCode !== null && broker?.child?.signalCode !== undefined) return true; + return !alive(broker?.child?.pid); +} + +async function waitForBrokerExit(broker: CredentialBrokerHandle, timeoutMs: number, + { sleep, now, alive }: { sleep: (ms: number) => Promise; now: () => number; alive: Alive }): Promise { + const deadline = now() + timeoutMs; + while (!brokerExited(broker, alive) && now() < deadline) await sleep(BROKER_DRAIN_POLL_MS); + return brokerExited(broker, alive); +} + +export function credentialBrokerDiagnostics(broker: CredentialBrokerHandle | null): BrokerDiagnostics | null { + if (!broker) return null; + if (broker.finalDiagnostics) return structuredClone(broker.finalDiagnostics); + const state = broker.processState ?? {}; + return { + schemaVersion: 1, + endpointKind: broker.endpointKind ?? 'local-credential-broker', + child: { pid: broker.child?.pid ?? null, + exitCode: state.exitCode ?? broker.child?.exitCode ?? null, + signal: state.signal ?? broker.child?.signalCode ?? null, + exitedAt: state.exitedAt ?? null, + stderrTail: state.stderrTail ? redactDiagnosticText(state.stderrTail, broker) : null, + stderrTruncated: state.stderrTruncated === true }, + drain: null, + termination: null, + ledger: null, + errors: [], + }; +} + +export async function stopCredentialBroker(broker: CredentialBrokerHandle | null, { + drainTimeoutMs = BROKER_DRAIN_TIMEOUT_MS, + pollMs = BROKER_DRAIN_POLL_MS, + gracefulTimeoutMs = BROKER_STOP_GRACE_MS, + forceTimeoutMs = BROKER_STOP_FORCE_MS, + readLedger = readCredentialBrokerLedger, + terminate = (pid: Parameters[0]) => { + if (broker?.container) signalBrokerContainer(broker.container, 'KILL'); + else killTree(pid); + }, + requestStop = (child: CredentialBrokerChild) => { + if (broker?.container) signalBrokerContainer(broker.container, 'TERM'); + else child.kill?.('SIGTERM'); + }, + alive = processAlive, + sleep = wait, + now = Date.now, +}: { drainTimeoutMs?: number; pollMs?: number; gracefulTimeoutMs?: number; forceTimeoutMs?: number; + readLedger?: typeof readCredentialBrokerLedger; terminate?: typeof killTree; + requestStop?: (child: CredentialBrokerChild) => boolean | void; alive?: Alive; + sleep?: (ms: number) => Promise; now?: () => number } = {}): Promise { + if (!broker) return null; + if (broker.finalDiagnostics) return structuredClone(broker.finalLedger ?? null); + let ledger: BrokerLedger | null = null; + const startedAt = now(); + let drainTimedOut = false; + let drainReason: string | null = null; + const errors: BrokerError[] = []; + const errorKeys = new Set(); + const recordError = (type: string, phase: string, error: unknown): void => { + const message = redactDiagnosticText(error instanceof Error ? error.message : error, broker) || 'unknown error'; + const key = `${type}:${phase}:${message}`; + if (errorKeys.has(key)) return; + errorKeys.add(key); + errors.push({ type, phase, message }); + }; + const read = (phase: string, expected: { model: string; maxBudgetUsd: number | null }): BrokerLedger | null => { + try { return readLedger(broker.ledgerPath, expected); } + catch (error) { recordError('ledger-read-error', phase, error); return null; } + }; + let gracefulRequested = false; + let forceRequested = false; + let exited = brokerExited(broker, alive); + const expected = { model: broker.model, maxBudgetUsd: broker.maxBudgetUsd }; + try { + const deadline = now() + drainTimeoutMs; + while (drainReason === null) { + ledger = read('drain', expected) ?? ledger; + if (ledger?.complete === true) { drainReason = 'ledger-complete'; break; } + exited = brokerExited(broker, alive); + if (exited) { drainReason = 'child-exited'; break; } + if (now() >= deadline) { drainTimedOut = true; drainReason = 'timeout'; break; } + await sleep(pollMs); + } + exited = brokerExited(broker, alive); + if (!exited) { + gracefulRequested = true; + try { requestStop(broker.child); } + catch (error) { recordError('termination-error', 'graceful-request', error); } + exited = await waitForBrokerExit(broker, gracefulTimeoutMs, { sleep, now, alive }); + } + if (!exited) { + forceRequested = true; + try { terminate(broker.child.pid); } + catch (error) { recordError('termination-error', 'force-request', error); } + exited = await waitForBrokerExit(broker, forceTimeoutMs, { sleep, now, alive }); + } + if (!exited) recordError('termination-error', 'exit-verification', + new Error('credential broker remained alive after forced termination')); + ledger = read('final', expected) ?? ledger; + } catch (error) { recordError('broker-stop-error', 'shutdown', error); } + finally { + clearInterval(broker.heartbeat); + const state = broker.processState ?? {}; + if (exited) { + try { + if (broker.container) brokerDocker(['rm', broker.container.id]); + // Grading can be interrupted before the caller saves its receipt. Keep the + // atomically written spend ledger; remove only the broker's credentials. + for (const name of ['config.json', 'ready.json']) { + rmSync(join(broker.root, name), { force: true }); + } + } + catch (error) { recordError('cleanup-error', 'private-root', error); } + } + broker.finalLedger = ledger; + broker.finalDiagnostics = { + ...(credentialBrokerDiagnostics(broker) as BrokerDiagnostics), + child: { pid: broker.child?.pid ?? null, + exitCode: state.exitCode ?? broker.child?.exitCode ?? null, + signal: state.signal ?? broker.child?.signalCode ?? null, + exitedAt: state.exitedAt ?? null, + stderrTail: state.stderrTail ? redactDiagnosticText(state.stderrTail, broker) : null, + stderrTruncated: state.stderrTruncated === true }, + drain: { timeoutMs: drainTimeoutMs, elapsedMs: Math.max(0, now() - startedAt), + timedOut: drainTimedOut, reason: drainReason, terminationRequested: gracefulRequested }, + termination: { gracefulRequested, forceRequested, exited, gracefulTimeoutMs, forceTimeoutMs }, + ledger: ledger ? structuredClone(ledger) : null, + errors, + }; + } + return ledger; +} diff --git a/tools/stack-bench/container/credential-broker.ts b/tools/stack-bench/container/credential-broker.ts new file mode 100644 index 00000000000..bc7ece43e79 --- /dev/null +++ b/tools/stack-bench/container/credential-broker.ts @@ -0,0 +1,390 @@ +#!/usr/bin/env node +import { createServer } from 'node:http'; +import type { ClientRequest, IncomingMessage, OutgoingHttpHeaders, ServerResponse } from 'node:http'; +import { request as httpsRequest } from 'node:https'; +import type { RequestOptions } from 'node:https'; +import type { Socket } from 'node:net'; +import { readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import type { AddressInfo } from 'node:net'; +import { classifyProviderFailure } from '../src/agents/provider-failure.js'; +import type { ProviderFailure } from '../src/agents/provider-failure.js'; +import { brokerProtocol } from './broker-protocols.js'; + +import { normalizeClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import type { ClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import { BROKER_LEDGER_SCHEMA_VERSION, CLAUDE_USAGE_FIELDS, noEstimates, priceNormalizedClaudeUsage, + validateBrokerConfig, + writeCredentialBrokerLedger } from './credential-broker-accounting.js'; +import type { BrokerConfig, EstimateReason, PricingRates } + from './credential-broker-accounting.js'; + +const MAX_REQUEST_BYTES = 32 * 1024 * 1024; +const BROKER_SERVER_CLOSE_GRACE_MS = 1_000; +export type { ClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +type JsonRecord = Record; +export interface BrokerStats { + acceptedRequests: number; + billableRequests: number; + completedBillableRequests: number; + estimatedBillableRequests: number; + spentUsd: number; + reservedUsd: number; +} + +export interface CreatedCredentialBroker { + server: ReturnType; + stats: () => BrokerStats; +} + +type UpstreamRequest = (options: RequestOptions, + callback: (response: IncomingMessage) => void) => ClientRequest; + +const roundUsd = (value: number): number => Number(value.toFixed(6)); +const reserveUsd = (value: number): number => Math.ceil(value * 1e6) / 1e6; + +function fail(message: string): never { + throw new Error(`credential broker: ${message}`); +} + +function clientAuthorized(request: IncomingMessage, sessionToken: string): boolean { + return request.headers.authorization === `Bearer ${sessionToken}` + || request.headers['x-api-key'] === sessionToken; +} + +function requestPath(value: string | undefined): string | null { + try { return new URL(value ?? '', 'http://credential-broker.invalid').pathname; } + catch { return null; } +} + +function rejectRequest(request: IncomingMessage, response: ServerResponse, + status: number, message: string): void { + request.on('error', () => {}); + response.on('error', () => {}); + try { + response.shouldKeepAlive = false; + response.writeHead(status, { 'content-type': 'text/plain', connection: 'close' }); + response.end(message); + } catch { response.destroy(); } + request.resume(); +} + +function requestCostCeiling(bodyBytes: number, maxTokens: number, rates: PricingRates): number { + const inputRate = Math.max(rates.input, rates.cacheRead, rates.cacheWrite5m, rates.cacheWrite1h); + return bodyBytes * inputRate / 1e6 + maxTokens * rates.output / 1e6; +} + +export function createCredentialBroker(configInput: unknown, { + requestUpstream = httpsRequest as UpstreamRequest, + upstream, + maxRequestBytes = MAX_REQUEST_BYTES, +}: { requestUpstream?: UpstreamRequest; + upstream?: { protocol: string; hostname: string; port: number }; + maxRequestBytes?: number } = {}): CreatedCredentialBroker { + const config = validateBrokerConfig(configInput); + const protocol = brokerProtocol(config); + const destination = upstream ?? { protocol: 'https:', hostname: protocol.hostname, port: 443 }; + let acceptedRequests = 0; + let lastResponseRequest = 0; + let providerFailure: ProviderFailure | null = null; + const recordFailure = (request: number, failure: ProviderFailure | null): void => { + if (request >= lastResponseRequest) { lastResponseRequest = request; providerFailure = failure; } + }; + let billableRequests = 0; + let completedBillableRequests = 0; + let estimatedBillableRequests = 0; + const estimatedByReason = noEstimates(); + let spentUsd = 0; + let providerReportedCostUsd = 0; + let providerIntegrityError: string | undefined; + const upstreamProviders = new Set(); + let reservedUsd = 0; + const usageTotals: ClaudeUsage = { input: 0, output: 0, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }; + const recordLedger = () => writeCredentialBrokerLedger(config.ledgerPath, { + schemaVersion: BROKER_LEDGER_SCHEMA_VERSION, + ...(config.provider === 'openrouter' ? { provider: config.provider, providerRoute: config.providerRoute, + providerReportedCostUsd: roundUsd(providerReportedCostUsd), upstreamProviders: [...upstreamProviders], + ...(providerIntegrityError ? { providerIntegrityError } : {}) } : {}), + providerFailure, + model: config.model, + maxBudgetUsd: config.maxBudgetUsd ?? null, + acceptedRequests, + billableRequests, + completedBillableRequests, + estimatedBillableRequests, + estimatedByReason, + spentUsd: Number(spentUsd.toFixed(6)), + reservedUsd: Number(reservedUsd.toFixed(6)), + usage: usageTotals, + complete: reservedUsd === 0 && completedBillableRequests === billableRequests, + updatedAt: new Date().toISOString(), + }); + recordLedger(); + const server = createServer((request, response) => { + // A client can disappear while the broker is still draining an upstream + // response. Socket errors must not terminate the broker and strand a paid + // request reservation in the ledger. + request.on('error', () => {}); + request.on('aborted', () => {}); + response.on('error', () => {}); + const responseOpen = (): boolean => !response.destroyed && !response.writableEnded; + const writeHead = (status: number, headers: OutgoingHttpHeaders): void => { + if (!responseOpen() || response.headersSent) return; + try { response.writeHead(status, headers); } + catch { response.destroy(); } + }; + const endResponse = (body?: string | Buffer): void => { + if (!responseOpen()) return; + try { response.end(body); } + catch { response.destroy(); } + }; + if (!clientAuthorized(request, config.sessionToken)) { + rejectRequest(request, response, 401, 'unauthorized'); + return; + } + if (providerIntegrityError) { + rejectRequest(request, response, 502, 'provider accounting or routing validation failed'); + return; + } + const path = requestPath(request.url); + if (request.method !== 'POST' || path === null || !protocol.allowedPaths.has(path)) { + recordFailure(acceptedRequests + 1, { category: 'request', status: 404, code: 'broker-path' }); + recordLedger(); + rejectRequest(request, response, 404, 'not found'); + return; + } + acceptedRequests += 1; + const requestOrdinal = acceptedRequests; + recordLedger(); + + const chunks: Buffer[] = []; + let received = 0; + let tooLarge = false; + request.on('data', (chunk: Buffer) => { + if (tooLarge) return; + received += chunk.length; + if (received > maxRequestBytes) { + tooLarge = true; + recordFailure(requestOrdinal, { category: 'request', status: 413, code: 'broker-body-limit' }); + recordLedger(); + writeHead(413, { 'content-type': 'text/plain' }); + endResponse('request is too large'); + return; + } + chunks.push(chunk); + }); + request.on('end', () => { + if (tooLarge) return; + const body = Buffer.concat(chunks); + let payload: JsonRecord; + try { payload = protocol.parseRequest(body, path); } + catch { + recordFailure(requestOrdinal, { category: 'request', status: 400, code: 'broker-request-invalid' }); + recordLedger(); + writeHead(400, { 'content-type': 'text/plain' }); + endResponse('invalid provider request'); + return; + } + const billable = protocol.billable(path) && config.maxBudgetUsd != null; + const costCeiling = billable + ? reserveUsd(requestCostCeiling(received + (protocol.inputTokenAdjustment?.(payload) ?? 0), protocol.outputLimit(payload), + config.pricingRates as PricingRates)) : 0; + const budget = config.maxBudgetUsd; + if (billable && budget !== null && budget !== undefined + && spentUsd + reservedUsd + costCeiling > budget) { + const measuredSpend = config.provider === 'openrouter' ? providerReportedCostUsd + : priceNormalizedClaudeUsage(usageTotals, config.pricingRates as PricingRates); + recordFailure(requestOrdinal, { category: 'broker-budget', status: 402, code: 'reservation-exceeds-budget', + budget: { maxBudgetUsd: budget, spentUsd, reservedUsd, requestCeilingUsd: costCeiling, + estimatedSpendUsd: roundUsd(Math.max(0, spentUsd - measuredSpend)) } }); + recordLedger(); + writeHead(402, { 'content-type': 'text/plain' }); + endResponse('session budget cannot cover the next request reservation'); + return; + } + if (billable) billableRequests += 1; + reservedUsd = roundUsd(reservedUsd + costCeiling); + recordLedger(); + let billableSettled = !billable; + const settleBillable = ({ usage = null, estimated = null, reportedCost = null }: + { usage?: ClaudeUsage | null; estimated?: EstimateReason | null; reportedCost?: number | null } = {}): void => { + if (billableSettled) return; + billableSettled = true; + reservedUsd = roundUsd(reservedUsd - costCeiling); + completedBillableRequests += 1; + if (estimated) { + estimatedBillableRequests += 1; + estimatedByReason[estimated] += 1; + spentUsd = roundUsd(spentUsd + costCeiling); + } else if (usage) { + spentUsd = roundUsd(spentUsd + (reportedCost ?? priceNormalizedClaudeUsage(usage, config.pricingRates as PricingRates))); + if (reportedCost !== null) { + providerReportedCostUsd = roundUsd(providerReportedCostUsd + reportedCost); + if (reportedCost > costCeiling + 0.000001) { + providerIntegrityError = 'OpenRouter reported cost exceeds the request reservation'; + } + } + for (const field of CLAUDE_USAGE_FIELDS) usageTotals[field] += usage[field]; + } + recordLedger(); + }; + const headers = protocol.headers(request); + for (const name of ['connection', 'keep-alive', 'proxy-connection', 'te', 'trailer', + 'transfer-encoding', 'upgrade']) delete headers[name]; + const forwardedBody = Buffer.from(JSON.stringify(payload)); + headers['content-length'] = String(forwardedBody.length); + const upstreamRequest = requestUpstream({ + protocol: destination.protocol, + hostname: destination.hostname, + port: destination.port, + method: request.method, + path: protocol.upstreamPath(path + new URL(request.url ?? '', 'http://credential-broker.invalid').search), + headers, + }, upstreamResponse => { + writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); + const responseChunks: Buffer[] = []; + let responseBytes = 0; + upstreamResponse.on('data', (chunk: Buffer) => { + responseBytes += chunk.length; + if (responseBytes <= maxRequestBytes) responseChunks.push(chunk); + if (responseOpen()) { + try { response.write(chunk); } + catch { response.destroy(); } + } + }); + upstreamResponse.on('end', () => { + const status = upstreamResponse.statusCode ?? 502; + recordFailure(requestOrdinal, status >= 200 && status < 300 ? null + : classifyProviderFailure(status, Buffer.concat(responseChunks))); + recordLedger(); + endResponse(); + if (!billable) return; + if ((upstreamResponse.statusCode ?? 502) >= 200 + && (upstreamResponse.statusCode ?? 502) < 300) { + const usage = responseBytes <= maxRequestBytes + ? protocol.responseUsage(Buffer.concat(responseChunks), upstreamResponse.headers['content-encoding']) + : null; + if (!usage) { + if (config.provider === 'openrouter') providerIntegrityError = 'OpenRouter response lacks verified cost and routing metadata'; + recordFailure(requestOrdinal, { category: 'transport', status, code: 'incomplete-response' }); + settleBillable({ estimated: 'no-usage' }); + } + else try { + if (typeof usage.upstream_provider === 'string') upstreamProviders.add(usage.upstream_provider); + settleBillable({ usage: normalizeClaudeUsage(usage), + reportedCost: typeof usage.provider_reported_cost_usd === 'number' ? usage.provider_reported_cost_usd : null }); + } + catch { + recordFailure(requestOrdinal, { category: 'transport', status, code: 'invalid-usage' }); + settleBillable({ estimated: 'no-usage' }); + } + } else { + settleBillable(); + } + }); + const settleAbortedResponse = () => { + recordFailure(requestOrdinal, { category: 'transport', status: null, code: null }); + settleBillable({ estimated: 'response-aborted' }); + if (responseOpen()) response.destroy(); + }; + upstreamResponse.once('aborted', settleAbortedResponse); + upstreamResponse.once('error', settleAbortedResponse); + }); + upstreamRequest.on('error', () => { + recordFailure(requestOrdinal, { category: 'transport', status: null, code: null }); + settleBillable({ estimated: 'upstream-error' }); + writeHead(502, { 'content-type': 'text/plain' }); + endResponse('upstream request failed'); + }); + upstreamRequest.end(forwardedBody); + }); + }); + server.on('clientError', (_error: Error, socket: Socket) => { + socket.on('error', () => {}); + if (socket.writable) socket.end('HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n'); + else socket.destroy(); + }); + return { server, stats: () => ({ acceptedRequests, + billableRequests, completedBillableRequests, estimatedBillableRequests, + estimatedByReason: { ...estimatedByReason }, + spentUsd: Number(spentUsd.toFixed(6)), reservedUsd: Number(reservedUsd.toFixed(6)) }) }; +} + +function parseArgs(argv: string[]): string { + const { values } = parseNodeArgs({ args: argv, options: { config: { type: 'string' } } }); + const configPath = values.config; + if (!configPath || argv.length !== 2) fail('use --config '); + return resolve(configPath); +} + +async function main() { + const configPath = parseArgs(process.argv.slice(2)); + let config: BrokerConfig; + try { config = validateBrokerConfig(JSON.parse(readFileSync(configPath, 'utf8'))); } + finally { rmSync(configPath, { force: true }); } + if (!config.readyPath) fail('readyPath is invalid'); + const { server } = createCredentialBroker(config); + const sockets = new Set(); + server.on('connection', (socket: Socket) => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + }); + server.on('error', (error: Error) => { + process.stderr.write(`credential broker: ${error.message}\n`); + process.exitCode = 1; + }); + const readyPath = config.readyPath; + server.listen(0, config.listenHost ?? '127.0.0.1', () => { + const address: string | AddressInfo | null = server.address(); + if (!address || typeof address === 'string') fail('listener address is unavailable'); + writeFileSync(readyPath, `${JSON.stringify({ host: address.address, port: address.port })}\n`, + { flag: 'wx', mode: 0o600 }); + }); + let stopping = false; + const stop = () => { + if (stopping) return; + stopping = true; + const force = setTimeout(() => { + for (const socket of sockets) socket.destroy(); + server.closeAllConnections?.(); + process.exit(0); + }, BROKER_SERVER_CLOSE_GRACE_MS); + force.unref(); + server.close(() => { + clearTimeout(force); + process.exit(0); + }); + server.closeIdleConnections?.(); + }; + const parentPid = config.parentPid; + if (parentPid) { + setInterval(() => { + try { process.kill(parentPid, 0); } + catch { stop(); } + }, 1_000).unref(); + } + // A Docker broker cannot see the controller process; the controller touches this + // file instead, and a stale file means it died. + const heartbeatPath = config.heartbeatPath; + if (heartbeatPath) { + setInterval(() => { if (heartbeatStale(heartbeatPath, Date.now())) stop(); }, 5_000).unref(); + } + const expiresAt = config.expiresAt; + if (expiresAt) setTimeout(stop, Math.max(1, expiresAt - Date.now())).unref(); + process.on('SIGINT', stop); + process.on('SIGTERM', stop); +} + +export function heartbeatStale(path: string, now: number): boolean { + try { return now - statSync(path).mtimeMs > 60_000; } + catch { return true; } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/tools/stack-bench/container/reconcile-build-container.ts b/tools/stack-bench/container/reconcile-build-container.ts new file mode 100644 index 00000000000..a58fce33aaf --- /dev/null +++ b/tools/stack-bench/container/reconcile-build-container.ts @@ -0,0 +1,77 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import type { SpawnSyncOptionsWithStringEncoding, SpawnSyncReturns } from 'node:child_process'; + +export const BUILD_CONTAINER_CREATION_LABEL = 'com.clockworklabs.stack-bench.creation'; + +export function buildContainerName(lease: { runId: string; + resources: { buildContainer?: { name: string } | null } }): string { + return lease.resources.buildContainer?.name + ?? `sb-${createHash('sha256').update(lease.runId).digest('hex').slice(0, 16)}-build`; +} + +const CONTAINER_ID = /^[a-f0-9]{64}$/i; + +type DockerResult = SpawnSyncReturns; +type DockerExecute = (command: string, args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding) => DockerResult; + +export interface RemoveFailedBuildContainerOptions { + containerName: string; + creationToken: string; + createdId?: string | null; + dockerEnv?: NodeJS.ProcessEnv; + timeoutMs?: number; + execute?: DockerExecute; +} + +export function containerIdFromDockerOutput(output: unknown): string | null { + return String(output ?? '').split(/\r?\n/).map(line => line.trim()) + .find(line => CONTAINER_ID.test(line)) ?? null; +} + +function detail(result: DockerResult): string { + return String(result.stderr || result.stdout || result.error?.message + || `exit ${result.status}`).trim(); +} + +export function removeFailedBuildContainer({ containerName, creationToken, createdId = null, + dockerEnv = process.env, timeoutMs = 120_000, + execute = spawnSync as DockerExecute }: RemoveFailedBuildContainerOptions): { + removed: boolean; absent: boolean; id?: string; +} { + if (typeof containerName !== 'string' || !containerName) { + throw new Error('failed build-container cleanup requires a container name'); + } + if (typeof creationToken !== 'string' || !creationToken) { + throw new Error('failed build-container cleanup requires a creation token'); + } + + let id = containerIdFromDockerOutput(createdId); + if (!id) { + const inspected = execute('docker', ['inspect', '--format', + `{{.Id}} {{index .Config.Labels "${BUILD_CONTAINER_CREATION_LABEL}"}}`, containerName], { + encoding: 'utf8', env: dockerEnv, timeout: timeoutMs, + }); + if (inspected.status !== 0) { + const reason = detail(inspected); + if (/no such (?:object|container)/i.test(reason)) return { removed: false, absent: true }; + throw new Error(`cannot prove cleanup of failed container ${containerName}: ${reason}`); + } + const [inspectedId, label, ...extra] = String(inspected.stdout ?? '').trim().split(/\s+/); + if (!CONTAINER_ID.test(inspectedId ?? '') || label !== creationToken || extra.length > 0) { + throw new Error(`refusing to remove ${containerName}: its creation identity does not match`); + } + id = inspectedId ?? null; + } + + if (!id) throw new Error(`cannot prove cleanup of failed container ${containerName}`); + + const removed = execute('docker', ['rm', '-f', id], { + encoding: 'utf8', env: dockerEnv, timeout: timeoutMs, + }); + if (removed.status !== 0) { + throw new Error(`could not remove failed build container ${id}: ${detail(removed)}`); + } + return { removed: true, absent: false, id }; +} diff --git a/tools/stack-bench/container/recover-build-container.ts b/tools/stack-bench/container/recover-build-container.ts new file mode 100644 index 00000000000..acefebcb228 --- /dev/null +++ b/tools/stack-bench/container/recover-build-container.ts @@ -0,0 +1,77 @@ +import { spawnSync } from 'node:child_process'; +import type { SpawnSyncOptionsWithStringEncoding } from 'node:child_process'; + +import { updateBackendLease } from '../src/runtime/backend-lease.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; + +interface StoppedBuildContainer { + id: string; + running: false; +} + +interface LeaseContext { + path: string; + lease: BackendLease; +} + +export interface DockerExecuteResult { + status: number | null; + stdout?: string; + stderr?: string; + error?: Error; +} + +export type DockerExecute = (command: string, args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding) => DockerExecuteResult; + +export interface RecoverStoppedBuildContainerOptions { + existing: StoppedBuildContainer; + containerName: string; + leaseContext: LeaseContext; + backend: string; + dockerEnv?: NodeJS.ProcessEnv; + timeoutMs?: number; + execute?: DockerExecute; +} + +function clearBuildContainerLease(leaseContext: LeaseContext, backend: string, + containerId: string, description: string): LeaseContext { + const lease = updateBackendLease(leaseContext.path, { + token: leaseContext.lease.ownershipToken, backend, runId: leaseContext.lease.runId, + }, next => { + if (next.resources.buildContainer?.id !== containerId) { + throw new Error(`${description} ownership changed before recovery`); + } + next.resources.buildContainer = null; + return next; + }); + return { path: leaseContext.path, lease }; +} + +export function recoverStoppedBuildContainer({ existing, containerName, leaseContext, backend, + dockerEnv = process.env, timeoutMs = 120_000, + execute = spawnSync }: RecoverStoppedBuildContainerOptions): LeaseContext { + const prior = leaseContext?.lease?.resources?.buildContainer ?? null; + if (!existing || existing.running) throw new Error('recovery requires a stopped container'); + if (!prior || prior.name !== containerName || prior.id !== existing.id) { + throw new Error('stopped container does not match the authenticated lease'); + } + const removed = execute('docker', ['rm', existing.id], { + encoding: 'utf8', env: dockerEnv, timeout: timeoutMs, + }); + if (removed.status !== 0) { + throw new Error(`could not remove exact stopped leased container ${existing.id}: ` + + String(removed.stderr || removed.stdout || removed.error?.message || `exit ${removed.status}`).trim()); + } + return clearBuildContainerLease(leaseContext, backend, existing.id, 'stopped container'); +} + +export function clearMissingBuildContainerLease({ containerName, leaseContext, backend }: { + containerName: string; leaseContext: LeaseContext; backend: string; +}): LeaseContext { + const prior = leaseContext?.lease?.resources?.buildContainer ?? null; + if (!prior || prior.name !== containerName) { + throw new Error('missing container does not match the authenticated lease'); + } + return clearBuildContainerLease(leaseContext, backend, prior.id, 'missing container'); +} diff --git a/tools/stack-bench/container/run-build.ts b/tools/stack-bench/container/run-build.ts new file mode 100644 index 00000000000..8e4b1d5c4e7 --- /dev/null +++ b/tools/stack-bench/container/run-build.ts @@ -0,0 +1,703 @@ +#!/usr/bin/env node +// The build container must not expose Stack Bench source or grading material. +import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { chmodSync, existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync } from 'node:fs'; +import { resolve, dirname } from 'node:path'; +import { parseArgs } from 'node:util'; +import { leaseFromEnv, updateBackendLease } from '../src/runtime/backend-lease.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { leasedDatabaseEnvironment, STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { BUILD_CONTAINER_RESOURCE_LIMITS, DEFAULT_BUILD_IMAGE } + from '../src/composition/product-config.js'; +import { dockerMountArguments } from '../src/runtime/container-mount.js'; +import type { ContainerMount } from '../src/runtime/container-mount.js'; +import { dockerHostGatewayArguments, requireAttemptNetwork, attemptControllerImage, + recordAttemptCreation, ATTEMPT_CREATION_LABEL } from '../src/runtime/docker-network.js'; +import { packageRegistry, packageRegistryEnvironment } from '../src/runtime/package-registry.js'; +import { resolveContainerAuth } from './container-auth.js'; +import { hasRequiredBuildContainerIsolation, inspectBuildContainer, parseCgroupResources, + parsePublishedPorts, waitForBuildContainerReady } + from './build-container-inspection.js'; +import { reconcileCredentialBrokerReceipt } from './credential-broker-accounting.js'; +import { credentialBrokerDiagnostics, startCredentialBroker, stopCredentialBroker } + from './credential-broker-process.js'; +import { clearMissingBuildContainerLease, + recoverStoppedBuildContainer } from './recover-build-container.js'; +import { BUILD_CONTAINER_CREATION_LABEL, buildContainerName, containerIdFromDockerOutput, + removeFailedBuildContainer } from './reconcile-build-container.js'; +import { CODING_SESSION_TIMEOUT_MS } from '../src/agents/coding-session-timeouts.js'; +import { CODING_CONTAINER_AGENT, CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_CONTROL_DIR, + CODING_CONTAINER_PROCESS_IDENTITY, + codingContainerAgentEnvironment, codingContainerTranscriptHandoffCommands, + codingContainerWorkspaceHandoffCommands } + from '../src/runtime/coding-container-policy.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import { CODING_PROVIDERS, parseCodingProvider } from './coding-providers.js'; +import { validateProviderRoute, validateProviderOutputLimit } from '../src/agents/agent-adapter-contract.js'; +import { REPOSITORY_ROOT } from '../src/package-root.js'; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function runBuild(): Promise { +const { values } = parseArgs({ options: { + app: { type: 'string' }, backend: { type: 'string' }, 'prepare-only': { type: 'boolean' }, + provider: { type: 'string' }, image: { type: 'string' }, effort: { type: 'string' }, model: { type: 'string' }, + 'provider-route': { type: 'string' }, + 'max-output-tokens': { type: 'string' }, + 'max-budget-usd': { type: 'string' }, 'pricing-json': { type: 'string' }, + 'resume-session': { type: 'string' }, 'recover-stopped-container': { type: 'boolean' }, + 'completion-marker': { type: 'string' }, ports: { type: 'string' }, +} }); +const prepareOnly = values['prepare-only'] ?? false; + +const appDir = values.app; +if (!appDir) { console.error('run-build.js: --app is required'); process.exit(2); } +const backend = values.backend; +if (!backend) { console.error('run-build.js: --backend is required'); process.exit(2); } +let adapter; +try { adapter = STACK_ADAPTER_REGISTRY.get(backend); } +catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } +const provider = parseCodingProvider(values.provider ?? 'anthropic'); +const providerRoute = validateProviderRoute(provider, values['provider-route']); +const maxOutputTokens = validateProviderOutputLimit(provider, + values['max-output-tokens'] === undefined ? undefined : Number(values['max-output-tokens'])); +const codingProvider = CODING_PROVIDERS[provider]; +const DOCKER_TIMEOUT_MS = 120_000; +const DOCKER_PROBE_TIMEOUT_MS = 10_000; +const { uid: AGENT_UID, gid: AGENT_GID, home: AGENT_HOME } = CODING_CONTAINER_AGENT; +const CONTROLLER_GID = process.getgid?.() ?? 0; +const AGENT_ENVIRONMENT = codingContainerAgentEnvironment(); +const CONTROL_DIR = CODING_CONTAINER_CONTROL_DIR; +const REQUIRED_CAPABILITIES = Object.freeze([ + 'CHOWN', 'DAC_OVERRIDE', 'FOWNER', 'KILL', 'SETGID', 'SETUID', +]); +const REQUIRED_TMPFS = Object.freeze({ + '/tmp': 'rw,nosuid,nodev,mode=1777', + [AGENT_HOME]: `rw,nosuid,nodev,uid=${AGENT_UID},gid=${AGENT_GID},mode=0700`, + [`${AGENT_HOME}/.claude`]: `rw,nosuid,nodev,uid=${AGENT_UID},gid=${AGENT_GID},mode=0700`, + '/deps': 'rw,exec,nosuid,nodev,mode=0755', + [CONTROL_DIR]: 'rw,nosuid,nodev,mode=0700', +}); + +const REPO = REPOSITORY_ROOT; +const imageReference = values.image ?? DEFAULT_BUILD_IMAGE; +let imageIdentity; +try { imageIdentity = resolveContainerImage(imageReference); } +catch (error) { + console.error(`run-build.js: cannot resolve image ${imageReference}: ${errorMessage(error)}`); + process.exit(2); +} +const image = imageIdentity.id; +const effort = values.effort ?? ''; +const model = values.model ?? ''; +if (!prepareOnly && (!effort || !model)) { + console.error('run-build.js: --effort and --model are required'); + process.exit(2); +} +const maxBudgetUsd = values['max-budget-usd'] ?? null; +if (!prepareOnly && codingProvider.requiresBudget && maxBudgetUsd === null) { + throw new Error('this coding provider requires --max-budget-usd and explicit pricing'); +} +if (maxBudgetUsd !== null && (!Number.isFinite(Number(maxBudgetUsd)) || Number(maxBudgetUsd) <= 0)) { + console.error('run-build.js: --max-budget-usd must be a positive number'); + process.exit(2); +} +let pricing = null; +try { + const supplied = values['pricing-json'] ?? null; + if (supplied !== null) { + pricing = validatePricingAuthority(JSON.parse(supplied), { at: '--pricing-json' }); + } else if (maxBudgetUsd !== null) { + const rates = codingProvider.rates(model); + if (!rates) throw new Error(`no default pricing is recorded for model ${model}`); + pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } +} catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} +const resumeSession = values['resume-session'] ?? null; +if (resumeSession !== null + && !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(resumeSession)) { + console.error('run-build.js: --resume-session must be a UUID'); + process.exit(2); +} +const recoverStoppedContainer = values['recover-stopped-container'] ?? false; +const completionMarker = values['completion-marker'] ?? null; +if (!prepareOnly && !/^[A-Z][A-Z0-9_]*$/.test(completionMarker ?? '')) { + console.error('run-build.js: --completion-marker must be an uppercase marker'); + process.exit(2); +} +let ports: string[] = []; +try { ports = parsePublishedPorts(values.ports); } +catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } + +const containerPlan = adapter!.buildContainer.plan({ + repo: REPO, appDir, env: process.env, +}); + +// Auth is resolved in the controller. A short-lived broker forwards model API +// requests later. The coding container never receives the long-lived provider +// credential or a credential file. +const apiKey = process.env.STACK_BENCH_AGENT_API_KEY + ?? process.env[codingProvider.apiKeyEnvironment] ?? ''; +let auth = null; +if (!prepareOnly) { + try { auth = resolveContainerAuth({ provider, apiKey, env: process.env, credentialsPath: codingProvider.credentialPath }); } + catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } +} + +// Persist this run's transcript without exposing other local sessions. +const projects = prepareOnly ? null : codingProvider.projects(appDir); +const containerTranscripts = codingProvider.containerTranscripts; +function ensureAgentDirectory(directory: string): void { + mkdirSync(directory, { recursive: true, + mode: process.env.STACK_BENCH_APPLIANCE === '1' ? 0o700 : 0o777 }); + if (process.env.STACK_BENCH_APPLIANCE !== '1') chmodSync(directory, 0o777); +} + +ensureAgentDirectory(appDir); +if (projects) ensureAgentDirectory(projects); +for (const directory of containerPlan.ensureDirectories) ensureAgentDirectory(directory); + +const dockerEnv: NodeJS.ProcessEnv = { ...process.env, MSYS_NO_PATHCONV: '1' }; + +function resolveNetworkMode(): string { + if (process.env.STACK_BENCH_APPLIANCE !== '1') return 'bridge'; + return requireAttemptNetwork(leaseFromEnv(process.env, { backend, active: true }).lease); +} + +let expectedNetworkMode: string; +try { expectedNetworkMode = resolveNetworkMode(); } +catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} + +const inspectContainer = (name: string) => inspectBuildContainer(name, + { env: dockerEnv, timeoutMs: DOCKER_TIMEOUT_MS }); + +const hasRequiredIsolation = (container: NonNullable>, + expectedMounts: ContainerMount[]): boolean => hasRequiredBuildContainerIsolation(container, { + expectedMounts, + requiredTmpfs: REQUIRED_TMPFS, + requiredCapabilities: REQUIRED_CAPABILITIES, + pidsLimit: BUILD_CONTAINER_RESOURCE_LIMITS.pids, + cpuCount: BUILD_CONTAINER_RESOURCE_LIMITS.cpuCount, + memoryBytes: BUILD_CONTAINER_RESOURCE_LIMITS.memoryBytes, + memorySwapBytes: BUILD_CONTAINER_RESOURCE_LIMITS.memorySwapBytes, + image, +}); + +const expectedMounts: ContainerMount[] = [ + { kind: 'bind' as const, source: resolve(appDir), target: CODING_CONTAINER_APP_ROOT, readOnly: false }, + ...(projects ? [{ kind: 'bind' as const, source: projects, + target: containerTranscripts, readOnly: false }] : []), + ...containerPlan.mounts, +]; + +// Only the lease's immutable container id grants reuse or deletion authority. +let leaseContext; +try { leaseContext = leaseFromEnv(process.env, { backend, active: true }); } +catch (error) { + console.error(`run-build.js: an authenticated active backend lease is required: ${errorMessage(error)}`); + process.exit(3); +} + +// App directories can share a parent. Container identity belongs to the lease. +const containerName = buildContainerName(leaseContext.lease); +let existing = inspectContainer(containerName); +const priorContainer = leaseContext.lease.resources.buildContainer ?? null; +if (existing) { + if (!priorContainer) { + console.error(`run-build.js: refusing to adopt existing unleased container ${containerName}`); + process.exit(3); + } + if (priorContainer.name !== containerName || priorContainer.id !== existing.id) { + console.error(`run-build.js: existing container ${containerName}/${existing.id} does not match lease ` + + `${priorContainer.name}/${priorContainer.id}`); + process.exit(3); + } + if (!existing.running) { + if (!recoverStoppedContainer) { + console.error(`run-build.js: leased container ${containerName} stopped unexpectedly; refusing to replace it`); + process.exit(3); + } + try { + leaseContext = recoverStoppedBuildContainer({ existing: { ...existing, running: false }, containerName, leaseContext, backend, + dockerEnv, timeoutMs: DOCKER_TIMEOUT_MS }); + existing = null; + } catch (error) { + console.error(`run-build.js: could not recover stopped container: ${errorMessage(error)}`); + process.exit(3); + } + } + if (existing && existing.networkMode !== expectedNetworkMode) { + console.error(`run-build.js: leased container ${containerName} uses network ${existing.networkMode}, ` + + `expected ${expectedNetworkMode}`); + process.exit(3); + } + if (existing?.unsafeCredentialExposure) { + console.error(`run-build.js: leased container ${containerName} was created with a provider credential; ` + + 'reconcile the run and start it with the isolated credential broker'); + process.exit(3); + } + if (existing && !hasRequiredIsolation(existing, expectedMounts)) { + console.error(`run-build.js: leased container ${containerName} does not have the required isolation`); + process.exit(3); + } +} else if (priorContainer) { + const leasedById = inspectContainer(priorContainer.id); + if (leasedById) { + console.error(`run-build.js: leased container ${priorContainer.id} still exists under an unexpected name`); + process.exit(3); + } + if (!recoverStoppedContainer) { + console.error(`run-build.js: leased container ${priorContainer.name}/${priorContainer.id} is missing`); + process.exit(3); + } + try { + leaseContext = clearMissingBuildContainerLease({ containerName, leaseContext, backend }); + } catch (error) { + console.error(`run-build.js: could not recover missing container lease: ${errorMessage(error)}`); + process.exit(3); + } +} + +// Create it if this is the first round of the run; reuse it for every round +// after, so a repair finds the app, its node_modules and its servers exactly +// where the build round left them. +let containerInspection = existing; +if (!existing) { + const creationToken = randomBytes(16).toString('hex'); + if (leaseContext.lease.resources.network) updateBackendLease(leaseContext.path, + { token: leaseContext.lease.ownershipToken }, next => { + (next.resources.creationIntents ??= {}).build = { name: containerName, creationToken }; + return next; + }); + const create = [ + 'create', '--init', '--name', containerName, + '--label', `${BUILD_CONTAINER_CREATION_LABEL}=${creationToken}`, + '--label', `${ATTEMPT_CREATION_LABEL}=${creationToken}`, + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges:true', + '--pids-limit', String(BUILD_CONTAINER_RESOURCE_LIMITS.pids), + '--cpus', String(BUILD_CONTAINER_RESOURCE_LIMITS.cpuCount), + '--memory', String(BUILD_CONTAINER_RESOURCE_LIMITS.memoryBytes), + '--memory-swap', String(BUILD_CONTAINER_RESOURCE_LIMITS.memorySwapBytes), + // The agent may write the app, its own home directory, and temporary files. + // It must not replace system binaries or libraries used by later grading. + '--read-only', + '-v', `${resolve(appDir)}:${CODING_CONTAINER_APP_ROOT}`, + ]; + for (const capability of REQUIRED_CAPABILITIES) create.push('--cap-add', capability); + for (const [path, options] of Object.entries(REQUIRED_TMPFS)) { + create.push('--tmpfs', `${path}:${options}`); + } + create.push('--network', expectedNetworkMode); + create.push(...dockerHostGatewayArguments(expectedNetworkMode)); + if (projects) create.push('-v', `${projects}:${containerTranscripts}`); + // The selected adapter owns every stack-specific mount. Giving a treatment + // another stack's artifacts would violate the "only artifacts under test" + // boundary. + for (const requiredPath of containerPlan.requiredPaths) { + if (!existsSync(requiredPath)) { + console.error(`run-build.js: ${backend} container artifact is missing: ${requiredPath}`); + process.exit(2); + } + } + for (const mount of containerPlan.mounts) { + try { create.push(...dockerMountArguments(mount)); } + catch (error) { + console.error(`run-build.js: ${backend} adapter returned an invalid container mount: ${errorMessage(error)}`); + process.exit(2); + } + } + + // Publish the track's ports for the host grader. + if (expectedNetworkMode === 'bridge') for (const p of ports) create.push('-p', `127.0.0.1:${p}:${p}`); + // Container-level, so the agent's installs and every later exec share it. + for (const [key, value] of Object.entries( + packageRegistryEnvironment(packageRegistry(), expectedNetworkMode, leaseContext.lease.resources.network))) { + create.push('-e', `${key}=${value}`); + } + + // `--init` gives the container a real PID 1. Without it the dev servers the + // build leaves behind are reparented to `sleep`, which never reaps them. + const init = 'export HOME=/tmp npm_config_cache=/tmp/npm-cache; ' + + containerPlan.init; + create.push('-w', CODING_CONTAINER_APP_ROOT, image, 'sh', '-c', init); + + const made = spawnSync('docker', create, { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + if (made.status !== 0) { + console.error(`run-build.js: could not create ${containerName}`); + console.error(made.stderr || made.stdout || made.error?.message || ''); + try { + removeFailedBuildContainer({ containerName, creationToken, + createdId: containerIdFromDockerOutput(made.stdout), dockerEnv, + timeoutMs: DOCKER_TIMEOUT_MS }); + } catch (cleanupError) { + console.error(`run-build.js: ${errorMessage(cleanupError)}`); + process.exit(3); + } + process.exit(2); + } + + const createdId = containerIdFromDockerOutput(made.stdout); + try { containerInspection = inspectContainer(containerName); } + catch (error) { + console.error(`run-build.js: cannot inspect ${containerName}: ${errorMessage(error)}`); + } + if (!containerInspection) { + try { + removeFailedBuildContainer({ containerName, creationToken, createdId, dockerEnv, + timeoutMs: DOCKER_TIMEOUT_MS }); + } catch (cleanupError) { + console.error(`run-build.js: ${errorMessage(cleanupError)}`); + process.exit(3); + } + console.error(`run-build.js: cannot inspect ${containerName}`); + process.exit(2); + } + if (containerInspection.unsafeCredentialExposure + || !hasRequiredIsolation(containerInspection, expectedMounts)) { + try { + removeFailedBuildContainer({ containerName, creationToken, createdId, dockerEnv, + timeoutMs: DOCKER_TIMEOUT_MS }); + } catch (cleanupError) { + console.error(`run-build.js: ${errorMessage(cleanupError)}`); + process.exit(3); + } + console.error(`run-build.js: created container ${containerName} does not have the required isolation`); + process.exit(2); + } +} + +if (!containerInspection) { + console.error(`run-build.js: cannot inspect ${containerName}`); + process.exit(2); +} +const { id: containerId, image: containerImage } = containerInspection; +try { + const { path, lease } = leaseContext; + const prior = lease.resources.buildContainer; + if (prior && (prior.name !== containerName || prior.id !== containerId)) { + throw new Error(`running container ${containerName}/${containerId} does not match lease ` + + `${prior.name}/${prior.id}`); + } + updateBackendLease(path, { token: lease.ownershipToken, backend, runId: lease.runId }, next => { + next.resources.buildContainer = { + name: containerName, id: containerId, image: containerImage, owned: true, running: existing !== null, + networkMode: expectedNetworkMode, + resourceLimits: structuredClone(BUILD_CONTAINER_RESOURCE_LIMITS), + }; + return next; + }); +} catch (error) { + // Creation succeeded but ownership recording did not. Remove only the exact + // id created by this invocation; leaving an unleased container is not safe. + if (!existing) { + spawnSync('docker', ['rm', '-f', containerId], { + stdio: 'ignore', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + } + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(3); +} + +if (!existing) { + const started = spawnSync('docker', ['start', containerId], { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + if (started.status !== 0) { + console.error(`run-build.js: could not start leased container ${containerName}/${containerId}`); + console.error(started.stderr || started.stdout || started.error?.message || ''); + process.exit(2); + } + try { + const { path, lease } = leaseContext; + updateBackendLease(path, { token: lease.ownershipToken, backend, runId: lease.runId }, next => { + if (next.resources.buildContainer?.id !== containerId) { + throw new Error(`leased container changed before start: expected ${containerId}`); + } + next.resources.buildContainer.running = true; + return next; + }); + } catch (error) { + console.error(`run-build.js: started container ownership could not be recorded: ${errorMessage(error)}`); + process.exit(3); + } +} + +if (containerPlan.readyFile) { + // Wait until SDK staging finishes before starting the paid session. + try { + waitForBuildContainerReady(containerId, containerPlan.readyFile, + containerPlan.readyDescription ?? `${backend} setup`, { env: dockerEnv }); + } catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); + } +} + +if (process.env.STACK_BENCH_APPLIANCE === '1') { + const writableTargets = [AGENT_HOME, + ...expectedMounts.filter(mount => !mount.readOnly).map(mount => mount.target)]; + for (const [command, commandArgs] of [ + ['chown', ['-R', `${AGENT_UID}:${CONTROLLER_GID}`, '--', ...writableTargets]], + ['chmod', ['-R', 'u+rwX,g+rwX,o-rwx', '--', ...writableTargets]], + ] as const) { + const permissions = spawnSync('docker', ['exec', containerName, command, ...commandArgs], { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); + if (permissions.status !== 0) { + console.error(`run-build.js: could not secure writable paths in ${containerName}`); + console.error(permissions.stderr || permissions.stdout || permissions.error?.message || ''); + process.exit(2); + } + } +} + +// A nested transcript mount makes Docker create its parent directories as +// root. Confirm that the coding runner can create its private session state before a +// provider request can spend money. +const homeProbe = spawnSync('docker', [ + 'exec', '--user', `${AGENT_UID}:${AGENT_GID}`, '-e', `HOME=${AGENT_HOME}`, + containerName, 'sh', '-c', + 'umask 077; mkdir -p "$1" && test -w "$1"', 'home-probe', dirname(containerTranscripts), +], { encoding: 'utf8', env: dockerEnv, timeout: DOCKER_PROBE_TIMEOUT_MS }); +if (homeProbe.status !== 0) { + console.error(`run-build.js: agent home is not writable in ${containerName}`); + console.error(homeProbe.stderr || homeProbe.stdout || homeProbe.error?.message || ''); + process.exit(2); +} + +// Docker bind sources must be the same filesystem the controller audits. +if (projects) { + const probe = `.mount-probe-${randomBytes(12).toString('hex')}`; + const expected = randomBytes(24).toString('hex'); + const path = resolve(projects, probe); + writeFileSync(path, expected, { mode: 0o644 }); + try { + const result = spawnSync('docker', ['exec', '--user', `${AGENT_UID}:${AGENT_GID}`, + containerName, 'cat', `${containerTranscripts}/${probe}`], + { encoding: 'utf8', env: dockerEnv, timeout: DOCKER_PROBE_TIMEOUT_MS }); + if (result.status !== 0 || result.stdout !== expected) { + throw new Error('Transcript mount is not shared with the controller. Use the shared controller HOME before starting a session.'); + } + } finally { unlinkSync(path); } +} + +if (prepareOnly) { + process.stdout.write(`${JSON.stringify({ containerName, + identity: `${containerId} ${containerImage}`, + networkMode: expectedNetworkMode })}\n`); + return; +} + +const args = ['exec', '-i', '--user', `${AGENT_UID}:${AGENT_GID}`, '-w', CODING_CONTAINER_APP_ROOT]; + +args.push('-e', `HOME=${AGENT_ENVIRONMENT.HOME}`, '-e', `USER=${AGENT_ENVIRONMENT.USER}`); +const leasedEnvironment = leasedDatabaseEnvironment(adapter!, { + database: leaseContext.lease.resources.database, networkMode: expectedNetworkMode, lease: leaseContext.lease, +}); +for (const key of Object.keys(leasedEnvironment)) args.push('-e', key); +const dockerExecEnv: NodeJS.ProcessEnv = { ...process.env, ...leasedEnvironment, MSYS_NO_PATHCONV: '1' }; +if (!projects) throw new Error('transcript directory is unavailable'); +// Forward only benchmark-owned environment settings. +if (provider === 'anthropic' && process.env.MAX_THINKING_TOKENS) { + args.push('-e', `MAX_THINKING_TOKENS=${process.env.MAX_THINKING_TOKENS}`); +} + +// Record the exact remote PID. Killing the local `docker exec` client does not +// guarantee that the coding runner stops inside the long-lived build container. +const invocationToken = randomBytes(16).toString('hex'); +const processRecord = `${CODING_CONTAINER_PROCESS_IDENTITY.recordPrefix}${invocationToken}.pid`; +const sessionWrapper = 'umask 022; record="$1"; shift; ' + + 'start="$(awk \'{print $22}\' /proc/$$/stat)" || exit 1; ' + + 'printf \'%s %s\\n\' "$$" "$start" > "$record"; exec "$@"'; + +if (!auth) throw new Error('container authentication is unavailable'); +let credentialBroker: Awaited> | null = null; +try { + const docker = leaseContext.lease.resources.network ? (() => { + const intent = recordAttemptCreation(leaseContext.path, leaseContext.lease, 'broker'); + return { imageId: attemptControllerImage(), + networkContainerId: leaseContext.lease.resources.network!.namespaceContainerId!, + ...intent, privateDirectory: dirname(leaseContext.path), + onCreated: (container: import('./credential-broker-process.js').CredentialBrokerContainer) => { + updateBackendLease(leaseContext.path, { token: leaseContext.lease.ownershipToken }, next => { + next.resources.brokerContainer = container; return next; + }); + } }; + })() : undefined; + credentialBroker = await startCredentialBroker(auth, + { networkMode: expectedNetworkMode, deadlineMs: CODING_SESSION_TIMEOUT_MS, model, + providerRoute, maxOutputTokens, + docker, + maxBudgetUsd: maxBudgetUsd === null ? null : Number(maxBudgetUsd), + pricingRates: maxBudgetUsd === null ? null : pricing!.rates }); +} catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} +if (!credentialBroker) throw new Error('credential broker is unavailable'); +const tokenEnvironment = codingProvider.tokenEnvironment; +dockerExecEnv[tokenEnvironment] = credentialBroker.sessionToken; +args.push('-e', tokenEnvironment, ...codingProvider.environment(credentialBroker.baseUrl).flatMap(value => ['-e', value]), + containerName, 'sh', '-c', sessionWrapper, CODING_CONTAINER_PROCESS_IDENTITY.sessionLabel, + processRecord, + codingProvider.executable, ...codingProvider.args({ model, effort, baseUrl: credentialBroker.baseUrl, + resumeSession, maxBudgetUsd })); + +// MSYS_NO_PATHCONV: Git Bash rewrites container-side paths like /app into +// Windows paths (C:/Program Files/Git/app) and every mount silently lands +// somewhere wrong. +const promptInput = process.stdin.isTTY ? '' : readFileSync(0, 'utf8'); +function signalSession(signal: 'TERM' | 'KILL') { + const script = 'record="$1"; signal="$2"; test -r "$record" || exit 4; ' + + 'read -r pid expected < "$record"; ' + + 'current="$(awk \'{print $22}\' "/proc/$pid/stat" 2>/dev/null)" || exit 5; ' + + 'test "$current" = "$expected" || exit 3; kill "-$signal" "$pid"'; + // The agent can rewrite its own record, so signal with the agent's authority, not root's. + return spawnSync('docker', ['exec', '--user', `${AGENT_UID}:${AGENT_GID}`, containerName, 'sh', '-c', script, + CODING_CONTAINER_PROCESS_IDENTITY.stopLabel, processRecord, signal], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); +} +function terminateSession(child: { kill(signal?: NodeJS.Signals): boolean }): void { + const term = signalSession('TERM'); + if (term.status !== 0) child.kill('SIGTERM'); + const force = setTimeout(() => { + signalSession('KILL'); + child.kill('SIGKILL'); + }, 5_000); + force.unref(); +} + +let res: Awaited> | undefined; +let sessionError: unknown = null; +let brokerLedger = null; +let brokerDiagnostics = null; +const cleanupErrors: string[] = []; +const runCleanupCommand = (description: string, command: readonly string[]): void => { + const result = spawnSync('docker', ['exec', containerName, ...command], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); + if (result.status !== 0) cleanupErrors.push(`${description}: ${String(result.stderr || result.stdout + || result.error?.message || `exit ${result.status}`).trim()}`); +}; +try { + res = await codingProvider.run({ command: 'docker', args, input: promptInput, + env: dockerExecEnv, timeoutMs: CODING_SESSION_TIMEOUT_MS, terminate: terminateSession, + projects, containerId, marker: completionMarker as string, model, + pricingRates: pricing?.rates ?? null, resumeSession }); +} catch (error) { + sessionError = error; +} finally { + brokerLedger = await stopCredentialBroker(credentialBroker); + brokerDiagnostics = credentialBrokerDiagnostics(credentialBroker); + if (credentialBroker.container && brokerDiagnostics?.termination?.exited + && !brokerDiagnostics.errors.some(error => error.type === 'cleanup-error')) { + updateBackendLease(leaseContext.path, { token: leaseContext.lease.ownershipToken }, next => { + delete next.resources.brokerContainer; + delete next.resources.creationIntents?.broker; + return next; + }); + } + for (const command of codingContainerTranscriptHandoffCommands(CONTROLLER_GID, containerTranscripts)) { + runCleanupCommand('transcript handoff', command); + } + const handoff = process.env.STACK_BENCH_APPLIANCE === '1' + ? codingContainerWorkspaceHandoffCommands(CONTROLLER_GID) + : [['chmod', '-R', 'a+rwX', CODING_CONTAINER_APP_ROOT]]; + for (const command of handoff) runCleanupCommand('workspace handoff', command); + runCleanupCommand('process-record cleanup', ['rm', '-f', processRecord]); +} + +if (sessionError) { + if (cleanupErrors.length) { + throw new AggregateError([sessionError, ...cleanupErrors.map(message => new Error(message))], + 'coding session and container cleanup failed'); + } + throw sessionError; +} +if (!res) throw new Error('coding session returned no process result'); +if (cleanupErrors.length) { + res.status = res.status === 0 ? 3 : res.status ?? 3; + res.stderr = `${res.stderr ?? ''}${res.stderr ? '\n' : ''}` + + `run-build.js: container cleanup failed: ${cleanupErrors.join('; ')}\n`; +} + +const cliResult = codingProvider.result(String(res.stdout ?? '').trim(), appDir, invocationToken); +if (cliResult) cliResult.stack_bench_auth_mode = auth.mode; +const memory = spawnSync('docker', ['exec', containerName, 'sh', '-c', + 'for f in memory.events memory.current memory.peak memory.max pids.current pids.peak pids.max pids.events; do ' + + 'p="/sys/fs/cgroup/$f"; if test -r "$p"; then echo "[$f]"; cat "$p"; fi; done'], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, +}); +const resources = { + ...(memory.status === 0 ? parseCgroupResources(memory.stdout) + : { buildContainerMemory: null, buildContainerPids: null }), + memoryProbeError: memory.status === 0 ? null + : memory.stderr?.trim() || (memory.error instanceof Error ? memory.error.message : null) + || `exit ${memory.status}`, +}; +if (maxBudgetUsd !== null) { + const reconciled = reconcileCredentialBrokerReceipt({ + ledger: brokerLedger, + provider, + cliResult, + model, + maxBudgetUsd: Number(maxBudgetUsd), + pricingRates: pricing!.rates, + brokerDiagnostics, + }); + reconciled.result.stack_bench_resources = resources; + res.stdout = `${JSON.stringify(reconciled.result)}\n`; + if (!reconciled.ok) { + res.status = res.status === 0 ? 3 : res.status ?? 3; + res.stderr = `${res.stderr ?? ''}${res.stderr ? '\n' : ''}` + + `run-build.js: ${reconciled.receipt.error}\n`; + } +} else if (cliResult && typeof cliResult === 'object' && !Array.isArray(cliResult)) { + cliResult.stack_bench_credential_broker = brokerDiagnostics; + cliResult.stack_bench_resources = resources; + res.stdout = `${JSON.stringify(cliResult)}\n`; +} + +if ((res.status ?? 1) !== 0) { + const state = spawnSync('docker', ['inspect', '--format', '{{json .State}}', containerName], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); + let containerState = null; + try { containerState = JSON.parse(state.stdout?.trim() || 'null'); } catch { /* retain raw text below */ } + const diagnostic = { + schemaVersion: 1, + kind: 'coding-process-exit', + status: res.status ?? null, + signal: res.signal ?? null, + error: res.error instanceof Error ? res.error.message : null, + container: containerState ?? { inspectError: state.stderr?.trim() + || (state.error instanceof Error ? state.error.message : null) }, + cgroupMemory: memory.stdout?.trim() || null, + cgroupProbeError: memory.status === 0 ? null + : memory.stderr?.trim() || (memory.error instanceof Error ? memory.error.message : null) + || `exit ${memory.status}`, + }; + process.stderr.write(`STACK_BENCH_CODING_PROCESS_DIAGNOSTIC ${JSON.stringify(diagnostic)}\n`); +} + +if (res.stdout) process.stdout.write(res.stdout); +if (res.stderr) process.stderr.write(res.stderr); +if (res.error) process.stderr.write(`run-build.js: coding session failed: ${errorMessage(res.error)}\n`); +process.exit(res.status ?? 1); +} + +await runBuild(); diff --git a/tools/stack-bench/container/spacetime-dev.ts b/tools/stack-bench/container/spacetime-dev.ts new file mode 100644 index 00000000000..161dee0b0a2 --- /dev/null +++ b/tools/stack-bench/container/spacetime-dev.ts @@ -0,0 +1,94 @@ +// Standalone development process command; Node built-ins only. +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync, writeFileSync, openSync, closeSync, rmSync, realpathSync } from 'node:fs'; +import { resolve, relative, isAbsolute, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; + +function marker(pid: number): string | null { + try { + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); + const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + return fields[0] === 'Z' ? null : fields[19] ?? null; + } catch { return null; } +} + +export function validateProject(root: string, server: string, database: string): void { + const read = (name: string) => JSON.parse(readFileSync(join(root, name), 'utf8')); + const config = { ...read('spacetime.json'), + ...(existsSync(join(root, 'spacetime.local.json')) ? read('spacetime.local.json') : {}) }; + if (config.server !== server || config.database !== database || config.publish !== undefined) { + throw new Error('Configure one database using the supplied server URL and database name.'); + } + const contained = (path: unknown): string => { + if (typeof path !== 'string' || !path.trim()) throw new Error('Configure module-path and generate out-dir.'); + const full = resolve(root, path); + let existing = full; + while (!existsSync(existing)) existing = dirname(existing); + for (const candidate of [full, realpathSync(existing)]) { + const rel = relative(realpathSync(root), candidate); + if (isAbsolute(rel) || rel === '..' || rel.startsWith('../')) throw new Error('Project paths must stay inside the application directory.'); + } + return full; + }; + if (!existsSync(contained(config['module-path']))) throw new Error('Create the module directory before starting development.'); + if (!Array.isArray(config.generate) || !config.generate.length) throw new Error('Configure at least one generate target.'); + for (const target of config.generate) { + if (!target || target.language !== 'typescript') throw new Error('Configure TypeScript generate targets.'); + contained(target['out-dir']); + if (target['module-path'] !== undefined) contained(target['module-path']); + if ((target.server !== undefined && target.server !== server) + || (target.database !== undefined && target.database !== database)) throw new Error('Generate targets must use the supplied database.'); + } +} + +// Calls are serialized by the installed flock wrapper. The detached watcher is +// still owned by the agent UID and the existing container teardown stops it. +export async function main(args: string[]): Promise { + const [root, state, cli, server, database, command = 'status', ...extra] = args; + if (!root || !state || !cli || !server || !database || extra.length + || !['start', 'status', 'stop'].includes(command)) throw new Error('Usage: spacetime-dev start|status|stop'); + const record = join(state, 'process.json'), ready = join(state, 'ready'), log = join(state, 'watcher.log'); + const previous = existsSync(record) ? JSON.parse(readFileSync(record, 'utf8')) : null; + const alive = () => previous && Number.isSafeInteger(previous.pid) && previous.pid > 1 + && typeof previous.marker === 'string' && marker(previous.pid) === previous.marker; + if (command === 'stop') { + if (alive()) { + // Stop the whole build group together, including a compiler child. + process.kill(-previous.pid, 'SIGKILL'); + for (let i = 0; i < 50 && alive(); i++) await delay(100); + if (alive()) throw new Error(`Watcher did not stop. Read ${log}`); + } + rmSync(record, { force: true }); rmSync(ready, { force: true }); + console.log(`Stopped. Log: ${log}`); return; + } + if (alive()) { + console.log(`${existsSync(ready) ? 'Running; initial publish and bindings completed' : 'Starting; initial publish not yet confirmed'}. Log: ${log}`); + return; + } + if (command === 'status') { console.log(`Not running. Log: ${log}`); return; } + validateProject(root, server, database); + rmSync(ready, { force: true }); + const output = openSync(log, 'w', 0o600); + const child = spawn(cli, ['dev', '--yes', '--delete-data=never', '--server-only', '--ready-file', ready], + { cwd: root, detached: true, stdio: ['ignore', output, output] }); + closeSync(output); + await new Promise((done, reject) => { child.once('spawn', done); child.once('error', reject); }); + const pid = child.pid!; + const identity = marker(pid); + if (!identity) throw new Error(`Watcher exited at startup. Read ${log}`); + writeFileSync(record, JSON.stringify({ pid, marker: identity })); + child.unref(); + // Bounded startup observation; a slow compile keeps running and status can + // inspect it later. Never mistake a live process for a completed publish. + for (let i = 0; i < 50; i++) { + if (marker(pid) !== identity) throw new Error(`Watcher exited. Read ${log}`); + if (existsSync(ready)) { console.log(`Running; initial publish and bindings completed. Log: ${log}`); return; } + await delay(100); + } + console.log(`Starting; initial publish not yet confirmed. Run spacetime-dev status. Log: ${log}`); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)).catch(error => { console.error(error.message); process.exitCode = 1; }); +} diff --git a/tools/stack-bench/container/spacetimedb-binaries.json b/tools/stack-bench/container/spacetimedb-binaries.json new file mode 100644 index 00000000000..51e4e8a5fd4 --- /dev/null +++ b/tools/stack-bench/container/spacetimedb-binaries.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 2, + "platform": "linux/amd64", + "builderImage": "rust:1.93-slim-bookworm@sha256:8f8609d448e821fbc0e44241bc5ca4ce49663cc6306ff1a17f655a0e2a7cd084", + "source": { + "identityScheme": "git-object-content-v1", + "revision": "81ab0550c6e07cd24b264a9c32d9240ff4cf2d75", + "sha256": "bacff6fdae08ea390eabbfd1fe908b3cf1338a744272c0684d7857c77f630f64", + "files": 2599 + }, + "binaries": { + "spacetimedb-cli": { + "sha256": "2f671b85f51beac7ab6d32ccf707f08e3ca936d876fb2bdc874e02e497e5287f", + "size": 47642320 + }, + "spacetimedb-standalone": { + "sha256": "53df19d77e81ef61426e0bfd154d78e94b4e9a04754647d79777ff4e056163c9", + "size": 132331544 + } + } +} diff --git a/tools/stack-bench/dashboard/README.md b/tools/stack-bench/dashboard/README.md new file mode 100644 index 00000000000..95571d67472 --- /dev/null +++ b/tools/stack-bench/dashboard/README.md @@ -0,0 +1,205 @@ +# Stack Bench dashboard + +The dashboard is an optional local view over Stack Bench results. It does not +schedule attempts, grade applications, or repair source itself. +Campaign plans, durable campaign state, and run artifacts remain the source of +truth. Appliance controls call the shared job and campaign operations. The +dashboard does not have a separate execution engine. + +## Pages + +- Campaigns (`/`) — a lane per running attempt, then one row per campaign with + its shape, status, and per-stack score. A campaign whose plan or state this + build cannot read appears with the status `unreadable` and the reason in + place of its title. +- Campaign (`/c/:key`) — plan facts, completion, scores, repairs, time, spend, + and attempts. The chart switches between completion, cost, and distribution + with `?chart=completion|cost|distribution`. Features are the default unit; + `&unit=features|checks` switches completion and distribution. Cost keeps these + controls visible but disabled. Toggle a stack or a repetition to show or hide it. + Dependency campaigns add questline rows, which + `?questlines=grid|graph|replay` switches between; `&step=N` moves the replay + cursor. Sequential campaigns show one row pair per level instead. +- Attempt (`/c/:key/a/:attemptId`) — attempt figures, the dependency graph, and + `?tab=checks|transcript|screenshots|files|log`. The transcript shows build and + repair sessions, including tool calls. It follows live work at the newest + page and pauses updates while you read earlier messages. The log tab shows + controller output separately. + In Checks, expand a requirement to see the recorded status, summary, expected + value and observation for each grade. Missing details remain explicit. These + are raw grading observations, including unsuccessful repairs; the accepted + score remains in the run summary. Blocked, inconclusive and harness failures + retain their recorded status. Credentials and marked sensitive details are omitted. +- New run (`/new`) — select workload, level, stacks, models, guidance, repetitions, + repairs, and limits. Review the attempt count and cost cap, then start. + The default is progressive dependency work with SDK skills and dev workflow on, + when available, 240 minutes and a $50 normalized cost cap per attempt. + **SDK skills** and **Dev workflow** are independent + [guidance choices](../docs/prompting.md#guidance-profiles); compare them in + separate runs. Review shows work delivery separately from concurrency. When + repeating a study, compare the saved campaign's mode, guidance, model, repair + policy, and budgets before launch; do not infer them from “L3” or “3×3.” +- Saved plans (`/plans`) — inspect the exact configuration behind each run. +- Checks (`/checks`) — search current check definitions and inspect setup, steps, + waits, pass rules, defect-control targets, and exact inputs. Defaults to ecommerce + dependency L1–L3; other selections are available through a filter. This reads + the dashboard's installed definitions, not a historical campaign's frozen grader. + +## Modes + +"Before repairs" uses first-build evidence before repairs at each level. +Later levels retain earlier fixes and feedback; this is not a feedback-free run. + +Inside the appliance (`STACK_BENCH_APPLIANCE=1`) the dashboard runs in +controller mode: Start and Resume launch the CLI in an owned controller +container. Stop sends a durable request to that controller instance. +Stop interrupts the active attempt; it does not pause it. Resume starts +scheduled dependency work and does not restart a stopped sequential attempt. +It cannot restore a lost database or agent session. A planned depth pause uses +the CLI's `pause-status` and `continue-depth` commands and requires the original +controller to stay running. See [planned depth pause](../appliance/README.md#planned-depth-pause). +Elsewhere it runs read-only and those controls are unavailable; +`GET /api/health` reports which mode is active. + +From `tools/stack-bench`, `npm run dashboard` starts a read-only host view over +`tools/stack-bench/results`. Pass `--port` to move it off 7331 and `--results` +to point it at another results directory. + +## Appliance + +Run these commands from `tools/stack-bench` after the +[appliance setup](../appliance/README.md). + +```sh +docker compose --env-file operator.env \ + -f appliance/docker-compose.yaml --profile dashboard up -d dashboard +``` + +Open `http://127.0.0.1:7331`. Docker publishes that port only on the host's +loopback interface. Stop it with: + +```sh +docker compose --env-file operator.env \ + -f appliance/docker-compose.yaml --profile dashboard stop dashboard +``` + +Run controls need no separate password. This is a local, single-user dashboard: +the port binds to loopback, requests must use a loopback Host, and writes require +the exact browser origin and a per-server CSRF token. Other websites cannot read +that token. Local processes that can access the dashboard are trusted. Do not +publish this service through a proxy or on a shared network without authentication. +Model credentials remain private. Starting a run submits +an idempotent execution job and starts its worker in an +owned controller. Retrying Start with the same reviewed settings returns the same +job. A queued job has a status page before its campaign artifacts exist. + +The campaign page lists every attempt with its variant and repetition. Check +completion uses all selected checks, including checks not reached. Feature +completion requires all selected checks of a feature to pass, including its +production guarantees. Weighted score remains a separate measure. Spend includes +all executions and shows upper bounds and unknown values. Different comparison +conditions do not share one score average. Files links to the report and its +public export manifest; the manifest lists evidence and any reconstruction gaps. +Charts connect saved observations; intermediate values are not measured. +Comparison summaries and the distribution use eligible completed attempts. +Excluded attempts remain labelled in chart controls and the Runs table. Cost and +progress-over-time charts retain their observations, with the same exclusion label. +Live progress and total spend include unfinished work; total spend also includes +excluded attempts. These operational values are separate from comparison metrics. + +For single-execution Claude Code and Codex attempts, `~$` marks a live estimate +from completed response usage at the plan's pinned rates. Final receipts replace +that estimate. Unsupported or incomplete usage keeps the saved cost visible. Live +estimates do not enter scores, reports, or budget enforcement. Planned depth +holds show their paused state; elapsed time includes those holds. + +## Routes + +| route | returns | +| --- | --- | +| `GET /api/health` | `read-only` or `controller` | +| `GET /api/overview` | one summary per campaign | +| `GET /api/campaigns/:key` | the campaign sheet | +| `GET /api/campaigns/:key/live` | live spend, cost observations, activity, and phase | +| `GET /api/campaigns/:key/progression` | the dependency graph and its replay | +| `GET /api/campaigns/:key/attempts/:id/checks` | per-check outcome and history | +| `GET /api/campaigns/:key/attempts/:id/package` | the evidence listing | +| `GET /api/campaigns/:key/attempts/:id/log?from=N` | log bytes after `N` | +| `GET /api/campaigns/:key/attempts/:id/transcript` | selected session and paged transcript messages | +| `GET /api/campaigns/:key/attempts/:id/time` | time allowance, grants, and continuation eligibility | +| `POST /api/campaigns/:key/attempts/:id/time` | request additional time | +| `GET /api/campaigns/:key/artifacts/:name` | one allowlisted artifact | +| `GET /api/events` | the change stream | +| `GET /api/plans` | the discovered plans | +| `POST /api/campaigns` | start a run | +| `POST /api/campaigns/:key/resume` | run eligible scheduled dependency work | +| `POST /api/campaigns/:key/stop` | stop the exact controller shown by the page | + +The [job API](../docs/execution-jobs.md#api-and-service-integration) adds durable +submission, listing, status, and cancellation at `/api/jobs`. Submission queues +work; an enabled worker must claim it before execution starts. + +Each payload covers one question, so opening a campaign or a tab is what pays +for reading it. The overview and the sheet are cached against the size and +modification time of the evidence they read, including while a campaign runs. + +## The event stream + +`GET /api/events` is a server-sent event stream. A `campaign` event names a +campaign whose plan, state, run output, or progression state changed; a `log` +event names an attempt whose stdout grew. Changes are debounced for 500 ms and +the stream sends a comment every 25 seconds so an idle connection stays open. +Campaign events refresh the affected evidence. Log events fetch only live fields +and the open log or transcript. The client also refreshes live fields every five +seconds while runs are active; Claude Code and Codex usage can advance without a +controller log write. Live-cost reads share a server cache and concurrent reads. +Docker transcript reads time out after five seconds; a failed read keeps saved +receipts visible. Logs do not invalidate the evidence sheet or graph replay. +While the stream is down, a full refresh every 15 seconds recovers missed evidence +changes. Hidden tabs stop both the event stream and refresh work. + +The watcher uses a recursive `fs.watch` per campaign directory. Where the +platform or the mount does not support one it polls the same file fingerprints +every 5 seconds instead. The server logs which mode it opened with when the +first client subscribes. + +## What it touches + +It reads plans from `/plans`, campaigns from `/campaigns`, and +jobs from `/jobs`. Authorized controls use the shared APIs to write job, +cancellation, and time-grant records. The dashboard records direct controller +operations in `/dashboard/operations.jsonl` and retains their output +under `/dashboard/operations`. Live transcript reads inspect the exact +owned coding container; saved transcripts use the attempt's transcript files. +It does not edit grades or source. + +## Workload setup and AI access + +Appliance setup installs workload presets under `results/run-presets/`. Each preset +uses the existing campaign manifest format. It supplies the supported levels, +stacks, priced models, guidance conditions, and pinned runtime. Operators can add +approved models and conditions there. Setup does not replace existing presets; +update their runtime pins when deploying a new release. Invalid presets report their +errors. Model prices are recorded values; the dashboard does not guess prices. + +Both interfaces use `src/campaigns/run-setup.ts` and the existing execution jobs: + +```sh +node dist/commands/job-cli.js options --results /path/to/results +node dist/commands/job-cli.js prepare selections.json --results /path/to/results > review.json +node dist/commands/job-cli.js start review.json --results /path/to/results --host local +``` + +`options` returns each workload's choices and defaults. `prepare` takes `key`, +`workload`, `workloadSha256` (from options), `level`, `stacks`, `agents` (`index` and `effort`), `conditions`, +`repetitions`, `parallelism`, `repairs`, `timeoutMinutes`, `maxCostUsd`, +`pauseAfterDepth` (null for none), and `credentials` (empty for appliance defaults). +The response records the review identity, immutable plan, cost cap, account mode, +and grading qualification. `start` accepts that response. Any change requires a +new review. It starts the same worker as the dashboard and returns the job ID before +waiting for completion. No model or reasoning level is substituted. + +HTTP clients use `GET /api/run-setup`, `POST /api/runs/prepare`, and `POST /api/runs`. +Writes require the same origin and browser token as other controls. +Named credential profiles expose only their labels, provider, version, and account +mode. Secret paths and values remain on the server. diff --git a/tools/stack-bench/dashboard/check-guide-steps.ts b/tools/stack-bench/dashboard/check-guide-steps.ts new file mode 100644 index 00000000000..6d384fcfe94 --- /dev/null +++ b/tools/stack-bench/dashboard/check-guide-steps.ts @@ -0,0 +1,119 @@ +import type { CompiledStep } from '../src/composition/definition-compiler.js'; + +// Read the authored form so loops stay visible instead of expanding to thousands of rows. +export interface GuideStep extends CompiledStep { + repeat?: number; + forEach?: unknown[]; + steps?: GuideStep[]; + branches?: GuideStep[][]; + alongside?: GuideStep[]; + storage?: { cart?: boolean; warehouses?: boolean }; + swap?: { find: unknown; with: unknown }; + senders?: { actor: string; count: number; prefix: string }[]; +} +const q = (value: unknown) => JSON.stringify(value); +const words = (value: unknown) => String(value ?? '').replaceAll('-', ' '); +const actor = (s: GuideStep) => s.actor ? `In the ${s.actor} browser, ` : ''; +const target = (s: GuideStep) => `${words(s.testid)}${s.contains ? ` matching ${q(s.contains)}` : ''}${s.in ? ` inside ${words(s.in.testid)}${s.in.contains ? ` for ${q(s.in.contains)}` : ''}` : ''}`; +const seconds = (ms: unknown) => `${Number((Number(ms) / 1000).toFixed(3))} seconds`; +const wanted = (s: GuideStep) => s.relativeTo ? `the saved ${q(s.relativeTo)} value ${Number(s.plus ?? 0) < 0 ? 'minus' : 'plus'} ${Math.abs(s.plus ?? 0)}` + : s.equals !== undefined ? q(s.equals) : s.value !== undefined ? q(s.value) : s.containsText ? `text containing ${q(s.containsText)}` + : s.atLeast !== undefined ? `at least ${s.atLeast}` : s.atMost !== undefined ? `at most ${s.atMost}` : 'the required value'; +const meanings: Record string> = { + signUp: s => `${actor(s)}register ${q(s.name)}${s.password ? ' with the specified test password' : ''}.`, + signIn: s => `${actor(s)}sign in as ${q(s.name)}${s.exact ? ' using the exact seeded account name' : ''}.`, + ensureSignedIn: s => `${actor(s)}keep the current session if it is ready; otherwise sign in as ${q(s.name)}.`, + click: s => `${actor(s)}click ${target(s)}${s.ifAvailable ? ' if available' : ''}${s.unlessVisible ? `, unless ${words(s.unlessVisible)} is already visible` : ''}.`, + openItem: s => `${actor(s)}open the product ${q(s.item)}${s.unlessVisible ? ` unless ${words(s.unlessVisible)} is already visible` : ''}.`, + fill: s => `${actor(s)}set ${target(s)} to ${q(s.text)}${s.enter ? ' and press Enter' : ''}.`, + pressKey: s => `${actor(s)}press ${q(s.key)}.`, + reload: s => `${actor(s)}reload the page.`, + freshClient: s => `Open a separate browser for ${s.actor}${s.preserveStorage ? ', copying its cookies, local storage, IndexedDB and session storage to test retained access' : ' with clean storage'}; later steps refer to ${s.actor}-fresh.`, + openClient: s => `Reopen the ${s.actor} browser.`, closeClient: s => `Close the ${s.actor} browser.`, + setOffline: s => `${s.offline === false ? 'Restore' : 'Disconnect'} the ${s.actor} browser's network connection.`, + expect: s => `${actor(s)}${s.absent ? 'watch for and reject any visible' : 'check for visible'} ${target(s)}${s.count !== undefined ? `; require ${s.count} matching elements` : ''}${s.value !== undefined ? `; require value ${q(s.value)}` : ''}${s.nonEmpty ? '; require nonempty content' : ''}${s.notContains ? `; reject text containing ${q(s.notContains)}` : ''}${s.ignoreCase ? ' (ignore letter case)' : ''}.`, + expectNumber: s => `${actor(s)}check that ${target(s)} is ${s.comparison === 'atMost' ? 'no greater than ' : s.comparison === 'atLeast' ? 'at least ' : ''}${wanted(s)}.`, + expectElementCount: s => `${actor(s)}count ${target(s)}; require ${wanted(s)}.`, + expectSequence: s => `${actor(s)}read every ${target(s)} in displayed order; require exactly ${q(s.equals)}.`, + expectAgreement: s => `Compare ${target(s)} in browsers ${(s.actors ?? []).join(', ')}; require equal ${s.numeric ? 'numbers' : 'values'}.`, + expectActorsWith: s => `Require exactly ${s.equals} of browsers ${(s.actors ?? []).join(', ')} to show ${target(s)}${s.maxEach !== undefined ? `, with at most ${s.maxEach} per browser` : ''}.`, + expectUnavailable: s => `${actor(s)}check that ${target(s)} is absent, hidden or disabled.`, + waitUntilAbsent: s => `${actor(s)}wait until ${target(s)} is absent.`, + recordNumber: s => `${actor(s)}save the number from ${target(s)} as ${q(s.as)}.`, + recordTime: s => `Record the current time as ${q(s.as)}.`, + expectElapsed: s => `Require this observation to start within ${seconds(s.atMost)} of ${q(s.since)}; otherwise the measurement is inconclusive.`, + wait: s => s.since ? `Wait until ${seconds(s.ms)} after ${q(s.since)}.` : `Wait ${seconds(s.ms)}.`, + dbSetStock: s => `Set stored stock for ${q(s.item)} in ${q(s.warehouse)} to ${s.quantity}, independently of the app's handlers.`, + dbRecordStock: s => `Read authoritative stock for ${q(s.item)}${s.warehouse ? ` in ${q(s.warehouse)}` : ' across warehouses'}; save it as ${q(s.as)}.`, + dbExpectStock: s => `Independently read stored stock for ${q(s.item)}${s.warehouse ? ` in ${q(s.warehouse)}` : ''}; require ${wanted(s)}.`, + dbRecordCheckout: s => `Read stored account, product and order state for ${q(s.account)} and ${q(s.item)}; save it as ${q(s.as)}${s.storage ? ` (cart: ${!!s.storage.cart}; warehouse detail: ${!!s.storage.warehouses})` : ''}.`, + dbExpectCatalogItem: s => `Read the database after product creation. Relative to ${q(s.before)}, require exactly one new product named ${q(s.name)} with price ${s.priceMinor} minor units. This is also the committed-write barrier before the next creation.`, + dbExpectCheckout: s => `Compare stored state with ${q(s.before)} and the prepared cart ${q(s.prepared)}. Require complete checkout effects for quantity ${q(s.quantity)}${s.alongsideAdd ? ` while preserving a concurrent add of ${q(s.alongsideAdd)}` : ''}${s.actor ? `, reconciled with ${s.actor}'s request outcome` : ''}.`, + dbExpectCancellation: s => `Compare stored state with ${q(s.before)}. Require cancellation to restore the original stock and booked amount exactly once.`, + dbExpectNoPurchase: s => `Compare stored state with ${q(s.before)}. Require no purchase effects.`, + dbExpectPurchase: s => `Compare stored orders with ${q(s.before)}${s.stockBefore ? ` and stock with ${q(s.stockBefore)}` : ''}; require the purchase and its accounting effects.`, + dbExpectPurchases: s => `Reconcile stored orders and warehouse effects against the saved buyer snapshots ${q(s.before)}; require ${s.purchases} complete purchases.`, + dbExpectPurchaseCount: s => `Read all stored orders against snapshots ${q(s.before)}. Require ${s.purchasesEach} additional purchases for each account, their stored prices, and warehouse effects where those snapshots include warehouses.`, + dbExpectOperation: s => `Check the complete stored-state transition for ${s.actor}'s ${s.operation}, against ${q(s.before)}${s.otherBefore ? ` and ${q(s.otherBefore)}` : ''}.`, + callAction: s => `${actor(s)}send the real application ${q(s.action)} request${s.authentication === 'none' ? ' without credentials' : s.authentication === 'tampered-session' ? ' with a tampered session credential' : s.authentication === 'optional' ? ' with whatever credentials this browser actually has; allow it to be signed out' : " with this actor's credentials"}${s.input ? `, taking ${s.input.attribute} from ${words(s.input.testid)}${s.input.contains ? ` matching ${q(s.input.contains)}` : ''} in ${s.from ?? s.actor}'s browser` : ''}.`, + expectActionOutcome: s => `Require ${s.actor}'s application request to be ${s.outcome}.`, + callConcurrently: s => `Send ${s.requests ?? (s.actors ?? []).length} ${q(s.action)} requests across browsers ${(s.actors ?? []).join(', ')} concurrently; retain every response or unknown outcome${s.delayMs ? ` (delay this request group by ${seconds(s.delayMs)})` : ''}.`, + clickConcurrently: s => `In browsers ${(s.actors ?? []).join(', ')}, click ${target(s)} concurrently.`, + expectCallOutcomes: s => `Check all recorded concurrent request outcomes${s.accepted !== undefined ? `; require exactly ${s.accepted} accepted calls` : ''}; unknown outcomes remain inconclusive rather than assumed refusals.`, + replayAs: s => `Replay the observed ${s.namedAction?.id ?? s.action ?? s.match ?? 'write'} from ${s.from ?? 'the source actor'} with ${s.actor}'s authority${s.namedTarget ? ', selecting the declared target entity' : ''}.`, + replayConcurrently: s => `Replay recorded writes concurrently from ${(s.actors ?? []).join(', ')}.`, + expectReplayRejected: s => `Require the replay by ${s.actor} to be rejected. Later state assertions, where present, check its effects.`, + expectReplayCompleted: s => `Require ${s.actor}'s replay to complete${s.requireAccepted ? ' and be accepted' : ''}.`, + forgeWrite: s => `Send a write as ${s.actor} while claiming ${s.fromActor}'s identity.`, + expectForgeryRejected: s => `Require the forged write by ${s.actor} to be rejected.`, + prepareResponseLoss: s => `Prepare to intercept ${s.actor}'s checkout reply on its actual transport.`, + loseCheckoutResponse: s => `Submit ${s.actor}'s checkout, lose its reply, and independently inspect stored effects against ${q(s.before)} and ${q(s.prepared)}.`, + confirmCheckout: s => `Confirm a working authenticated checkout path for ${s.actor}.`, + crashCheckout: s => `Start ${s.requests} checkout requests from ${s.actor}; crash the ${s.target} process at offset ${s.offsetMs} ms. Recover it, inspect stored state against ${q(s.before)} and ${q(s.prepared)}, and save ${q(s.as)}.`, + expectCrashCheckout: s => `Read crash evidence ${q(s.from)} and require ${s.verdict === 'atomicity' ? 'either the prepared cart or one complete order, never partial effects' : 'acknowledged work and earlier orders to survive'}.`, + restartBackend: () => 'Restart the owned database backend with retained storage and wait for readiness.', + stopAppServer: () => 'Stop the application server.', startAppServer: () => 'Start the application server and wait for readiness.', + expectReceived: s => `Require ${s.actor}'s observed incoming data to contain ${q(s.contains)}.`, + expectNotReceived: s => `Watch ${s.actor}'s observed incoming data; reject ${q(s.contains)}. This is a bounded observation, not a proof against every possible leak.`, + armScriptCanary: s => `Install a harmless execution marker in ${s.actor}'s browser.`, + expectNoScriptExecution: s => `Require ${s.actor}'s execution marker to remain unchanged after the supplied content is displayed.`, + createRoom: s => `${actor(s)}create chat room ${q(s.room)}.`, enterRoom: s => `${actor(s)}enter chat room ${q(s.room)}.`, + send: s => `${actor(s)}send message ${q(s.text)}.`, typeInto: s => `${actor(s)}type ${q(s.text)} without sending.`, + clearInput: s => `${actor(s)}clear the message input.`, + sendMany: s => `${actor(s)}send ${s.count} numbered messages beginning ${q(s.prefix)}, ${seconds(s.delayMs ?? 0)} apart.`, + sendConcurrently: s => `Send messages concurrently: ${(s.senders ?? []).map(x=>`${x.actor}: ${x.count} beginning ${q(x.prefix)}`).join('; ')}; spacing ${seconds(s.delayMs ?? 0)}.`, + expectAllPresent: s => `${actor(s)}require all ${s.count} messages beginning ${q(s.prefix)}.`, + expectOrderMatches: s => `Require browsers ${(s.actors ?? []).join(', ')} to show the same nonempty message order matching ${q(s.prefix)}.`, + expectStable: s => `${actor(s)}sample ${target(s)} ${s.samples ?? 4} times, waiting ${seconds(s.intervalMs ?? 700)} after each sample; require a stable value.`, + runScript: s => `Run the application's ${q(s.script)} with arguments ${q(s.args ?? [])}.`, +}; +export function stepsText(steps: GuideStep[], depth = 0): string[] { + const lines = []; + for (const s of steps) { + const pad = ' '.repeat(depth); + if (s.repeat !== undefined || s.forEach) { + const list = s.forEach; + lines.push(`${pad}- Repeat ${list ? list.length : s.repeat} times${list ? `, substituting each listed value (first ${q(list[0])}, last ${q(list.at(-1))}; exact list in source)` : ''}:`); + lines.push(...stepsText(s.steps ?? [], depth + 1)); continue; + } + if (s.do === 'race') { + lines.push(`${pad}- Run these branches concurrently and wait for all of them:`); + (s.branches ?? []).forEach((branch, i) => { lines.push(`${pad} - Branch ${i + 1}:`); lines.push(...stepsText(branch, depth + 2)); }); + } else { + const describe = meanings[s.do]; + lines.push(`${pad}- ${describe ? describe(s) : `Run ${q(s.do)}. See the exact input below for its fields.`}`); + } + if (s.input?.overrides) lines.push(`${pad} Replace submitted fields with ${q(s.input.overrides)}; this is deliberately untrusted client input.`); + if (s.browserOrigin) lines.push(`${pad} Send from a ${s.browserOrigin === 'cross-site' ? 'different browser origin' : 'same-site browser origin'}.`); + if (s.swap) lines.push(`${pad} In the replayed request, replace ${q(s.swap.find)} with ${q(s.swap.with)}.`); + if (s.containsText) lines.push(`${pad} Require displayed text containing ${q(s.containsText)}.`); + if (s.attribute) lines.push(`${pad} Inspect the ${q(s.attribute)} attribute.`); + if (s.provenBy) lines.push(`${pad} This check uses proof recorded by ${q(s.provenBy)}.`); + if (s.reuseCombinedFrom) lines.push(`${pad} If application and database are the same process, reuse ${q(s.reuseCombinedFrom)} rather than counting another crash.`); + for (const group of s.alongside ?? []) lines.push(`${pad} At the same time: ${meanings.callConcurrently!(group)}`); + if (s.within !== undefined) lines.push(`${pad} Observation deadline: ${seconds(s.within)}${s.absent || s.do === 'expectNotReceived' ? '; this includes an absence observation window' : '; normally completes sooner if the condition is met'}.`); + if (s.settleMs) lines.push(`${pad} Always wait another ${seconds(s.settleMs)} after this action.`); + + } + return lines; +} diff --git a/tools/stack-bench/dashboard/check-guide.ts b/tools/stack-bench/dashboard/check-guide.ts new file mode 100644 index 00000000000..2df8279fede --- /dev/null +++ b/tools/stack-bench/dashboard/check-guide.ts @@ -0,0 +1,135 @@ +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { createHash } from 'node:crypto'; +import { loadTrack, listTracks } from '../src/composition/tracks.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { resolveFeatureCatalog } from '../src/progression/feature-catalog-selection.js'; +import { progressionLevels, selectFeatureCatalogLevels } from '../src/progression/progression-definition.js'; +import { resolveProgressionRecipeLevelSelection } from '../src/progression/progression-recipe-selection.js'; +import type { CompiledCriterion, CompiledFeature } from '../src/composition/definition-compiler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stepsText, type GuideStep } from './check-guide-steps.js'; +import { esc } from './public/format.js'; +import { topbar } from './public/views/plans.js'; + +interface Check { + key: string; + active: boolean; + source: string; + sourceSha256: string; + feature: CompiledFeature; + criterion: CompiledCriterion; + controls: string[]; +} + +function inventory() { + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 3, 'ecommerce.progression-catalog'); + if (!binding) throw new Error('The ecommerce dependency recipe is unavailable.'); + const full = resolveFeatureCatalog('progression/ecommerce.json', track); + const catalog = selectFeatureCatalogLevels(full, progressionLevels(full).filter(level => level <= 3)); + const selected = new Set(resolveProgressionRecipeLevelSelection(binding, catalog, 3, + { cumulative: true }).grader.checkKeys); + const mutationRoot = join(STACK_BENCH_ROOT, 'grader', 'mutations'); + const mutations = readdirSync(mutationRoot).filter(file => file.endsWith('-ecommerce.json')).flatMap(file => { + const manifest = JSON.parse(readFileSync(join(mutationRoot, file), 'utf8')) as { + mutations: { id: string; desc: string; targets?: string[] }[]; + }; + return manifest.mutations.map(mutation => ({ ...mutation, backend: file.replace('-ecommerce.json', '') })); + }); + const sources = new Map(); + const read = (source: string) => { + let value = sources.get(source); + if (!value) { + const text = readFileSync(join(STACK_BENCH_ROOT, source), 'utf8'); + value = { ...JSON.parse(text), sha256: createHash('sha256').update(text).digest('hex') }; + sources.set(source, value!); + } + return value!; + }; + const checks: Check[] = binding.release.checkCatalog.map(check => { + const source = `tracks/ecommerce/${check.source}`; + const scenario = read(source); + const feature = scenario.features.find(feature => feature.id === check.featureId); + const criterion = feature?.criteria.find(criterion => criterion.id === check.criterionId); + if (!feature || !criterion) throw new Error(`Missing source for ${check.stableKey}`); + return { key: check.stableKey, active: selected.has(check.stableKey), source, + sourceSha256: scenario.sha256, feature, criterion, + controls: mutations.filter(mutation => mutation.targets?.includes(check.stableKey)) + .map(mutation => `${mutation.backend}: ${mutation.desc} (${mutation.id})`) }; + }); + const included = new Set(checks.map(c => `${c.source}:${c.feature.id}:${c.criterion.id}`)); + for (const name of listTracks()) { + const other = loadTrack(name); + for (const file of readdirSync(other.scenarios).filter(file => file.endsWith('.json')).sort()) { + const source = `tracks/${name}/scenarios/${file}`; + const scenario = read(source); + for (const feature of scenario.features) for (const criterion of feature.criteria) { + const key = `${source}:${feature.id}:${criterion.id}`; + if (!included.has(key)) checks.push({ key, active: false, source, + sourceSha256: scenario.sha256, feature, criterion, controls: [] }); + } + } + } + return { checks, selected: selected.size, recipe: binding.release.id, + sha256: binding.release.contentSha256 }; +} + +function procedure(steps: GuideStep[]): string { + return stepsText(steps).map(line => { + const text = line.trimStart(); + const kind = /Repeat \d|branches concurrently|Branch \d/.test(text) ? 'repeat' + : /^(?:- )?(?:Wait |Always wait|Observation deadline)/.test(text) ? 'wait' + : /require|reject|check that|check for|check all|compare stored|compare the|inspect stored/i.test(text) ? 'assert' + : 'action'; + const indent = Math.min(4, (line.length - text.length) / 2); + return `
${ + { repeat: 'Repeat / parallel', wait: 'Wait / deadline', assert: 'Verify', action: 'Action' }[kind] + }${esc(text.replace(/^- /, ''))}
`; + }).join(''); +} + +function entry(check: Check): string { + const { feature, criterion } = check; + const search = `${check.key} ${criterion.desc} ${feature.name} ${check.source}`.toLowerCase(); + const setup = procedure((feature.setup ?? []) as GuideStep[]); + return `
` + + `${esc(criterion.id)}${esc(criterion.desc)}` + + `${check.active ? 'L1–L3' : 'Other'}
` + + (criterion.statedBy ? `

Required behavior

${esc(criterion.statedBy)}

` : '') + + (criterion.provenBy ? `

Uses earlier proof: ${esc(criterion.provenBy)}

` : '') + + (setup ? '

Setup

' + setup : '') + + '

Steps

' + procedure(criterion.steps as GuideStep[]) + + '
Technical details' + + `

${esc(check.key)}

` + + `

${criterion.points} ${criterion.points === 1 ? 'point' : 'points'} · Browsers: ${esc((feature.actors ?? []).join(', ') || 'none')}

` + + (criterion.note ? `

${esc(criterion.note)}

` : '') + + '

Declared defect controls (targets, not proof of a passing qualification run):

' + + (check.controls.length ? '
    ' + check.controls.map(control => `
  • ${esc(control)}
    • `).join('') + '
' + : '

None linked to this exact check.

') + + `

${esc(check.source)} · SHA-256 ${check.sourceSha256}

` + + `
${esc(JSON.stringify({ actors: feature.actors, setup: feature.setup ?? [], criterion }, null, 2))}
` + + '
'; +} + +/** Current local definitions only. Historical campaign evidence retains its frozen version. */ +export function checkGuidePage(): string { + const data = inventory(); + return '' + + '' + + 'Checks · Stack Bench' + + '' + + topbar({ page: 'check-guide', key: '', canStart: false, resumable: false, error: '' }) + + '

Checks

' + + `

What each check does, step by step. ${data.selected} checks in the current ecommerce dependency L1–L3 selection.

` + + '
How checks are graded' + + '

A check passes only when every step runs and every Verify step holds. A failed prerequisite blocks the checks that depend on it. Unknown results and harness errors are not passes. Setup can be shared between checks; the runner keeps prerequisite order.

' + + '

This page shows the current local definitions, not a historical run\'s frozen version, and does not certify qualification. Waits and deadlines are authored values; omitted deadlines use grader defaults, so the steps are not an elapsed-time estimate.

' + + `

Recipe: ${esc(data.recipe)} · SHA-256 ${data.sha256}

` + + '
' + + '
' + + '' + + '
' + + '' + + data.checks.map(entry).join('') + '
'; +} diff --git a/tools/stack-bench/dashboard/dashboard-events.ts b/tools/stack-bench/dashboard/dashboard-events.ts new file mode 100644 index 00000000000..09bb3279b16 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-events.ts @@ -0,0 +1,164 @@ +import { existsSync, readdirSync, statSync, watch } from 'node:fs'; +import type { FSWatcher } from 'node:fs'; +import { join } from 'node:path'; + +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; +import { CAMPAIGN_FILE } from '../src/campaigns/campaign-path.js'; + +const DEBOUNCE_MS = 500; +const POLL_MS = 5000; +const LOG_FILE = 'process.stdout.log'; +const CAMPAIGN_FILES = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state, 'depth-release.json'] as const; +const EXECUTION_FILES = [ARTIFACT_FILE.run, ARTIFACT_FILE.progressionState, 'depth-pause.json'] as const; + +export interface CampaignChange { + type: 'campaign' | 'log'; + key: string; + attemptId?: string; +} + +export type WatchMode = 'watch' | 'poll'; + +export interface CampaignWatcher { + close(): void; +} + +interface CampaignFingerprint { + campaign: string; + logs: Map; +} + +function stamp(path: string): string | null { + if (!existsSync(path)) return null; + const stat = statSync(path); + return `${stat.size}:${stat.mtimeMs}`; +} + +function directories(root: string): string[] { + if (!existsSync(root)) return []; + return readdirSync(root, { withFileTypes: true }) + .filter(entry => entry.isDirectory()).map(entry => entry.name); +} + +// The evidence a view reads, in one pass: the campaign files and each +// execution's result, and the log sizes that tell a follower there are new +// bytes to fetch. +function fingerprintCampaign(directory: string): CampaignFingerprint { + const parts = CAMPAIGN_FILES.map(file => `${file}:${stamp(join(directory, file)) ?? 'missing'}`); + const logs = new Map(); + const attemptsRoot = join(directory, 'attempts'); + for (const attempt of directories(attemptsRoot)) { + const attemptDirectory = join(attemptsRoot, attempt); + let bytes = 0; + for (const execution of directories(attemptDirectory)) { + const executionDirectory = join(attemptDirectory, execution); + for (const file of EXECUTION_FILES) { + const value = stamp(join(executionDirectory, file)); + if (value) parts.push(`${attempt}/${execution}/${file}:${value}`); + } + const log = join(executionDirectory, LOG_FILE); + if (existsSync(log)) bytes += statSync(log).size; + } + logs.set(attempt, bytes); + } + return { campaign: parts.sort().join('|'), logs }; +} + +// One watcher for the whole server: a recursive watch per campaign directory +// where the platform supports it (Windows, macOS, and Linux on Node 20 and +// later), and a poll of the same fingerprints where it does not. +export function watchCampaigns(campaignsRoot: string, + emit: (change: CampaignChange) => void, + onMode: (mode: WatchMode) => void = () => {}): CampaignWatcher { + const fingerprints = new Map(); + const watchers = new Map(); + const timers = new Map(); + let rootWatcher: FSWatcher | null = null; + let poll: NodeJS.Timeout | null = null; + let closed = false; + + const check = (key: string): void => { + timers.delete(key); + const directory = join(campaignsRoot, key); + if (!existsSync(directory)) { + fingerprints.delete(key); + watchers.get(key)?.close(); + watchers.delete(key); + return; + } + const next = fingerprintCampaign(directory); + const previous = fingerprints.get(key); + fingerprints.set(key, next); + if (!previous) return; + if (previous.campaign !== next.campaign) emit({ type: 'campaign', key }); + for (const [attemptId, bytes] of next.logs) { + if ((previous.logs.get(attemptId) ?? 0) !== bytes) emit({ type: 'log', key, attemptId }); + } + }; + + const schedule = (key: string): void => { + if (closed || timers.has(key)) return; + timers.set(key, setTimeout(() => check(key), DEBOUNCE_MS).unref()); + }; + + const startPoll = (): void => { + if (closed || poll) return; + onMode('poll'); + poll = setInterval(() => { + attach(); + for (const key of directories(campaignsRoot)) check(key); + }, POLL_MS).unref(); + }; + + const attach = (): void => { + if (closed) return; + if (!rootWatcher && existsSync(campaignsRoot)) { + try { + rootWatcher = watch(campaignsRoot, { persistent: false }, (_event, name) => { + const key = String(name ?? '').split(/[/\\]/)[0]; + if (key) schedule(key); + attach(); + }); + rootWatcher.once('error', () => { rootWatcher = null; startPoll(); }); + } catch { startPoll(); } + } + for (const key of directories(campaignsRoot)) { + // A campaign that appeared since the last pass has everything to report. + if (!fingerprints.has(key)) { + fingerprints.set(key, { campaign: '', logs: new Map() }); + schedule(key); + } + if (watchers.has(key) || poll) continue; + try { + const watcher = watch(join(campaignsRoot, key), { persistent: false, recursive: true }, + () => schedule(key)); + watcher.once('error', () => { watchers.delete(key); startPoll(); }); + watchers.set(key, watcher); + } catch { + // No recursive watch on this platform: the poll reads the same files. + startPoll(); + return; + } + } + }; + + for (const key of directories(campaignsRoot)) { + fingerprints.set(key, fingerprintCampaign(join(campaignsRoot, key))); + } + attach(); + if (!rootWatcher) startPoll(); + else if (!poll) onMode('watch'); + return { + close() { + closed = true; + for (const timer of timers.values()) clearTimeout(timer); + timers.clear(); + if (poll) clearInterval(poll); + poll = null; + rootWatcher?.close(); + rootWatcher = null; + for (const watcher of watchers.values()) watcher.close(); + watchers.clear(); + }, + }; +} diff --git a/tools/stack-bench/dashboard/dashboard-live-cost.ts b/tools/stack-bench/dashboard/dashboard-live-cost.ts new file mode 100644 index 00000000000..d38466bb0fe --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-live-cost.ts @@ -0,0 +1,119 @@ +import { attemptTranscriptFiles } from './dashboard-transcript.js'; +import { normalizeClaudeUsage, priceClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import type { PricingRates } from '../src/evidence/pricing-authority.js'; + +interface UsagePoint { id: string; completedAt: string; costUsd: number; signature: string } +export function liveCostTotal(status: string, observed: number | undefined, saved: number | null): number | undefined { + return status === 'running' && observed !== undefined && observed >= (saved ?? 0) ? observed : undefined; +} +const object = (value: unknown): value is Record => + !!value && typeof value === 'object' && !Array.isArray(value); + +// Display-only usage. Never write these observations into benchmark evidence. +export function responseCosts(text: string, rates: PricingRates, model: string, startedAt: string): UsagePoint[] { + const points: UsagePoint[] = []; + for (const line of text.split('\n')) { + if (!line.trim()) continue; + let event: unknown; + try { event = JSON.parse(line); } catch { throw new Error('Incomplete usage transcript'); } + if (!object(event) || event.type !== 'assistant' || !object(event.message) || !event.message.usage) continue; + const message = event.message; + if (typeof message.stop_reason !== 'string' || !message.stop_reason) continue; + const timestamp = typeof event.timestamp === 'string' ? Date.parse(event.timestamp) : NaN; + if (!Number.isFinite(timestamp)) throw new Error('Usage timestamp unavailable'); + if (timestamp < Date.parse(startedAt)) continue; + if (typeof message.model !== 'string' || !(message.model === model || message.model.startsWith(`${model}-`))) { + throw new Error('No pinned price for transcript model'); + } + const id = event.requestId ?? message.id ?? event.uuid; + if (typeof id !== 'string' || !id) throw new Error('Usage request identity unavailable'); + points.push({ id, completedAt: new Date(timestamp).toISOString(), + costUsd: priceClaudeUsage(message.usage, rates), + signature: JSON.stringify([message.model, normalizeClaudeUsage(message.usage)]) }); + } + return points; +} + +export function cumulativeResponseCosts(points: readonly UsagePoint[]): Array<{ completedAt: string; costUsd: number }> { + const unique = new Map(); + for (const point of points) { + const prior = unique.get(point.id); + if (prior && prior.signature !== point.signature) throw new Error('Conflicting usage for request'); + if (!prior) unique.set(point.id, point); + } + let total = 0; + return [...unique.values()].sort((a, b) => a.completedAt.localeCompare(b.completedAt)).map(point => ({ + completedAt: point.completedAt, costUsd: Number((total += point.costUsd).toFixed(6)), + })); +} + +interface CodexUsageState { session?: string; model?: string; totals?: [number, number, number] } + +export function codexResponseCosts(text: string, rates: PricingRates, model: string, startedAt: string): UsagePoint[] { + const state: CodexUsageState = {}; + const points: UsagePoint[] = []; + for (const line of text.split('\n')) { + if (!line.trim()) continue; + const event: unknown = JSON.parse(line); + if (!object(event) || !object(event.payload)) continue; + const payload = event.payload; + if (event.type === 'session_meta') { + if (typeof payload.id !== 'string' || !payload.id) throw new Error('Usage session identity unavailable'); + if (state.session && state.session !== payload.id) throw new Error('Usage session changed'); + state.session = payload.id; + } + if (event.type === 'turn_context') state.model = typeof payload.model === 'string' ? payload.model : undefined; + if (event.type !== 'event_msg' || payload.type !== 'token_count' || payload.info === null) continue; + const usage = object(payload.info) && object(payload.info.total_token_usage) + ? payload.info.total_token_usage : {}; + const totals = [usage.input_tokens, usage.cached_input_tokens, usage.output_tokens]; + if (!totals.every(value => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) + || Number(totals[1]) > Number(totals[0])) throw new Error('Invalid Codex token usage'); + const counts = totals as [number, number, number]; + const previous = state.totals ?? [0, 0, 0]; + const delta = [counts[0] - previous[0], counts[1] - previous[1], counts[2] - previous[2]] as const; + if (delta.some(value => value < 0) || delta[1] > delta[0]) throw new Error('Codex usage totals decreased'); + state.totals = counts; + const timestamp = typeof event.timestamp === 'string' ? Date.parse(event.timestamp) : NaN; + if (!Number.isFinite(timestamp)) throw new Error('Usage timestamp unavailable'); + if (timestamp < Date.parse(startedAt) || delta.every(value => value === 0)) continue; + if (!state.session) throw new Error('Usage session identity unavailable'); + if (state.model !== model) throw new Error('No pinned price for transcript model'); + points.push({ id: `${state.session}:${counts.join(':')}`, completedAt: new Date(timestamp).toISOString(), + costUsd: ((delta[0] - delta[1]) * rates.input + delta[1] * rates.cacheRead + delta[2] * rates.output) / 1e6, + signature: JSON.stringify([state.model, delta]) }); + } + return points; +} + +const cache = new Map(); + +export async function liveTranscriptCost(directory: string, adapter: string, rates: PricingRates, + model: string, startedAt: string) { + const files = await attemptTranscriptFiles([{ directory, label: 'Execution' }], adapter); + const activityUpdatedAt = files.length ? new Date(Math.max(...files.map(file => file.modified))).toISOString() : null; + const points: UsagePoint[] = []; + for (const file of files) { + const key = `${directory}/${file.id}/${startedAt}/${JSON.stringify(rates)}/${model}`; + const prior = cache.get(key); + if (prior?.size === file.size && prior.modified === file.modified) { points.push(...prior.points); continue; } + // Read Codex session metadata and cumulative counters together. + const offsetStart = adapter !== 'codex' && prior && file.size > prior.size ? prior.offset : 0; + // Bound catch-up work; do not label a partial file as a complete live total. + if (file.size - offsetStart > 16 * 1024 * 1024) return { activityUpdatedAt, costs: [] }; + const chunks: Buffer[] = []; + for (let offset = offsetStart; offset < file.size; offset += 256 * 1024) { + chunks.push(await file.read(offset, Math.min(256 * 1024, file.size - offset))); + } + const bytes = Buffer.concat(chunks); + const end = bytes.lastIndexOf(10); + const text = end < 0 ? '' : bytes.subarray(0, end + 1).toString(); + const parsed = [...(offsetStart ? prior!.points : []), ...(adapter === 'codex' + ? codexResponseCosts(text, rates, model, startedAt) + : responseCosts(text, rates, model, startedAt))]; + if (cache.size >= 128) cache.delete(cache.keys().next().value!); + cache.set(key, { size: file.size, modified: file.modified, offset: offsetStart + end + 1, points: parsed }); + points.push(...parsed); + } + return { activityUpdatedAt, costs: cumulativeResponseCosts(points) }; +} diff --git a/tools/stack-bench/dashboard/dashboard-model.ts b/tools/stack-bench/dashboard/dashboard-model.ts new file mode 100644 index 00000000000..908a9429aa3 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-model.ts @@ -0,0 +1,521 @@ +import { closeSync, existsSync, fstatSync, openSync, readFileSync, readSync, readdirSync, + lstatSync, realpathSync, statSync, +} from 'node:fs'; +import { basename, dirname, extname, join, relative, resolve, sep } from 'node:path'; + +import type { CompiledCampaignPlan } + from '../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptState } from '../src/campaigns/campaign-scheduler.js'; +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { campaignLockIsActive, readCampaignLock } from '../src/campaigns/campaign-lock.js'; +import { readDepthPause } from '../src/campaigns/campaign-depth-pause.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { campaignFacts, inspectCampaignAttempt } from '../src/campaigns/campaign-inspection.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { CAMPAIGN_FILE } from '../src/campaigns/campaign-path.js'; +import { repairBudgetLimit } from '../src/progression/repair-plan.js'; + +export const MAX_LOG_BYTES = 96 * 1024; +const MAX_PUBLIC_TEXT_BYTES = 8 * 1024 * 1024; +const MAX_ARTIFACTS_PER_EXECUTION = 512; +const IMAGE_TYPES = new Map([ + ['.png', 'image/png'], ['.jpg', 'image/jpeg'], ['.jpeg', 'image/jpeg'], ['.webp', 'image/webp'], +]); +const CAMPAIGN_ARTIFACT = /^(?:plan\.json|state\.json|report\/(?:report\.(?:html|json)|export-manifest\.json))$/; +const EXECUTION_ARTIFACT = /^(?:run\.json|preflight\.json|recovery\.json|progression-state\.json|process\.json|process\.(?:stdout|stderr)\.log|backend\.log|level-l\d+-checkpoint\.json|progression\/attempt-\d+\/(?:bundle\.json|contract-lint\.json|actions\.json|grading-[^/]+\.json|failure-media\/[^/]+\.(?:png|jpe?g|webp))|(?:first-build-l\d+-grading|l\d+-fix\d+-grading|grading)\/(?:bundle\.json|contract-lint\.json|actions\.json|grading-[^/]+\.json|failure-media\/[^/]+\.(?:png|jpe?g|webp)))$/i; + +type ControllerActive = (directory: string, campaign: CompiledCampaignPlan) => boolean; + +export interface DashboardArtifact { + id: string; + path: string; + name: string; + kind: 'visual' | 'report' | 'log' | 'data'; + contentType: string; + size: number; +} + +export interface ResolvedDashboardArtifact extends DashboardArtifact { + absolute: string; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function contained(root: string, path: string, label: string): string { + const absoluteRoot = resolve(root); + const absolute = resolve(absoluteRoot, path); + const rel = relative(absoluteRoot, absolute); + if (rel === '..' || rel.startsWith(`..${sep}`) || rel === '') { + throw new Error(`${label} is outside the configured dashboard root`); + } + return absolute; +} + +export function readTextTail(path: string, limit = MAX_LOG_BYTES): string { + if (!existsSync(path)) return ''; + const descriptor = openSync(path, 'r'); + try { + const size = fstatSync(descriptor).size; + const length = Math.min(size, limit); + const buffer = Buffer.alloc(length); + readSync(descriptor, buffer, 0, length, size - length); + return redactCredentials(buffer.toString('utf8')); + } finally { + closeSync(descriptor); + } +} + +function artifactId(relativePath: string): string { + return Buffer.from(relativePath, 'utf8').toString('base64url'); +} + +function artifactLabel(path: string): string { + const name = basename(path); + if (path === CAMPAIGN_FILE.plan) return 'Frozen plan'; + if (path === CAMPAIGN_FILE.state) return 'Campaign state'; + if (path === `report/${CAMPAIGN_FILE.reportHtml}`) return 'Campaign report'; + if (path === `report/${CAMPAIGN_FILE.reportJson}`) return 'Report data'; + if (name === ARTIFACT_FILE.run) return 'Run result'; + if (name === ARTIFACT_FILE.preflight) return 'Preflight result'; + if (name === ARTIFACT_FILE.recovery) return 'Recovery record'; + if (name === ARTIFACT_FILE.progressionState) return 'Dependency progress'; + if (name === 'process.stdout.log') return 'Run output'; + if (name === 'process.stderr.log') return 'Run errors'; + if (name === 'backend.log') return 'Backend output'; + if (name === ARTIFACT_FILE.gradeBundle) return `${basename(dirname(path))} bundle`; + if (name === ARTIFACT_FILE.actions) return `${basename(dirname(path))} actions`; + if (name === ARTIFACT_FILE.contractLint) return `${basename(dirname(path))} contract check`; + return name.replace(/[-_]/g, ' '); +} + +function artifactMetadata(campaignDirectory: string, path: string): DashboardArtifact { + const absolute = contained(campaignDirectory, path, 'campaign artifact'); + const size = statSync(absolute).size; + const extension = extname(path).toLowerCase(); + const kind = IMAGE_TYPES.has(extension) ? 'visual' + : path.endsWith('/report.html') ? 'report' + : path.endsWith('.log') ? 'log' : 'data'; + return { id: artifactId(path), path: path.replaceAll('\\', '/'), name: artifactLabel(path), + kind, contentType: IMAGE_TYPES.get(extension) ?? (kind === 'report' ? 'text/html' : 'text/plain'), + size }; +} + +function rejectSymlinkPath(root: string, path: string): void { + const rel = relative(resolve(root), resolve(path)); + let current = resolve(root); + for (const segment of rel.split(sep)) { + current = join(current, segment); + if (lstatSync(current).isSymbolicLink()) { + throw new Error('campaign artifact path contains a symbolic link'); + } + } +} + +export function walkPublicExecutionArtifacts(campaignDirectory: string, executionDirectory: string): { + artifacts: DashboardArtifact[]; + truncated: boolean; +} { + const found: DashboardArtifact[] = []; + let truncated = false; + const visit = (directory: string): void => { + const directoryRelative = relative(executionDirectory, directory).replaceAll('\\', '/'); + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (found.length >= MAX_ARTIFACTS_PER_EXECUTION) { + truncated = true; + return; + } + if (entry.isSymbolicLink()) continue; + const absolute = join(directory, entry.name); + if (entry.isDirectory()) { + const allowed = directoryRelative === '' + ? /^(?:first-build-l\d+-grading|l\d+-fix\d+-grading|grading|progression)$/i.test(entry.name) + : directoryRelative === 'progression' + ? /^attempt-\d+$/i.test(entry.name) + : (/^(?:progression\/attempt-\d+|(?:.*\/)?(?:first-build-l\d+-grading|l\d+-fix\d+-grading|grading))$/i + .test(directoryRelative) && entry.name === 'failure-media'); + if (allowed) visit(absolute); + if (truncated) return; + } + else if (entry.isFile()) { + const executionRelative = relative(executionDirectory, absolute).replaceAll('\\', '/'); + if (EXECUTION_ARTIFACT.test(executionRelative)) { + const campaignRelative = relative(campaignDirectory, absolute).replaceAll('\\', '/'); + found.push(artifactMetadata(campaignDirectory, campaignRelative)); + } + } + } + }; + if (existsSync(executionDirectory)) visit(executionDirectory); + return { artifacts: found.sort((left, right) => left.path.localeCompare(right.path)), truncated }; +} + +function campaignPackage(campaignDirectory: string, attempts: CampaignAttemptState[]) { + const campaign = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state, + `report/${CAMPAIGN_FILE.reportHtml}`, `report/${CAMPAIGN_FILE.reportJson}`] + .filter(path => existsSync(join(campaignDirectory, path))) + .map(path => artifactMetadata(campaignDirectory, path)); + const executions: Array<{ + attemptId: string; + stack: string; + executionId: string; + ordinal: number; + status: string; + artifacts: DashboardArtifact[]; + visuals: DashboardArtifact[]; + truncated: boolean; + }> = []; + for (const attempt of attempts) { + for (const execution of attempt.executions) { + const directory = contained(campaignDirectory, execution.output, 'campaign execution'); + const scanned = walkPublicExecutionArtifacts(campaignDirectory, directory); + const artifacts = scanned.artifacts; + executions.push({ attemptId: attempt.plan.id, stack: attempt.plan.stack, + executionId: execution.id, ordinal: execution.ordinal, status: execution.status, + artifacts, visuals: artifacts.filter(artifact => artifact.kind === 'visual'), + truncated: scanned.truncated }); + } + } + return { campaign, executions }; +} + +export function resolveCampaignArtifact(resultsRoot: string, key: string, + id: string): ResolvedDashboardArtifact { + if (!/^[a-z0-9][a-z0-9.-]*$/.test(key)) throw new Error('campaign key is invalid'); + let path; + try { path = Buffer.from(id, 'base64url').toString('utf8'); } + catch { throw new Error('campaign artifact id is invalid'); } + if (!path || artifactId(path) !== id || path.includes('\\') || path.startsWith('/')) { + throw new Error('campaign artifact id is invalid'); + } + const executionMatch = path.match(/^attempts\/([^/]+)\/(execution-\d+)\/(.+)$/); + const allowed = CAMPAIGN_ARTIFACT.test(path) + || (executionMatch !== null && EXECUTION_ARTIFACT.test(executionMatch[3] ?? '')); + if (!allowed) { + throw new Error('campaign artifact is not available in the dashboard'); + } + const campaignsRoot = join(resolve(resultsRoot), 'campaigns'); + const campaignDirectory = contained(campaignsRoot, key, 'campaign'); + const absolute = contained(campaignDirectory, path, 'campaign artifact'); + if (!existsSync(absolute) || !statSync(absolute).isFile()) throw new Error('campaign artifact does not exist'); + rejectSymlinkPath(campaignsRoot, absolute); + const realCampaign = realpathSync(campaignDirectory); + const realArtifact = realpathSync(absolute); + contained(realCampaign, relative(realCampaign, realArtifact), 'campaign artifact'); + return { ...artifactMetadata(campaignDirectory, path), absolute }; +} + +export function readCampaignArtifactBody(artifact: ResolvedDashboardArtifact): Buffer { + if (artifact.kind === 'visual') return readFileSync(artifact.absolute); + if (artifact.size > MAX_PUBLIC_TEXT_BYTES) throw new Error('campaign artifact is too large to view'); + return Buffer.from(redactCredentials(readFileSync(artifact.absolute, 'utf8'))); +} + +function matches(text: string, pattern: RegExp): Array { + return [...text.matchAll(pattern)].map(match => Object.assign(match, { index: match.index ?? 0 })); +} + +export function parseRunProgress(log: string, { repairs = 0, running = true, status = null, + dependency = false }: { + repairs?: number; + running?: boolean; + status?: string | null; + dependency?: boolean; +} = {}) { + const totals = matches(log, /^\s*TOTAL\b.*?(\d+)\/(\d+)\s*$/gm) + .map(match => ({ index: match.index, score: Number(match[1]), max: Number(match[2]) })); + // A run-wide repair prints "repair N/M"; a feature repair prints + // "feature repair N: title" because its limit belongs to the feature. + const roundMarkers = matches(log, + /^--- (?:feature )?repair (\d+)(?:\/(\d+))?(?:: (.+))? ---$/gm) + .map(match => ({ index: match.index, round: Number(match[1]), + budget: match[2] === undefined ? null : Number(match[2]), + target: match[3] ?? null })); + const grading = matches(log, /^===\s+[^\n]*?-l(\d+)(?:-(?:first|fix(\d+)))?\s+\([^\n]+\)\s*===$/gm) + .map(match => ({ index: match.index, level: Number(match[1]), + round: match[2] ? Number(match[2]) : 0 })); + const latestTotal = totals.at(-1) ?? null; + const latestRound = roundMarkers.at(-1) ?? null; + const latestGrading = grading.at(-1) ?? null; + const latestIndex = Math.max(latestTotal?.index ?? -1, latestRound?.index ?? -1, + latestGrading?.index ?? -1); + let phase = status === 'pending' ? 'Waiting to start' + : running ? 'Building the generated app' + : status === 'invalid' ? 'Stopped without a valid result' : 'Finished'; + const level = latestGrading?.level ?? null; + const round = latestRound?.round ?? latestGrading?.round ?? 0; + const budget = latestRound ? latestRound.budget : repairs; + const target = latestRound?.target ? ` for ${latestRound.target}` : ''; + const of = (limit: number | null): string => limit === null ? '' : ` of ${limit}`; + const stage = (value: number): string => dependency ? `depth ${value}` : `L${value}`; + if (running && latestIndex === latestGrading?.index) { + phase = latestGrading.round + ? `Grading ${stage(latestGrading.level)} after repair ${round}${of(budget)}${target}` + : `Grading the first ${stage(latestGrading.level)} build`; + } else if (running && latestIndex === latestRound?.index) { + phase = latestRound.target + ? `Repairing ${latestRound.target} · ${latestRound.round}${of(latestRound.budget)}` + : `Repairing ${stage(latestGrading?.level ?? 1)} · round ${latestRound.round}${of(latestRound.budget)}`; + } else if (latestIndex === latestTotal?.index && running) { + phase = 'Preparing the next step'; + } + return { + phase, + level, + repair: { round, budget }, + firstScore: totals[0] ? { score: totals[0].score, max: totals[0].max } : null, + latestScore: latestTotal ? { score: latestTotal.score, max: latestTotal.max } : null, + completedGrades: totals.length, + // Every completed grade in order — the attempt's trajectory — carrying the + // level it graded and whether it was the unaided build of that level. A + // view can draw the climb with its bands, and a flat tail is the stall an + // operator otherwise discovers by diffing round logs. + series: totals.map(total => { + const mark = grading.findLast(entry => entry.index < total.index) ?? null; + return { score: total.score, max: total.max, level: mark?.level ?? null, + unaided: mark ? mark.round === 0 : false }; + }), + }; +} + +export function attemptPause(plan: CompiledCampaignPlan, attempt: CampaignAttemptState, + directory: string) { + const depth = plan.definition.mode.pauseAfterDepth; + const execution = attempt.executions.at(-1); + const lock = depth === undefined ? null : readCampaignLock(directory); + if (depth === undefined || !execution || !lock) return null; + return readDepthPause(contained(directory, execution.output, 'campaign execution'), { + directory, depth, campaignSha256: plan.contentSha256, + ownershipMarkerSha256: lock.ownershipMarkerSha256, + attemptId: attempt.plan.id, executionId: execution.id, + }); +} + +function summarizeAttempt(plan: CompiledCampaignPlan, attempt: CampaignAttemptState, + campaignDirectory: string, repairs: number, { includeLog = false }: { + includeLog?: boolean; + } = {}) { + const inspected = inspectCampaignAttempt(plan, attempt, campaignDirectory); + const execution = inspected.execution; + let executionDirectory = null; + let log = ''; + let logUpdatedAt = null; + if (execution) { + executionDirectory = contained(campaignDirectory, execution.output, 'campaign execution'); + const logPath = join(executionDirectory, 'process.stdout.log'); + log = readTextTail(logPath); + // When the run last wrote anything. A running attempt whose output has + // been silent for a long time is wedged in a way no score can show. + if (existsSync(logPath)) logUpdatedAt = new Date(statSync(logPath).mtimeMs).toISOString(); + } + const progress = parseRunProgress(log, { repairs, running: attempt.status === 'running', + status: attempt.status, dependency: plan.definition.mode.id === 'dependency' }); + const pause = attemptPause(plan, attempt, campaignDirectory); + const paused = attempt.status === 'running' && pause?.resumedAt === null; + if (paused) progress.phase = `Paused at L${pause.depth}`; + if (inspected.result?.score) progress.latestScore = inspected.result.score; + return { + ...inspected, + progress, + paused, + logUpdatedAt, + ...(includeLog ? { log: log.split(/\r?\n/).slice(-160).join('\n') } : {}), + }; +} + +export function summarizeCampaign(directory: string, { + includeLogs = false, + includePackage = false, + includeAttempts = true, + controllerActive = null, +}: { + includeLogs?: boolean; + includePackage?: boolean; + includeAttempts?: boolean; + controllerActive?: ControllerActive | null; +} = {}) { + const { plan, state } = readCampaignState(directory, { requireCurrentInputs: false }); + let attempts = includeAttempts + ? state.attempts.map(attempt => summarizeAttempt(plan, attempt, directory, + repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }), { includeLog: includeLogs })) + : []; + const interrupted = state.status === 'running' && controllerActive !== null + && !controllerActive(directory, plan); + if (interrupted) { + attempts = attempts.map(attempt => attempt.status !== 'running' ? attempt : ({ + ...attempt, + status: 'interrupted', + execution: attempt.execution ? { ...attempt.execution, status: 'interrupted' } : null, + progress: { ...attempt.progress, phase: 'Controller stopped before completion' }, + })); + } + return { + key: basename(resolve(directory)), + id: plan.id, + version: plan.version, + sha256: plan.contentSha256, + title: plan.title, + state: plan.state, + mode: plan.definition.mode?.id ?? 'sequential', + status: interrupted ? 'attention-required' : state.status, + track: plan.definition.track, + levels: plan.definition.levels, + stacks: plan.stacks.map(stack => stack.id), + repetitions: plan.definition.repetitions, + maxParallel: state.maxParallel, + createdAt: state.createdAt, + updatedAt: state.updatedAt, + summary: interrupted ? { ...state.summary, interrupted: state.summary.running, running: 0 } + : state.summary, + interrupted, + ...(interrupted ? { statusReason: 'The campaign controller is no longer running.' } : {}), + budgets: plan.definition.budgets, + facts: campaignFacts(plan), + attempts, + ...(includePackage ? { package: campaignPackage(directory, state.attempts) } : {}), + }; +} + +export interface UnreadableDashboardCampaign { + key: string; + id: string; + title: string; + status: 'unreadable'; + error: string; + attempts: []; +} + +export type DashboardCampaign = ReturnType; +export type DashboardCampaignSummary = DashboardCampaign | UnreadableDashboardCampaign; + +const overviewCampaignCache = new Map(); + +function summarizeOverviewCampaign(directory: string, includeAttempts: boolean, + controllerActive: ControllerActive): DashboardCampaign { + const fingerprint = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state] + .map(file => { + const stat = statSync(join(directory, file)); + return `${stat.size}:${stat.mtimeMs}`; + }).join('|'); + const key = `${includeAttempts ? 'attempts' : 'summary'}:${directory}`; + const cached = overviewCampaignCache.get(key); + if (cached?.fingerprint === fingerprint) return cached.campaign; + const campaign = summarizeCampaign(directory, { includeAttempts, controllerActive }); + if (campaign.summary.running === 0) { + overviewCampaignCache.set(key, { fingerprint, campaign }); + } + return campaign; +} + +export function discoverCampaigns(campaignsRoot: string, { + includeLogs = false, + controllerActive = campaignLockIsActive, +}: { includeLogs?: boolean; controllerActive?: ControllerActive } = {}) { + if (!existsSync(campaignsRoot)) return []; + const campaigns: DashboardCampaignSummary[] = []; + for (const entry of readdirSync(campaignsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const directory = join(campaignsRoot, entry.name); + if (!existsSync(join(directory, CAMPAIGN_FILE.state)) + || !existsSync(join(directory, CAMPAIGN_FILE.plan))) continue; + try { + campaigns.push(includeLogs + ? summarizeCampaign(directory, { includeLogs, controllerActive }) + : summarizeOverviewCampaign(directory, false, controllerActive)); + } catch (error) { + campaigns.push({ key: entry.name, id: entry.name, title: entry.name, + status: 'unreadable', error: errorMessage(error), attempts: [] }); + } + } + campaigns.sort((left, right) => String('updatedAt' in right ? right.updatedAt ?? '' : '') + .localeCompare(String('updatedAt' in left ? left.updatedAt ?? '' : ''))); + if (includeLogs) return campaigns; + + const verdict = campaigns.find(campaign => campaign.status === 'completed' + && 'facts' in campaign && campaign.facts.grading.status === 'qualified'); + return campaigns.map(campaign => { + if (campaign.status !== 'running' && campaign !== verdict) return campaign; + try { + return summarizeOverviewCampaign(join(campaignsRoot, campaign.key), true, controllerActive); + } catch (error) { + const unreadable: UnreadableDashboardCampaign = { + key: campaign.key, + id: campaign.id, + title: campaign.title, + status: 'unreadable', + error: errorMessage(error), + attempts: [], + }; + return unreadable; + } + }); +} + +export interface DashboardPlan { + id: string; + version?: string; + title: string; + state: string; + mode?: string; + track?: string; + levels?: number[]; + stacks?: string[]; + attempts?: number; + parallelism?: number; + budgets?: CompiledCampaignPlan['definition']['budgets']; + repairBudget?: number; + sha256?: string; + file: string; + error?: string; +} + +export function discoverPlans(plansRoot: string): DashboardPlan[] { + if (!existsSync(plansRoot)) return []; + const plans: DashboardPlan[] = []; + for (const entry of readdirSync(plansRoot, { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith('.json')) continue; + const path = join(plansRoot, entry.name); + try { + const plan = compileCampaignFile(path); + plans.push({ id: plan.id, version: plan.version, title: plan.title, state: plan.state, + mode: plan.definition.mode?.id ?? 'sequential', + track: plan.definition.track, levels: plan.definition.levels, + stacks: plan.stacks.map(stack => stack.id), attempts: plan.summary.attempts, + parallelism: plan.summary.parallelism, budgets: plan.definition.budgets, + repairBudget: repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }), + sha256: plan.contentSha256, file: entry.name }); + } catch (error) { + plans.push({ id: entry.name.slice(0, -5), title: entry.name, state: 'invalid', + error: errorMessage(error), file: entry.name }); + } + } + return plans.sort((left, right) => left.title.localeCompare(right.title)); +} + +export function readJsonLines(path: string): unknown[] { + if (!existsSync(path)) return []; + const lines = readFileSync(path, 'utf8').split(/\r?\n/); + const last = lines.findLastIndex(line => line.trim() !== ''); + const events: unknown[] = []; + for (let index = 0; index <= last; index += 1) { + const line = lines[index]; + if (!line?.trim()) continue; + try { events.push(JSON.parse(line)); } + catch { + if (index === last) break; + throw new Error(`dashboard operation feed line ${index + 1} is invalid JSON`); + } + } + return events; +} diff --git a/tools/stack-bench/dashboard/dashboard-reader.ts b/tools/stack-bench/dashboard/dashboard-reader.ts new file mode 100644 index 00000000000..65a1e59b821 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-reader.ts @@ -0,0 +1,57 @@ +import { isMainThread, parentPort, Worker } from 'node:worker_threads'; +import { overviewPage, campaignLiveSheet, campaignLiveUpdate, campaignLiveProgression, + attemptChecks, attemptPackage, attemptTranscript, attemptLogSlice } from './dashboard-views.js'; + +const reads = { overviewPage, campaignLiveSheet, campaignLiveUpdate, campaignLiveProgression, + attemptChecks, attemptPackage, attemptTranscript, attemptLogSlice }; +type ReadName = keyof typeof reads; + +if (!isMainThread) { + parentPort!.on('message', async ({ id, name, args }: { id: number; name: ReadName; args: never[] }) => { + try { parentPort!.postMessage({ id, value: await (reads[name] as (...input: never[]) => unknown)(...args) }); } + catch (error) { parentPort!.postMessage({ id, error: error instanceof Error ? error.message : String(error) }); } + }); +} + +// One reader retains the existing view caches. CPU-heavy evidence validation +// must not block health, events, or run controls on the HTTP thread. +export function createDashboardReader() { + let worker: Worker | null = null; + let sequence = 0; + let closed = false; + const pending = new Map void; reject: (error: Error) => void }>(); + const inflight = new Map>(); + function read(name: Name, ...args: Parameters): Promise>> { + if (closed) return Promise.reject(new Error('Dashboard reader closed')); + const key = JSON.stringify([name, args]); + if (!worker) { + const active = worker = new Worker(new URL('./dashboard-reader.js', import.meta.url)); + const fail = (error: Error): void => { + if (worker !== active) return; + for (const request of pending.values()) request.reject(error); + pending.clear(); + inflight.clear(); + worker = null; + }; + active.on('message', ({ id, value, error }) => { + const request = pending.get(id); + pending.delete(id); + if (error !== undefined) request?.reject(new Error(error)); + else request?.resolve(value); + }); + active.once('error', fail); + active.once('exit', code => fail(new Error(`Dashboard reader exited (${code})`))); + } + let result = inflight.get(key); + if (!result) { + const id = ++sequence; + result = new Promise((resolve, reject) => { + pending.set(id, { resolve, reject }); + worker!.postMessage({ id, name, args }); + }).finally(() => inflight.delete(key)); + inflight.set(key, result); + } + return result as Promise>>; + } + return { read, close: () => { closed = true; void worker?.terminate(); } }; +} diff --git a/tools/stack-bench/dashboard/dashboard-reference-runs.ts b/tools/stack-bench/dashboard/dashboard-reference-runs.ts new file mode 100644 index 00000000000..b0604434999 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-reference-runs.ts @@ -0,0 +1,81 @@ +import { execFile } from 'node:child_process'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { basename, join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +const exec = promisify(execFile); +const docker = async (args: string[]) => { + const result = await exec('docker', args, + { timeout: 5000, maxBuffer: 4 * 1024 * 1024, windowsHide: true }); + return result.stdout + (args[0] === 'logs' ? result.stderr : ''); +}; + +export interface ReferenceRun { + id: string; + title: string; + status: 'running' | 'passed' | 'failed' | 'incomplete'; + updatedAt: string; + points: { passed: number; measured: number; planned: number | null } | null; + log: string; +} + +export function referenceLogPoints(log: string): ReferenceRun['points'] { + // A repetition starts a new measurement. Do not add repeated scores together. + const latest = log.split(/qualifying [^\n]+: clean run \d+\/\d+/).at(-1) ?? ''; + const rows = [...latest.matchAll(/^\s+selected-source-\d+ \.\.\. (\d+)\/(\d+)\s*$/gm)]; + if (!rows.length) return null; + const scope = latest.match(/scope: \d+ check\(s\), (\d+) point\(s\)/); + return { passed: rows.reduce((n, row) => n + Number(row[1]), 0), + measured: rows.reduce((n, row) => n + Number(row[2]), 0), planned: scope ? Number(scope[1]) : null }; +} + +export async function referenceRuns(resultsRoot: string, readDocker = docker): Promise<{ runs: ReferenceRun[]; error: string | null }> { + const root = resolve(resultsRoot, 'reference-live'); + const runs = new Map(); + // Finished artifacts remain visible after their controller containers are removed. + if (existsSync(root)) for (const file of readdirSync(root, { withFileTypes: true })) { + if (!file.isFile() || !file.name.endsWith('.json') || file.name.endsWith('.inputs.json')) continue; + try { + const value = JSON.parse(readFileSync(join(root, file.name), 'utf8')); + const artifact = value.payload ?? value; + if (value.kind !== 'reference_qualification') continue; + const score = String(artifact.runs?.at(-1)?.score ?? '').match(/^(\d+)\/(\d+)$/); + runs.set(file.name, { id: file.name, title: String(artifact.fixture ?? file.name), + status: artifact.ok === true ? 'passed' : 'failed', + updatedAt: String(value.timestamps?.completedAt ?? value.timestamps?.startedAt ?? artifact.completedAt ?? artifact.startedAt ?? ''), + points: score ? { passed: Number(score[1]), measured: Number(score[2]), planned: Number(score[2]) } : null, + log: redactCredentials((artifact.runs ?? []).flatMap((run: { failures?: string[] }) => run.failures ?? []).join('\n')) }); + } catch { /* A qualification artifact may be in the middle of an atomic replacement. */ } + } + try { + const listing = await readDocker(['ps', '-a', '--no-trunc', '--format', '{{json .}}']); + const ids = listing.trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) + .filter(row => String(row.Command).includes('/references/reference-live.js')) + .map(row => String(row.ID)).filter(id => /^[a-f0-9]{64}$/.test(id)); + if (ids.length) { + const containers = JSON.parse(await readDocker(['inspect', ...ids])); + await Promise.all(containers.map(async (container: { Id: string; Args: string[]; + State: { Running: boolean; StartedAt: string; FinishedAt: string } }) => { + const args = container.Args; + const output = args[args.indexOf('--out') + 1]; + // Only this dashboard's reference output tree belongs here. + if (!args.includes('--out') || !output || resolve(output) !== join(root, basename(output))) return; + const id = basename(output), final = runs.get(id); + const level = args[args.indexOf('--level') + 1]; + const stack = args[args.indexOf('--backend') + 1]; + let log: string; + try { log = await readDocker(['logs', '--tail', '4000', container.Id]); } + catch { log = 'Controller log is unavailable.'; } + runs.set(id, { id, title: `${stack} L${level} reference validation`, + status: container.State.Running ? 'running' : final?.status ?? 'incomplete', + updatedAt: container.State.Running ? container.State.StartedAt : container.State.FinishedAt, + points: final?.points ?? referenceLogPoints(log), + log: redactCredentials(log).slice(-96 * 1024) + (final?.log ? `\n${final.log}` : '') }); + })); + } + return { runs: [...runs.values()].sort((a, b) => b.updatedAt.localeCompare(a.updatedAt)), error: null }; + } catch { + return { runs: [...runs.values()], error: 'Live reference status is unavailable. Docker could not be read.' }; + } +} diff --git a/tools/stack-bench/dashboard/dashboard-server.ts b/tools/stack-bench/dashboard/dashboard-server.ts new file mode 100644 index 00000000000..dd9cfdf2b2f --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-server.ts @@ -0,0 +1,632 @@ +#!/usr/bin/env node +import { prepareRun, runSetupCatalog, submitPreparedRun } from '../src/campaigns/run-setup.js'; + +import { randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'; +import { appendFileSync, closeSync, createReadStream, existsSync, mkdirSync, openSync, statSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { basename, dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { spawn } from 'node:child_process'; +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { contained, discoverPlans, readCampaignArtifactBody, + readJsonLines, resolveCampaignArtifact, summarizeCampaign, +} from './dashboard-model.js'; +import type { DashboardPlan } from './dashboard-model.js'; +import { createDashboardReader } from './dashboard-reader.js'; +import type { CampaignFilter } from './dashboard-views.js'; +import { watchCampaigns } from './dashboard-events.js'; +import type { CampaignChange, CampaignWatcher } from './dashboard-events.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { referenceRuns } from './dashboard-reference-runs.js'; +import { checkGuidePage } from './check-guide.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import { controllerRuntimeCommand, controllerChildEnvironment } from '../appliance/controller.js'; +import { requestCampaignCancellation } from '../src/campaigns/campaign-lock.js'; +import { readCampaignTimeBudget, requestCampaignTimeGrant } from '../src/campaigns/campaign-time-grant.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { submitExecutionJob, listExecutionJobs, readExecutionJob, cancelExecutionJob } + from '../src/campaigns/execution-jobs.js'; + +const DASHBOARD_ROOT = dirname(fileURLToPath(import.meta.url)); +const PUBLIC_ROOT = join(DASHBOARD_ROOT, 'public'); +const SAFE_NAME = /^[a-z0-9][a-z0-9.-]{2,119}$/; +const SPA_PATH = /^\/(?:new|plans|c\/[^/]+(?:\/a\/[^/]+)?)$/; +const HEARTBEAT_MS = 25_000; +const LOOPBACK = new Set(['127.0.0.1', '::1', 'localhost']); +const STATIC = new Map([ + ['/', ['index.html', 'text/html; charset=utf-8']], + ['/app.js', ['app.js', 'text/javascript; charset=utf-8']], + ['/check-guide.js', ['check-guide.js', 'text/javascript; charset=utf-8']], + ['/climb.js', ['climb.js', 'text/javascript; charset=utf-8']], + ['/format.js', ['format.js', 'text/javascript; charset=utf-8']], + ['/progress-chart.js', ['progress-chart.js', 'text/javascript; charset=utf-8']], + ['/graph.js', ['graph.js', 'text/javascript; charset=utf-8']], + ['/metrics.js', ['metrics.js', 'text/javascript; charset=utf-8']], + // Shared with the CLI so a state has one name on every surface. + ['/src/evidence/status-words.js', ['../../src/evidence/status-words.js', 'text/javascript; charset=utf-8']], + ['/views/attempt.js', ['views/attempt.js', 'text/javascript; charset=utf-8']], + ['/views/campaign.js', ['views/campaign.js', 'text/javascript; charset=utf-8']], + ['/views/campaigns.js', ['views/campaigns.js', 'text/javascript; charset=utf-8']], + ['/views/plans.js', ['views/plans.js', 'text/javascript; charset=utf-8']], + ['/views/run-setup.js', ['views/run-setup.js', 'text/javascript; charset=utf-8']], + ['/styles.css', ['styles.css', 'text/css; charset=utf-8']], + ['/spacetimedb-mark.svg', ['spacetimedb-mark.svg', 'image/svg+xml']], + // The brand faces are served from here rather than a CDN: the dashboard's own + // content-security-policy allows 'self' only, and the appliance has no + // outbound access to fetch them at view time. + ['/fonts/inter-latin-variable.woff2', ['fonts/inter-latin-variable.woff2', 'font/woff2']], + ['/fonts/source-code-pro-latin-variable.woff2', ['fonts/source-code-pro-latin-variable.woff2', 'font/woff2']], +]); + +interface DashboardArgs { + host: string; + port: number; + resultsRoot: string; + plansRoot: string; + allowContainerBind: boolean; +} + +export interface DashboardOperation { + id: string; + updatedAt: string; + [key: string]: unknown; +} + +function dashboardOperation(value: unknown): DashboardOperation { + if (!value || typeof value !== 'object') throw new Error('dashboard operation must be an object'); + const id = 'id' in value ? value.id : undefined; + const updatedAt = 'updatedAt' in value ? value.updatedAt : undefined; + if (typeof id !== 'string' || !id) throw new Error('dashboard operation id is required'); + if (typeof updatedAt !== 'string' || !updatedAt) { + throw new Error('dashboard operation updatedAt is required'); + } + return { ...value, id, updatedAt }; +} + +export interface OperationFeed { + readonly path?: string; + append(event: DashboardOperation): void; + list(): DashboardOperation[]; +} + +export interface LaunchChild { + pid?: number; + once(event: 'error', listener: (error: Error) => void): unknown; + once(event: 'exit', listener: (code: number | null, signal: NodeJS.Signals | null) => void): unknown; +} + +export interface LaunchInput { + command: 'resume' | 'work'; + jobId?: string; + plan: DashboardPlan & { path: string }; + output: string; + operationId: string; + resultsRoot: string; + feed: OperationFeed; + env?: NodeJS.ProcessEnv; +} + +export interface DashboardServerOptions { + resultsRoot: string; + plansRoot: string; + allowLaunch?: boolean; + token?: string; + feed?: OperationFeed; + launch?: (input: LaunchInput) => LaunchChild; + plans?: () => DashboardPlan[]; +} + +function errorMessage(error: unknown): string { + return redactCredentials(error instanceof Error ? error.message : String(error)); +} + +function loopbackHost(value: unknown): boolean { + return /^(?:localhost|127\.0\.0\.1|\[::1\])(?::\d+)?$/i.test(String(value ?? '')); +} + +function sameSecret(actual: unknown, expected: unknown): boolean { + if (typeof actual !== 'string' || typeof expected !== 'string') return false; + const left = Buffer.from(actual); + const right = Buffer.from(expected); + return left.length === right.length && timingSafeEqual(left, right); +} + +function controlAuthorized(request: IncomingMessage, host: string | undefined, + csrfToken: string): boolean { + return request.headers.origin === `http://${host}` + && sameSecret(request.headers['x-stack-bench-token'], csrfToken); +} + +export function parseDashboardArgs(argv: string[], env: NodeJS.ProcessEnv = process.env): DashboardArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + host: { type: 'string' }, port: { type: 'string' }, results: { type: 'string' }, + plans: { type: 'string' }, 'allow-container-bind': { type: 'boolean' }, + } }); + const args: DashboardArgs = { host: values.host ?? '127.0.0.1', + port: values.port === undefined ? 7331 : Number(values.port), + resultsRoot: stackBenchResultsRoot(STACK_BENCH_ROOT, env), + plansRoot: '', allowContainerBind: values['allow-container-bind'] ?? false }; + if (values.results) args.resultsRoot = resolve(values.results); + if (values.plans) args.plansRoot = resolve(values.plans); + args.plansRoot ||= join(args.resultsRoot, 'plans'); + const applianceContainerBind = args.allowContainerBind + && env.STACK_BENCH_APPLIANCE === '1' && args.host === '0.0.0.0'; + if (!LOOPBACK.has(args.host) && !applianceContainerBind) { + throw new Error('dashboard must bind to localhost or a loopback address'); + } + if (!Number.isInteger(args.port) || args.port < 1 || args.port > 65535) { + throw new Error('dashboard port must be an integer from 1 through 65535'); + } + return args; +} + +function json(response: ServerResponse, status: number, value: unknown): void { + const body = Buffer.from(`${JSON.stringify(value)}\n`); + response.writeHead(status, { 'content-type': 'application/json; charset=utf-8', + 'content-length': body.length, 'cache-control': 'no-store' }); + response.end(body); +} + +function securityHeaders(response: ServerResponse): void { + response.setHeader('content-security-policy', "default-src 'self'; connect-src 'self'; img-src 'self' data:; script-src 'self'; style-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'"); + response.setHeader('x-content-type-options', 'nosniff'); + response.setHeader('x-frame-options', 'DENY'); + response.setHeader('referrer-policy', 'no-referrer'); +} + +async function body(request: IncomingMessage): Promise { + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + bytes += buffer.length; + if (bytes > 16 * 1024) throw new Error('request body is too large'); + chunks.push(buffer); + } + try { return JSON.parse(Buffer.concat(chunks).toString('utf8')); + } catch { throw new Error('request body must be valid JSON'); } +} + +function createOperationFeed(resultsRoot: string): OperationFeed { + const root = join(resolve(resultsRoot), 'dashboard'); + const path = join(root, 'operations.jsonl'); + mkdirSync(root, { recursive: true }); + return { + path, + append(event: DashboardOperation) { + appendFileSync(path, `${JSON.stringify(event)}\n`, { encoding: 'utf8', mode: 0o600 }); + }, + list() { + const latest = new Map(); + for (const value of readJsonLines(path)) { + const event = dashboardOperation(value); + latest.set(event.id, { ...(latest.get(event.id) ?? {}), ...event }); + } + return [...latest.values()].sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)); + }, + }; +} + +function launchCampaign({ command, jobId, plan, output, operationId, resultsRoot, feed, + env = process.env }: LaunchInput): LaunchChild { + const runtime = controllerRuntimeCommand(command === 'work' + ? ['job', 'work', jobId!, '--results', resultsRoot, '--host', env.STACK_BENCH_HOST_ID ?? 'local'] + : ['campaign', command, plan.path, '--out', output], env); + feed.append({ id: operationId, updatedAt: new Date().toISOString(), + containerName: runtime.containerName, ownershipLabel: runtime.ownershipLabel }); + const operationsRoot = join(resolve(resultsRoot), 'dashboard', 'operations'); + mkdirSync(operationsRoot, { recursive: true }); + const stdoutPath = join(operationsRoot, `${operationId}.stdout.log`); + const stderrPath = join(operationsRoot, `${operationId}.stderr.log`); + const stdout = openSync(stdoutPath, 'a', 0o600); + const stderr = openSync(stderrPath, 'a', 0o600); + let child; + try { + child = spawn(runtime.executable, runtime.args, { + cwd: STACK_BENCH_ROOT, env: runtime.env, stdio: ['ignore', stdout, stderr], windowsHide: true, + }); + } finally { + closeSync(stdout); + closeSync(stderr); + } + child.once('error', error => feed.append({ schemaVersion: 1, id: operationId, + status: 'failed', updatedAt: new Date().toISOString(), error: errorMessage(error) })); + child.once('exit', (code, signal) => feed.append({ schemaVersion: 1, id: operationId, + status: code === 0 ? 'completed' : 'failed', updatedAt: new Date().toISOString(), + exitCode: code, signal })); + return child; +} + +export function createDashboardServer(options: DashboardServerOptions) { + const resultsRoot = resolve(options.resultsRoot); + const plansRoot = resolve(options.plansRoot); + const allowLaunch = options.allowLaunch ?? process.env.STACK_BENCH_APPLIANCE === '1'; + const token = options.token ?? randomBytes(24).toString('base64url'); + const feed = options.feed ?? createOperationFeed(resultsRoot); + const launch = options.launch ?? launchCampaign; + const plans = options.plans ?? (() => discoverPlans(plansRoot)); + const launchReservations = new Set(); + const dispatchJob = (job: ReturnType) => { + const status = readExecutionJob(resultsRoot, job.id); + const key = `job-${job.id}`; + if (status.status === 'queued' && !launchReservations.has(key)) { + launchReservations.add(key); + const now = new Date().toISOString(); + const operation = { schemaVersion: 1, id: randomUUID(), type: 'campaign.run', status: 'running', + createdAt: now, updatedAt: now, actor: 'local-operator', campaignId: job.key, + campaignSha256: job.planSha256, outputName: key }; + feed.append(operation); + try { + const child = launch({ command: 'work', jobId: job.id, + plan: { id: job.key, title: job.key, state: 'frozen', file: 'plan.json', + path: join(resultsRoot, 'jobs', job.id, 'plan.json') }, + output: status.campaignDirectory, operationId: operation.id, resultsRoot, feed }); + child.once('error', () => launchReservations.delete(key)); + child.once('exit', () => launchReservations.delete(key)); + feed.append({ ...operation, pid: child.pid ?? null }); + } catch (error) { + launchReservations.delete(key); + feed.append({ ...operation, status: 'failed', error: errorMessage(error) }); + throw new Error(`Job ${job.id} is saved but dispatch failed. Retry Start with the same setup. ${errorMessage(error)}`); + } + } + return { ...readExecutionJob(resultsRoot, job.id), campaignKey: key }; + }; + const campaignsRoot = join(resultsRoot, 'campaigns'); + const listeners = new Set(); + const reader = createDashboardReader(); + let watcher: CampaignWatcher | null = null; + let heartbeat: NodeJS.Timeout | null = null; + const broadcast = (change: CampaignChange): void => { + const frame = `event: ${change.type}\ndata: ${JSON.stringify({ key: change.key, + ...(change.attemptId === undefined ? {} : { attemptId: change.attemptId }) })}\n\n`; + for (const listener of listeners) listener.write(frame); + }; + const stopEvents = (): void => { + watcher?.close(); + watcher = null; + if (heartbeat) clearInterval(heartbeat); + heartbeat = null; + }; + const server = createServer(async (request, response) => { + securityHeaders(response); + try { + if (!loopbackHost(request.headers.host)) { + return json(response, 421, { error: 'Dashboard requests must use a loopback host.' }); + } + const url = new URL(request.url ?? '/', `http://${request.headers.host ?? 'localhost'}`); + if (request.method === 'GET' && url.pathname === '/checks') { + const html = checkGuidePage(); + response.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' }); + response.end(html); + return; + } + // The client routes are pages, not fragments: each serves the shell. + const staticFile = STATIC.get(url.pathname) + ?? (SPA_PATH.test(url.pathname) ? STATIC.get('/') : undefined); + if (request.method === 'GET' && staticFile) { + const [file, type] = staticFile; + const path = join(PUBLIC_ROOT, file); + const size = existsSync(path) ? statSync(path).size : 0; + if (!size) return json(response, 404, { error: 'Not found' }); + response.writeHead(200, { 'content-type': type, 'content-length': size, + 'cache-control': file === 'index.html' ? 'no-store' : 'no-cache' }); + createReadStream(path).pipe(response); + return; + } + if (request.method === 'GET' && url.pathname === '/api/run-setup') { + return json(response, 200, runSetupCatalog(resultsRoot)); + } + if (request.method === 'POST' && ['/api/runs/prepare', '/api/runs'].includes(url.pathname)) { + if (!allowLaunch) return json(response, 503, { error: 'Run controls require the appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The run request is not authorized.' }); + } + if (!String(request.headers['content-type'] ?? '').toLowerCase().startsWith('application/json')) { + return json(response, 415, { error: 'Run requests must use JSON.' }); + } + try { + const input = await body(request); + if (url.pathname.endsWith('/prepare')) return json(response, 200, prepareRun(resultsRoot, input, options.launch ? process.env : controllerChildEnvironment(process.env))); + // Validate dispatch configuration before publishing a job for workers. + if (!options.launch) controllerRuntimeCommand(['job', 'work'], process.env); + const job = submitPreparedRun(resultsRoot, input, options.launch ? process.env : controllerChildEnvironment(process.env)); + return json(response, 202, dispatchJob(job)); + } catch (error) { return json(response, 400, { error: errorMessage(error) }); } + } + if (request.method === 'GET' && url.pathname === '/api/health') { + return json(response, 200, { ok: true, mode: allowLaunch ? 'controller' : 'read-only' }); + } + if (request.method === 'GET' && url.pathname === '/api/overview') { + const page = Number(url.searchParams.get('page') ?? 1); + const filter = url.searchParams.get('filter') ?? 'all'; + if (!Number.isSafeInteger(page) || page < 1 || !['all', 'attention', 'completed', 'ready'].includes(filter)) { + return json(response, 400, { error: 'Use a positive page number and a valid campaign filter.' }); + } + return json(response, 200, { ...await reader.read('overviewPage', campaignsRoot, page, filter as CampaignFilter), + canStart: allowLaunch, csrfToken: token }); + } + if (request.method === 'GET' && url.pathname === '/api/reference-runs') { + return json(response, 200, await referenceRuns(resultsRoot)); + } + if (request.method === 'GET' && url.pathname === '/api/session') { + return json(response, 200, { canStart: allowLaunch, csrfToken: token }); + } + if (request.method === 'GET' && url.pathname === '/api/plans') { + return json(response, 200, plans()); + } + const jobRoute = url.pathname.match(/^\/api\/jobs(?:\/([a-f0-9]{64})(?:\/(cancel|start))?)?$/); + if (jobRoute) { + const id = jobRoute[1], cancel = jobRoute[2] !== undefined; + if (request.method === 'GET' && !cancel) { + if (!id) { + const after = url.searchParams.get('after') ?? ''; + const limit = Number(url.searchParams.get('limit') ?? 50); + if ((after && !/^[a-f0-9]{64}$/.test(after)) || !Number.isSafeInteger(limit) || limit < 1 || limit > 200) { + return json(response, 400, { error: 'Use a valid job cursor and a page size from 1 through 200.' }); + } + return json(response, 200, listExecutionJobs(resultsRoot, { after, limit })); + } + if (!existsSync(join(resultsRoot, 'jobs', id, 'job.json'))) return json(response, 404, { error: 'Job not found.' }); + return json(response, 200, readExecutionJob(resultsRoot, id)); + } + if (request.method === 'POST' && (!id || cancel)) { + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The job request is not authorized.' }); + } + if (id) { + if (!existsSync(join(resultsRoot, 'jobs', id, 'job.json'))) return json(response, 404, { error: 'Job not found.' }); + if (jobRoute[2] === 'start') { + if (!options.launch) controllerRuntimeCommand(['job', 'work'], process.env); + return json(response, 202, dispatchJob(readExecutionJob(resultsRoot, id).job)); + } + cancelExecutionJob(resultsRoot, id); + return json(response, 202, readExecutionJob(resultsRoot, id)); + } + if (!String(request.headers['content-type'] ?? '').toLowerCase().startsWith('application/json')) { + return json(response, 415, { error: 'Job submissions must use JSON.' }); + } + try { + const job = submitExecutionJob(resultsRoot, await body(request)); + return json(response, 202, readExecutionJob(resultsRoot, job.id)); + } catch (error) { return json(response, 400, { error: errorMessage(error) }); } + } + } + if (request.method === 'GET' && url.pathname === '/api/events') { + response.writeHead(200, { 'content-type': 'text/event-stream; charset=utf-8', + 'cache-control': 'no-store', connection: 'keep-alive' }); + response.write(': open\n\n'); + listeners.add(response); + watcher ??= watchCampaigns(campaignsRoot, broadcast, + mode => console.log(`Stack Bench dashboard: campaign watcher ${mode}`)); + // A silent connection is dropped by proxies long before a campaign + // writes anything. + heartbeat ??= setInterval(() => { + for (const listener of listeners) listener.write('event: reference\ndata: {}\n\n'); + }, HEARTBEAT_MS).unref(); + request.once('close', () => { + listeners.delete(response); + if (!listeners.size) stopEvents(); + }); + return; + } + const timeRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/attempts\/([^/]+)\/time$/); + if (timeRoute && (request.method === 'GET' || request.method === 'POST')) { + const key = decodeURIComponent(timeRoute[1] ?? ''); + if (!SAFE_NAME.test(key)) return json(response, 400, { error: 'The campaign name is invalid.' }); + const directory = contained(campaignsRoot, key, 'campaign'); + const attemptId = decodeURIComponent(timeRoute[2] ?? ''); + if (request.method === 'GET') return json(response, 200, readCampaignTimeBudget(directory, attemptId)); + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The time request is not authorized.' }); + } + const input = await body(request) as { minutes?: unknown; grantId?: unknown } | null; + if (!input || typeof input.minutes !== 'number' || !Number.isSafeInteger(input.minutes * 60_000) + || !Number.isInteger(input.minutes) || input.minutes <= 0 || typeof input.grantId !== 'string') { + return json(response, 400, { error: 'Positive whole minutes and a grant ID are required.' }); + } + try { + const receipt = requestCampaignTimeGrant(directory, { + attemptId, grantId: input.grantId, minutes: input.minutes, + }); + return json(response, receipt.disposition === 'rejected' ? 409 : 202, + receipt.disposition === 'rejected' ? { ...receipt, error: receipt.reason } : receipt); + } catch (error) { + return json(response, 409, { error: errorMessage(error) }); + } + } + const stopRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/stop$/); + if (request.method === 'POST' && stopRoute) { + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The stop request is not authorized.' }); + } + const key = decodeURIComponent(stopRoute[1] ?? ''); + if (!SAFE_NAME.test(key)) return json(response, 400, { error: 'The campaign name is invalid.' }); + const input = await body(request); + const owner = input && typeof input === 'object' && 'owner' in input ? input.owner : null; + if (typeof owner !== 'string' || !/^[a-f0-9]{64}$/.test(owner)) { + return json(response, 400, { error: 'The current controller identity is required.' }); + } + const directory = contained(campaignsRoot, key, 'campaign'); + const campaign = summarizeCampaign(directory, { includeAttempts: false }); + if (!requestCampaignCancellation(directory, + { id: campaign.id, contentSha256: campaign.sha256 }, owner)) { + return json(response, 409, { error: 'The controller changed or stopped. Refresh the campaign.' }); + } + const now = new Date().toISOString(); + const operation = { id: randomUUID(), type: 'campaign.stop', status: 'requested', + updatedAt: now, campaignId: campaign.id, outputName: key, owner }; + feed.append(operation); + return json(response, 202, operation); + } + const resumeRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/resume$/); + if (request.method === 'POST' && resumeRoute) { + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The run request is not authorized.' }); + } + const key = decodeURIComponent(resumeRoute[1] ?? ''); + if (!SAFE_NAME.test(key)) return json(response, 400, { error: 'The campaign name is invalid.' }); + const campaign = summarizeCampaign( + contained(join(resultsRoot, 'campaigns'), key, 'campaign'), { includeAttempts: false }); + const priorExecutions = campaign.summary?.executions ?? 0; + if (campaign.mode !== 'dependency' || campaign.status !== 'prepared' || priorExecutions < 1) { + return json(response, 409, { error: 'Only an interrupted campaign that is ready can resume.' }); + } + const plan = plans().find(item => item.id === campaign.id && item.sha256 === campaign.sha256); + if (!plan || plan.state !== 'frozen') { + return json(response, 409, { error: 'The test plan used by this campaign is unavailable.' }); + } + const reservation = `${campaign.id}:${campaign.sha256}:${key}`; + if (launchReservations.has(reservation)) { + return json(response, 409, { error: 'This campaign already has an active controller.' }); + } + launchReservations.add(reservation); + const now = new Date().toISOString(); + const operation = { schemaVersion: 1, id: randomUUID(), type: 'campaign.resume', + status: 'running', createdAt: now, updatedAt: now, actor: 'local-operator', + campaignId: campaign.id, campaignSha256: campaign.sha256, outputName: key }; + feed.append(operation); + const output = join(resultsRoot, 'campaigns', key); + try { + const child = launch({ command: 'resume', plan: { ...plan, path: join(plansRoot, plan.file) }, output, + operationId: operation.id, resultsRoot, feed, env: process.env }); + if (typeof child?.once === 'function') { + child.once('error', () => launchReservations.delete(reservation)); + child.once('exit', () => launchReservations.delete(reservation)); + } else { + launchReservations.delete(reservation); + } + feed.append({ ...operation, pid: child?.pid ?? null }); + } catch (error) { + launchReservations.delete(reservation); + feed.append({ schemaVersion: 1, id: operation.id, status: 'failed', + updatedAt: new Date().toISOString(), error: errorMessage(error) }); + throw error; + } + return json(response, 202, operation); + } + const artifactRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/artifacts\/([^/]+)$/); + if (request.method === 'GET' && artifactRoute) { + let artifact; + try { + artifact = resolveCampaignArtifact(resultsRoot, decodeURIComponent(artifactRoute[1] ?? ''), + decodeURIComponent(artifactRoute[2] ?? '')); + } catch { + return json(response, 404, { error: 'Campaign artifact not found.' }); + } + const body = readCampaignArtifactBody(artifact); + const download = url.searchParams.get('download') === '1'; + const type = artifact.kind === 'visual' ? artifact.contentType + : artifact.kind === 'report' && !download ? 'text/html; charset=utf-8' + : 'text/plain; charset=utf-8'; + if (artifact.kind === 'report' && !download) { + response.setHeader('content-security-policy', "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:"); + } + response.writeHead(200, { 'content-type': type, 'content-length': body.length, + 'cache-control': 'no-store', 'content-disposition': `${download ? 'attachment' : 'inline'}; filename*=UTF-8''${encodeURIComponent(basename(artifact.path))}` }); + response.end(body); + return; + } + const campaignRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)(?:\/(.*))?$/); + if (request.method === 'GET' && campaignRoute) { + const key = decodeURIComponent(campaignRoute[1] ?? ''); + const rest = campaignRoute[2] ?? ''; + if (!SAFE_NAME.test(key)) { + return json(response, 400, { error: 'The campaign name is invalid.' }); + } + if (!rest && /^job-[a-f0-9]{64}$/.test(key) + && !existsSync(join(campaignsRoot, key, 'state.json')) + && existsSync(join(resultsRoot, 'jobs', key.slice(4), 'job.json'))) { + const pendingJob = readExecutionJob(resultsRoot, key.slice(4)); + const operation = feed.list().find(op => op.outputName === key); + return json(response, 200, { pendingJob, dispatchError: operation?.status === 'failed' ? operation.error ?? 'Worker exited before campaign startup.' : null }); + } + if (!existsSync(contained(campaignsRoot, key, 'campaign'))) { + return json(response, 404, { error: 'Not found' }); + } + const attemptRoute = rest.match(/^attempts\/([^/]+)\/(checks|package|log|transcript)$/); + const attemptId = attemptRoute ? decodeURIComponent(attemptRoute[1] ?? '') : ''; + if (attemptRoute && !SAFE_NAME.test(attemptId)) { + return json(response, 400, { error: 'The attempt name is invalid.' }); + } + const from = url.searchParams.get('from') ?? '0'; + if (attemptRoute?.[2] === 'log' && (!/^\d+$/.test(from) || !Number.isSafeInteger(Number(from)))) { + return json(response, 400, { error: 'The log offset must be a whole number of bytes.' }); + } + try { + if (!rest) return json(response, 200, await reader.read('campaignLiveSheet', resultsRoot, key)); + if (rest === 'live') return json(response, 200, await reader.read('campaignLiveUpdate', resultsRoot, key)); + if (rest === 'progression') { + const progression = await reader.read('campaignLiveProgression', resultsRoot, key); + return progression + ? json(response, 200, progression) + : json(response, 404, { error: 'Progression is recorded for dependency campaigns only.' }); + } + if (attemptRoute?.[2] === 'transcript') { + const before = url.searchParams.get('before'); + if (before !== null && (!/^\d+$/.test(before) || !Number.isSafeInteger(Number(before)))) { + return json(response, 400, { error: 'Invalid transcript offset' }); + } + return json(response, 200, await reader.read('attemptTranscript', resultsRoot, key, attemptId, + url.searchParams.get('session') ?? '', before === null ? undefined : Number(before))); + } + if (attemptRoute?.[2] === 'checks') { + return json(response, 200, await reader.read('attemptChecks', resultsRoot, key, attemptId)); + } + if (attemptRoute?.[2] === 'package') { + return json(response, 200, await reader.read('attemptPackage', resultsRoot, key, attemptId)); + } + if (attemptRoute) { + const slice = await reader.read('attemptLogSlice', resultsRoot, key, attemptId, Number(from)); + const text = Buffer.from(slice.text); + response.writeHead(200, { 'content-type': 'text/plain; charset=utf-8', + 'content-length': text.length, 'cache-control': 'no-store', + 'x-stack-bench-log-offset': String(slice.offset) }); + response.end(text); + return; + } + } catch (error) { + if (error instanceof Error && error.message === 'campaign attempt does not exist') { + return json(response, 404, { error: 'Not found' }); + } + return json(response, 422, { error: `Cannot read campaign evidence: ${errorMessage(error)}` }); + } + return json(response, 404, { error: 'Not found' }); + } + return json(response, 404, { error: 'Not found' }); + } catch (error) { + return json(response, 500, { error: errorMessage(error) }); + } + }); + // An open event stream is not an idle connection: the watchers stop and the + // streams end as the server closes, not once it has. + const closeServer = server.close.bind(server); + server.close = ((callback?: (error?: Error) => void) => { + reader.close(); + stopEvents(); + for (const listener of listeners) listener.end(); + listeners.clear(); + return closeServer(callback); + }) as typeof server.close; + return { server, token, allowLaunch }; +} + +async function main() { + const args = parseDashboardArgs(process.argv); + const { server, allowLaunch } = createDashboardServer(args); + await new Promise((resolveListen, reject) => { + server.once('error', reject); + server.listen(args.port, args.host, resolveListen); + }); + console.log(`Stack Bench dashboard: http://${args.host}:${args.port} (${allowLaunch ? 'controller' : 'read-only'})`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main().catch(error => { console.error(`stack-bench-dashboard: ${errorMessage(error)}`); process.exitCode = 2; }); +} diff --git a/tools/stack-bench/dashboard/dashboard-transcript.ts b/tools/stack-bench/dashboard/dashboard-transcript.ts new file mode 100644 index 00000000000..51a8459a332 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-transcript.ts @@ -0,0 +1,180 @@ +import { sha256 } from '../src/evidence/provenance.js'; +import { loadTrack, workDirFor } from '../src/composition/tracks.js'; +import { execFile } from 'node:child_process'; +import { open, readdir, realpath, stat } from 'node:fs/promises'; +import { existsSync, realpathSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import { CODING_PROVIDERS } from '../container/coding-providers.js'; +import { CONTAINER_CLAUDE_TRANSCRIPT_READ } from '../container/claude-transcript-reader.js'; +import { AGENT_ADAPTER_REGISTRY } from '../src/agents/agent-adapters.js'; +import type { PublicBackendLease } from '../src/runtime/backend-lease.js'; +import { publicBackendLease, readBackendLease } from '../src/runtime/backend-lease.js'; +import { readArtifactPayload } from '../src/evidence/artifacts.js'; +import { codingContainerAgentExecOptions } from '../src/runtime/coding-container-policy.js'; +import { inspectBuildContainer } from '../src/stacks/hosted-lifecycle.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +export interface TranscriptMessage { id: string; role: string; text: string; tool: boolean } +export interface TranscriptPage { + sessions: Array<{ id: string; label: string }>; + session: string; + before: number | null; + messages: TranscriptMessage[]; + skipped: number; +} +const record = (value: unknown): value is Record => + !!value && typeof value === 'object' && !Array.isArray(value); + +export function transcriptMessages(text: string): { messages: TranscriptMessage[]; skipped: number } { + const messages: TranscriptMessage[] = []; + let skipped = 0; + let eventId = ''; + let blockIndex = 0; + const add = (role: string, value: unknown, tool = false) => { + if (typeof value === 'string' && value.trim()) messages.push({ id: `${eventId}-${blockIndex++}`, role, + text: redactCredentials(value), tool }); + }; + for (const line of text.split('\n').filter(line => line.trim())) { + eventId = sha256(line); + blockIndex = 0; + let event: unknown; + try { event = JSON.parse(line); } catch { skipped++; continue; } + if (!record(event)) continue; + const message = record(event.message) ? event.message + : event.type === 'response_item' && record(event.payload) ? event.payload : null; + if (message) { + const role = String(message.role ?? event.type ?? 'Agent'); + if (typeof message.content === 'string') add(role, message.content); + if (Array.isArray(message.content)) for (const block of message.content) { + if (!record(block)) continue; + if (['text', 'input_text', 'output_text'].includes(String(block.type))) add(role, block.text); + if (block.type === 'tool_use') add(String(block.name ?? 'Tool'), JSON.stringify(block.input, null, 2), true); + if (block.type === 'tool_result') add('Tool result', typeof block.content === 'string' + ? block.content : JSON.stringify(block.content, null, 2), true); + } + if (message.type === 'function_call') add(String(message.name ?? 'Tool'), message.arguments, true); + if (message.type === 'function_call_output') add('Tool result', message.output, true); + } + if (event.type === 'item.completed' && record(event.item)) { + const item = event.item; + if (item.type === 'agent_message') add('assistant', item.text); + if (item.type === 'command_execution') add('Command', `${item.command ?? ''}\n${item.aggregated_output ?? ''}`, true); + if (item.type === 'file_change') add('File changes', JSON.stringify(item.changes, null, 2), true); + } + } + return { messages, skipped }; +} + +export interface TranscriptFile { + id: string; + label: string; + size: number; + modified: number; + read(start: number, count: number): Promise; +} +const pendingReads = new Map>(); +function dockerRead(args: string[]): Promise { + const key = JSON.stringify(args); + const pending = pendingReads.get(key); + if (pending) return pending; + const result = new Promise((resolve, reject) => { + execFile('docker', args, { timeout: 5_000, maxBuffer: 2 * 1024 * 1024, encoding: 'buffer' }, + (error, stdout) => error ? reject(error) : resolve(stdout)); + }).finally(() => pendingReads.delete(key)); + pendingReads.set(key, result); + return result; +} + +export function transcriptLease(directory: string, runtimeRoot = process.env.STACK_BENCH_RUNTIME_DIR + ?? join(tmpdir(), 'stack-bench-runtime')): PublicBackendLease | null { + const evidence = join(directory, 'backend-lease.json'); + if (existsSync(evidence)) return readArtifactPayload(evidence, + { expectedKind: 'backend_lease_evidence' }); + const runPath = join(directory, 'run.json'); + if (!existsSync(runPath)) return null; + const initial = readArtifactPayload<{ backendLease?: PublicBackendLease }>(runPath, + { expectedKind: 'benchmark_run' }).backendLease; + if (!initial) return null; + if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(initial.runId)) throw new Error('Invalid transcript run identity'); + const runtime = resolve(runtimeRoot, initial.runId); + const path = join(runtime, 'backend-lease.json'); + if (!existsSync(path)) return null; + if (dirname(realpathSync(runtime)) !== realpathSync(resolve(runtimeRoot)) + || dirname(realpathSync(path)) !== realpathSync(runtime)) throw new Error('Transcript lease is outside runtime directory'); + const current = publicBackendLease(readBackendLease(path, { runId: initial.runId, backend: initial.backend, active: true })); + if (current.ownership.markerSha256 !== initial.ownership?.markerSha256 + || current.track !== initial.track || current.runIndex !== initial.runIndex) { + throw new Error('Transcript lease ownership changed'); + } + return current; +} + +// Reads only this attempt's transcript mounts. Never scans another account's sessions. +export async function attemptTranscriptFiles(executions: Array<{ directory: string; label: string }>, + adapterId: string): Promise { + const provider = AGENT_ADAPTER_REGISTRY.get(adapterId).provider; + if (!provider || !(provider in CODING_PROVIDERS)) return []; + const config = CODING_PROVIDERS[provider as keyof typeof CODING_PROVIDERS]; + const files: TranscriptFile[] = []; + for (const execution of executions) { + const lease = transcriptLease(execution.directory); + if (!lease) continue; + const root = config.projects(join(workDirFor(loadTrack(lease.track), lease.backend, lease.runIndex, lease.runId), 'app')); + let remote: ((name: string, start: number, count: number) => Promise) | null = null; + if (lease.state === 'active' && lease.resources.buildContainer?.owned) { + const actual = await dockerRead(['inspect', '--format', '{{.Id}}', lease.resources.buildContainer.name]); + const container = inspectBuildContainer(lease, () => actual.toString()); + remote = (name, start, count) => dockerRead(['exec', ...codingContainerAgentExecOptions(), + container.id, 'node', '-e', CONTAINER_CLAUDE_TRANSCRIPT_READ, + config.containerTranscripts, name, String(start), String(count)]); + } + let entries: Array<[string, number, number]>; + if (remote) entries = JSON.parse((await remote('', 0, 0)).toString()) as Array<[string, number, number]>; + else { + if (!existsSync(root)) continue; + const resolvedRoot = await realpath(root); + entries = []; + for (const entry of await readdir(root, { recursive: true, withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith('.jsonl')) continue; + const path = join(entry.parentPath, entry.name); + if (!(await realpath(path)).startsWith(resolvedRoot + sep)) continue; + const info = await stat(path); + entries.push([relative(root, path), info.size, info.mtimeMs]); + } + } + for (const [name, size, modified] of entries) { + const reader = remote; + files.push({ id: Buffer.from(`${execution.label}/${name}`).toString('base64url'), + label: `${execution.label} / ${new Date(modified).toISOString().replace('T', ' ').slice(0, 16)} UTC`, size, modified, + read: async (start, count) => { + if (!Number.isSafeInteger(start) || start < 0 || !Number.isSafeInteger(count) + || count < 0 || count > 256 * 1024) throw new Error('Invalid transcript range'); + if (reader) return reader(name, start, count); + const path = join(root, name); + if (!(await realpath(path)).startsWith(await realpath(root) + sep)) { + throw new Error('transcript is outside the attempt directory'); + } + const fd = await open(path, 'r'), buffer = Buffer.alloc(count); + try { return buffer.subarray(0, (await fd.read(buffer, 0, count, start)).bytesRead); } + finally { await fd.close(); } + } }); + } + } + return files.sort((a, b) => a.modified - b.modified); +} +export async function readAttemptTranscript(executions: Array<{ directory: string; label: string }>, + adapterId: string, session = '', before?: number): Promise { + const files = await attemptTranscriptFiles(executions, adapterId); + const file = (session ? files.find(file => file.id === session) : files.at(-1)); + if (session && !file) throw new Error('Transcript session not found'); + if (!file) return { sessions: [], session: '', before: null, messages: [], skipped: 0 }; + const end = Math.min(before ?? file.size, file.size); + const start = Math.max(0, end - 256 * 1024); + const bytes = await file.read(start, end - start); + const first = start ? bytes.indexOf(10) + 1 : 0; + const last = bytes.lastIndexOf(10); + const content = last >= first ? bytes.subarray(first, last + 1).toString('utf8') : ''; + return { sessions: files.map(({ id, label }) => ({ id, label })), session: file.id, + before: start > 0 ? start + first : null, ...transcriptMessages(content) }; +} diff --git a/tools/stack-bench/dashboard/dashboard-views.ts b/tools/stack-bench/dashboard/dashboard-views.ts new file mode 100644 index 00000000000..eb4dae113c2 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-views.ts @@ -0,0 +1,1068 @@ +import { readAttemptTranscript } from './dashboard-transcript.js'; +import { liveCostTotal, liveTranscriptCost } from './dashboard-live-cost.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { sha256 } from '../src/evidence/provenance.js'; +import { closeSync, existsSync, fstatSync, openSync, readFileSync, readSync, readdirSync, statSync } + from 'node:fs'; +import { basename, join, resolve } from 'node:path'; + +import type { CampaignAttemptState } from '../src/campaigns/campaign-scheduler.js'; +import type { DependencyPromptSelection, DependencyState } + from '../src/progression/dependency-mode.js'; +import type { ProgressionState } from '../src/progression/progression-state.js'; +import type { CompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js'; +import type { DependencyProgress } from '../src/campaigns/campaign-inspection.js'; +import type { GradeBundlePayload } from '../src/evidence/benchmark-run.js'; +import { executionSpend } from '../src/campaigns/campaign-report.js'; +import type { CostEvidence } from '../src/evidence/cost-proof.js'; +import type { RunCheckpoint } from '../src/evidence/run-checkpoints.js'; +import { scoreDependencyState, dependencyCompletionBreakdown, type DependencyCompletionBreakdown } from '../src/progression/dependency-score.js'; +import type { CheckCompletion } from '../src/evidence/check-completion.js'; +import { ARTIFACT_FILE, readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { CAMPAIGN_FILE } from '../src/campaigns/campaign-path.js'; +import { campaignFacts, inspectCampaignAttempt } from '../src/campaigns/campaign-inspection.js'; +import { campaignLockIsActive, readCampaignLock } from '../src/campaigns/campaign-lock.js'; +import { campaignProgressionOwner } from '../src/campaigns/campaign-compiler.js'; +import { compileProgressionInput, dependencyRuntimeDefinition } + from '../src/progression/progression-definition.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { readProgressionState } from '../src/progression/progression-state.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { CHECK_EVIDENCE_STATUSES, evidenceDisposition, type CheckEvidence } from '../src/evidence/check-evidence.js'; +import { repairBudgetLimit, type RepairBudget } from '../src/progression/repair-plan.js'; +import { MAX_LOG_BYTES, contained, parseRunProgress, readTextTail, attemptPause, + walkPublicExecutionArtifacts } from './dashboard-model.js'; +import type { DashboardArtifact } from './dashboard-model.js'; +import { attemptExcluded, attemptMetrics, attemptStalling, compareCampaign, median } + from './public/metrics.js'; + +const CAMPAIGN_KEY = /^[a-z0-9][a-z0-9.-]*$/; +const GRADE_DIRECTORY = /^(?:first-build-l(\d+)-grading|l(\d+)-fix(\d+)-grading|grading)$/i; +const PROGRESSION_ATTEMPT = /^attempt-(\d+)$/i; +const LOG_FILE = 'process.stdout.log'; + +type ControllerActive = (directory: string, campaign: CompiledCampaignPlan) => boolean; +type InspectedAttempt = ReturnType; + +interface ViewOptions { + controllerActive?: ControllerActive; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function percentage(value: number | null | undefined): number | null { + return value == null ? null : Math.round(value * 1000) / 10; +} + +function campaignDirectory(resultsRoot: string, key: string): string { + if (!CAMPAIGN_KEY.test(key)) throw new Error('campaign key is invalid'); + return contained(join(resolve(resultsRoot), 'campaigns'), key, 'campaign'); +} + +function fileFingerprint(path: string): string | null { + if (!existsSync(path)) return null; + const stat = statSync(path); + return `${stat.size}:${stat.mtimeMs}`; +} + +const campaignStateCache = new Map; +}>(); + +// Share frozen plan/state validation across dashboard resources. Execution +// evidence and controller liveness keep their own freshness checks. +function dashboardCampaignState(directory: string): ReturnType { + const fingerprint = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state] + .map(file => fileFingerprint(join(directory, file))).join('|'); + const cached = campaignStateCache.get(directory); + if (cached?.fingerprint === fingerprint) return cached.value; + const value = readCampaignState(directory, { requireCurrentInputs: false }); + campaignStateCache.set(directory, { fingerprint, value }); + return value; +} + +// Every file whose change can move a number in the view, and nothing else: a +// running campaign that has written nothing since the last read is unchanged. +function executionFingerprints(directory: string, files: readonly string[]): string[] { + const attemptsRoot = join(directory, 'attempts'); + if (!existsSync(attemptsRoot)) return []; + const parts: string[] = []; + for (const attempt of readdirSync(attemptsRoot, { withFileTypes: true })) { + if (!attempt.isDirectory()) continue; + const attemptDirectory = join(attemptsRoot, attempt.name); + for (const execution of readdirSync(attemptDirectory, { withFileTypes: true })) { + if (!execution.isDirectory()) continue; + for (const file of files) { + const stamp = fileFingerprint(join(attemptDirectory, execution.name, file)); + if (stamp) parts.push(`${attempt.name}/${execution.name}/${file}:${stamp}`); + } + } + } + return parts.sort(); +} + +function campaignFingerprint(directory: string, files: readonly string[], + executionFiles: readonly string[]): string { + return [...files.map(file => `${file}:${fileFingerprint(join(directory, file)) ?? 'missing'}`), + ...executionFingerprints(directory, executionFiles)].join('|'); +} + +// Overview + +export interface OverviewCampaign { + key: string; + id: string; + title: string; + status: string; + mode: string; + levels: number[]; + repetitions: number; + updatedAt: string | null; + // The mode's official score per stack, as a percentage; null until a stack + // has a comparable result. + scores: Record; + attempts: { total: number; running: number; completed: number }; +} + +export interface UnreadableOverviewCampaign { + key: string; + id: string; + title: string; + status: 'unreadable'; + error: string; +} + +export type OverviewEntry = OverviewCampaign | UnreadableOverviewCampaign; + +const overviewCache = new Map(); + +function overviewCampaign(directory: string): { + plan: CompiledCampaignPlan; + campaign: OverviewCampaign; +} { + const { plan, state } = dashboardCampaignState(directory); + // Only completed attempts can contribute to these scores. Live details and + // excluded evidence are read when their campaign is opened. + const attempts = state.attempts.filter(attempt => attempt.status === 'completed').map(attempt => + inspectCampaignAttempt(plan, attempt, directory)); + const comparison = compareCampaign({ attempts }); + const scores = Object.fromEntries(plan.stacks.map(stack => + [stack.id, percentage(comparison.rows.find(row => row.stack === stack.id)?.final ?? null)])); + return { + plan, + campaign: { + key: basename(resolve(directory)), + id: plan.id, + title: plan.title, + status: state.status, + mode: plan.definition.mode?.id ?? 'sequential', + levels: plan.definition.levels, + repetitions: plan.definition.repetitions, + // Current qualification is checked on the campaign sheet. Listing old + // runs must not recompile today's qualification for every history row. + updatedAt: state.updatedAt, + scores, + attempts: { total: state.summary.total, running: state.summary.running, + completed: state.summary.completed }, + }, + }; +} + +// Liveness is one more fact about a running campaign, not a condition of +// reading it: the read-only host view has no Docker socket to ask. +function controllerInterrupted(probe: ControllerActive, directory: string, + plan: CompiledCampaignPlan, status: string): boolean { + if (status !== 'running') return false; + try { return !probe(directory, plan); } catch { return false; } +} + +function withInterruption(campaign: OverviewCampaign, interrupted: boolean): OverviewCampaign { + if (!interrupted) return campaign; + return { ...campaign, status: 'attention-required', + attempts: { ...campaign.attempts, running: 0 } }; +} + +// Summaries only: no attempt list, no log, no plan. The fingerprint covers a +// running campaign too, so a poll that finds nothing changed costs one stat +// per evidence file instead of a full replay. +export function overviewSummary(campaignsRoot: string, + { controllerActive = campaignLockIsActive, keys }: ViewOptions & { keys?: readonly string[] } = {}): OverviewEntry[] { + if (!existsSync(campaignsRoot)) return []; + const campaigns: OverviewEntry[] = []; + for (const entry of readdirSync(campaignsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + if (keys && !keys.includes(entry.name)) continue; + const directory = join(campaignsRoot, entry.name); + if (!existsSync(join(directory, CAMPAIGN_FILE.state)) + || !existsSync(join(directory, CAMPAIGN_FILE.plan))) continue; + try { + const fingerprint = campaignFingerprint(directory, + [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state], [ARTIFACT_FILE.run]); + const cached = overviewCache.get(directory); + const fresh = cached?.fingerprint === fingerprint + ? { plan: cached.plan, campaign: cached.campaign } : overviewCampaign(directory); + overviewCache.set(directory, { fingerprint, ...fresh }); + campaigns.push(withInterruption(fresh.campaign, controllerInterrupted(controllerActive, + directory, fresh.plan, fresh.campaign.status))); + } catch (error) { + campaigns.push({ key: entry.name, id: entry.name, title: entry.name, + status: 'unreadable', error: errorMessage(error) }); + } + } + return campaigns.sort((left, right) => + String('updatedAt' in right ? right.updatedAt ?? '' : '') + .localeCompare(String('updatedAt' in left ? left.updatedAt ?? '' : ''))); +} + +export type CampaignFilter = 'all' | 'attention' | 'completed' | 'ready'; + +export function overviewPage(campaignsRoot: string, requestedPage = 1, + filter: CampaignFilter = 'all', options: ViewOptions = {}) { + const index: Array<{ key: string; status: string; updatedAt: string }> = []; + for (const entry of existsSync(campaignsRoot) ? readdirSync(campaignsRoot, { withFileTypes: true }) : []) { + if (!entry.isDirectory()) continue; + const directory = join(campaignsRoot, entry.name); + if (!existsSync(join(directory, CAMPAIGN_FILE.plan)) || !existsSync(join(directory, CAMPAIGN_FILE.state))) continue; + try { + const { plan, state } = dashboardCampaignState(directory); + const interrupted = controllerInterrupted(options.controllerActive ?? campaignLockIsActive, directory, plan, state.status); + index.push({ key: entry.name, status: interrupted ? 'attention-required' : state.status, updatedAt: state.updatedAt }); + } catch { index.push({ key: entry.name, status: 'unreadable', updatedAt: '' }); } + } + index.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt) || a.key.localeCompare(b.key)); + const matches = (status: string, selection: CampaignFilter): boolean => selection === 'all' + || (selection === 'attention' ? ['attention-required', 'unreadable'].includes(status) + : status === (selection === 'ready' ? 'prepared' : 'completed')); + const counts = Object.fromEntries((['all', 'attention', 'completed', 'ready'] as const) + .map(selection => [selection, index.filter(item => matches(item.status, selection)).length])) as Record; + const selected = index.filter(item => matches(item.status, filter)); + const pageSize = 20; + const pages = Math.max(1, Math.ceil(selected.length / pageSize)); + const page = Math.min(Math.max(1, requestedPage), pages); + const keys = selected.slice((page - 1) * pageSize, page * pageSize).map(item => item.key); + const summaries = new Map(overviewSummary(campaignsRoot, { ...options, keys }).map(item => [item.key, item])); + return { campaigns: keys.map(key => summaries.get(key)!).filter(Boolean), page, pages, + total: selected.length, pageSize, counts, running: index.filter(item => item.status === 'running').map(item => item.key) }; +} + +export type OverviewPage = ReturnType; + +// Campaign sheet + +export interface SheetFacts { + mode: string; + workSelection: string | null; + repairSelection: string | null; + repairLimits: RepairBudget; + agent: string | null; + model: string | null; + guidance: string | null; + productionQuality: boolean | null; + recipes: Array<{ level: number; id: string | null; contentSha256: string | null }>; + timeLimitMinutes: number; + spendLimitUsd: number | null; + controllerImage: string | null; + buildImage: string | null; + planSha256: string; + grading: string; + gradingReasons: string[]; +} + +export interface ClimbPoint { + score: number; + max: number; + level: number | null; + unaided: boolean; +} + +export interface SheetAttempt { + model?: string; + effort?: string; + liveSpend?: number; + id: string; + repetition: number; + status: string; + phase: string; + stalling: boolean; + excluded: string | null; + continued: boolean; + logUpdatedAt: string | null; + activityUpdatedAt?: string | null; + paused?: boolean; + score: number | null; + unaided: number | null; + repairs: { used: number; budget: number }; + timeSec: number | null; + executionStartedAt: string | null; + executionCompletedAt: string | null; + executionCost?: CostEvidence; + spend: CostEvidence; + spendPending: boolean; + completion: CheckCompletion | null; + featureCompletion?: DependencyCompletionBreakdown['featureCompletion'] | null; + checkCategories?: DependencyCompletionBreakdown['checkCategories'] | null; + variant: string; + climb: ClimbPoint[]; +} + +export interface SheetLevel { + level: number; + unaided: { score: number; max: number } | null; + score: { score: number; max: number } | null; + repairs: number; +} + +export interface SheetQuestline { + id: string; + title: string; + score: number | null; + nodes: Array<{ id: string; status: string }>; +} + +export interface SheetStack { + liveSpend?: number; + stack: string; + costPerValidRun: number | null; + selectedAttemptId: string | null; + score: number | null; + points: { score: number; max: number } | null; + unaided: number | null; + continued: boolean; + regressions: number | null; + timeSec: number | null; + spend: CostEvidence; + spendPending: boolean; + completionRate: number | null; + n: number; + attempts: SheetAttempt[]; + levels: SheetLevel[] | null; + questlines: SheetQuestline[] | null; +} + +export interface CampaignSheet { + key: string; + id: string; + title: string; + status: string; + mode: string; + levels: number[]; + repetitions: number; + provisional: boolean; + mixedScope: boolean; + executions: number; + // A dependency campaign that stopped between executions is the one thing an + // operator can restart; the server checks the same three facts again. + resumable: boolean; + controllerOwner?: string | null; + reportFiles?: string[]; + createdAt: string; + updatedAt: string; + facts: SheetFacts; + stacks: SheetStack[]; +} + +interface SheetAttemptView { + inspected: InspectedAttempt; + attempt: SheetAttempt; +} + +function sheetFacts(plan: CompiledCampaignPlan): SheetFacts { + const mode = plan.definition.mode; + const policy = plan.dependencyPolicy?.definition ?? null; + const agent = plan.agents[0] ?? null; + const facts = campaignFacts(plan); + return { + mode: mode.id, + workSelection: policy?.workSelection ?? mode.workSelection ?? null, + repairSelection: policy?.repair.selection ?? plan.definition.repair.selection, + repairLimits: plan.definition.repair.budget, + agent: agent?.adapter ?? null, + model: agent?.model ?? null, + guidance: plan.attempts[0]?.guidance ?? null, + productionQuality: plan.attempts.every(attempt => attempt.condition.productionQuality === true) + ? true : plan.attempts.some(attempt => attempt.condition.productionQuality === true) ? null : false, + recipes: facts.recipes, + timeLimitMinutes: plan.definition.budgets.attemptTimeoutMinutes, + spendLimitUsd: plan.definition.budgets.maxCostUsdPerAttempt, + controllerImage: facts.runtime.controllerImage, + buildImage: facts.runtime.buildImage, + planSha256: plan.contentSha256, + grading: gradingStatus(facts.grading), + gradingReasons: [...new Set(facts.grading.levels.flatMap(level => level.reasons ?? []))], + }; +} + +// A campaign whose levels disagree is partly publishable and says so. +function gradingStatus(grading: ReturnType['grading']): string { + const levels = new Set(grading.levels.map(level => level.status)); + return levels.size > 1 ? 'partial' : grading.status; +} + +function dependencyRepairs(plan: CompiledCampaignPlan, + dependency: DependencyProgress): { used: number; budget: number } { + return { + used: dependency.history?.repairAttempts ?? 0, + budget: repairBudgetLimit(plan.definition.repair, { + features: dependency.nodes.length, + depths: plan.definition.levels.length, + }), + }; +} + +function attemptRegressions(attempt: InspectedAttempt): number { + if (attempt.dependency) return attempt.dependency.regressions ?? 0; + return attempt.result?.regressions ?? 0; +} + +// Continued: the attempt resumed on a repair grant, so its first grade is a +// checkpoint baseline rather than an unaided build. +function attemptContinued(attempt: InspectedAttempt): boolean { + if (attempt.dependency) { + return attempt.dependency.attempts.features.some(feature => + typeof feature.granted === 'number' && feature.granted > 0); + } + return (attempt.result?.levels ?? []).some(level => level.continued); +} + +function sheetLevels(attempt: InspectedAttempt | null): SheetLevel[] { + return (attempt?.result?.levels ?? []).map(level => ({ + level: level.level, + unaided: level.firstAbort ? null : level.firstScore, + score: level.finalScore, + repairs: level.used, + })); +} + +function sheetQuestlines(dependency: DependencyProgress): SheetQuestline[] { + const status = new Map(dependency.nodes.map(node => [node.id, node.status])); + const scored = new Map((dependency.score?.questlines ?? []) + .map(questline => [questline.id, questline.percentage ?? null])); + return (dependency.questlines ?? []).map(questline => ({ + id: questline.id, + title: questline.title, + score: scored.get(questline.id) ?? null, + nodes: questline.nodes.map(id => ({ id, status: status.get(id) ?? 'locked' })), + })); +} + +function sheetAttemptView(plan: CompiledCampaignPlan, state: CampaignAttemptState, + directory: string, interrupted: boolean): SheetAttemptView { + const inspected = inspectCampaignAttempt(plan, state, directory); + const execution = inspected.execution; + const logPath = execution + ? join(contained(directory, execution.output, 'campaign execution'), LOG_FILE) : null; + const log = logPath ? readTextTail(logPath) : ''; + const logUpdatedAt = logPath && existsSync(logPath) + ? new Date(statSync(logPath).mtimeMs).toISOString() : null; + const running = inspected.status === 'running' && !interrupted; + const pause = attemptPause(plan, state, directory); + const paused = running && pause?.resumedAt === null; + const repairLimit = repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }); + const progress = parseRunProgress(log, { repairs: repairLimit, + running, status: inspected.status, + dependency: plan.definition.mode?.id === 'dependency' }); + const metrics = attemptMetrics({ ...inspected, logUpdatedAt }); + const repairs = inspected.dependency ? dependencyRepairs(plan, inspected.dependency) : null; + return { + inspected, + attempt: { + id: inspected.id, + repetition: inspected.repetition, + status: interrupted && inspected.status === 'running' ? 'interrupted' : inspected.status, + phase: interrupted && inspected.status === 'running' + ? 'Controller stopped before completion' : paused ? `Paused at L${pause.depth}` : progress.phase, + stalling: attemptStalling({ ...inspected, paused }), + paused, + excluded: attemptExcluded(inspected), + continued: attemptContinued(inspected), + logUpdatedAt, + score: percentage(metrics?.final ?? null), + unaided: percentage(metrics?.first ?? null), + repairs: repairs ?? { used: metrics?.repairs ?? 0, budget: repairLimit }, + timeSec: metrics?.duration ?? null, + executionStartedAt: execution?.startedAt ?? null, + executionCompletedAt: execution?.completedAt ?? null, + executionCost: inspected.cost, + spend: inspected.spend, + spendPending: inspected.status === 'running' || inspected.status === 'pending', + completion: inspected.completion, + featureCompletion: inspected.dependency?.featureCompletion ?? null, + checkCategories: inspected.dependency?.checkCategories ?? null, + model: inspected.model, + effort: plan.attempts.find(entry => entry.id === inspected.id)?.effort, + variant: inspected.variantLabel, + climb: progress.series, + }, + }; +} + +const sheetCache = new Map(); + +// Facts and per-stack figures. No log text and no package walk: the climb and +// the phase come from the run output the controller already writes. +export function campaignSheet(resultsRoot: string, key: string, + { controllerActive = campaignLockIsActive }: ViewOptions = {}): CampaignSheet { + const directory = campaignDirectory(resultsRoot, key); + const reportPaths = ['report/report.html', 'report/export-manifest.json']; + const fingerprint = campaignFingerprint(directory, [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state, ...reportPaths], + [ARTIFACT_FILE.run, ARTIFACT_FILE.progressionState, 'depth-pause.json']); + const { plan, state } = dashboardCampaignState(directory); + const interrupted = controllerInterrupted(controllerActive, directory, plan, state.status); + const controllerOwner = readCampaignLock(directory)?.ownershipMarkerSha256 ?? null; + const cacheKey = `${directory}:${interrupted ? 'interrupted' : 'live'}:${controllerOwner ?? ''}`; + const cached = sheetCache.get(cacheKey); + if (cached?.fingerprint === fingerprint) return cached.sheet; + const views = state.attempts.map(attempt => + sheetAttemptView(plan, attempt, directory, interrupted)); + const comparison = compareCampaign({ + attempts: views.map(view => view.inspected) }); + const dependency = plan.definition.mode?.id === 'dependency'; + const stacks = plan.stacks.map(stack => { + const owned = views.filter(view => view.inspected.stack === stack.id); + const row = comparison.rows.find(entry => entry.stack === stack.id); + const eligible = row?.scopes.length === 1 ? row.runs.map(run => run.attempt) : []; + // Figures a repetition cannot average — the climb, the questline board, the + // per-level rows — come from the newest attempt that actually ran. + const latest = owned.findLast(view => view.inspected.execution !== null) ?? null; + const lead = latest?.inspected ?? null; + const metrics = lead ? attemptMetrics(lead) : null; + return { + stack: stack.id, + costPerValidRun: row?.costPerValidRun ?? null, + selectedAttemptId: latest?.attempt.id ?? null, + score: percentage(row?.final ?? null), + points: dependency ? uniquePoints(lead?.dependency ?? null) : metrics?.raw.final ?? null, + unaided: percentage(row?.first ?? null), + continued: owned.some(view => view.attempt.continued), + regressions: median(eligible.map(attemptRegressions)), + timeSec: row?.duration ?? null, + spend: executionSpend(owned.map(view => ({ cost: view.inspected.spend, knownCostUsd: view.inspected.spend.knownCostUsd }))), + spendPending: owned.some(view => view.attempt.spendPending), + completionRate: eligible.every(attempt => attempt.completion?.rate != null) + ? median(eligible.flatMap(attempt => attempt.completion?.rate == null + ? [] : [attempt.completion.rate])) : null, + n: row?.n ?? 0, + attempts: owned.map(view => view.attempt), + levels: dependency ? null : sheetLevels(lead), + questlines: lead?.dependency ? sheetQuestlines(lead.dependency) : null, + }; + }); + const sheet: CampaignSheet = { + key: basename(resolve(directory)), + id: plan.id, + title: plan.title, + status: interrupted ? 'attention-required' : state.status, + mode: plan.definition.mode?.id ?? 'sequential', + levels: plan.definition.levels, + repetitions: plan.definition.repetitions, + provisional: campaignFacts(plan).grading.status !== 'qualified', + mixedScope: comparison.mixedScope, + executions: state.summary.executions, + resumable: dependency && state.status === 'prepared' && state.summary.executions > 0, + controllerOwner: interrupted ? null : controllerOwner, + reportFiles: reportPaths.filter(path => existsSync(join(directory, path)) && statSync(join(directory, path)).isFile()), + createdAt: state.createdAt, + updatedAt: state.updatedAt, + facts: sheetFacts(plan), + stacks, + }; + sheetCache.set(cacheKey, { fingerprint, sheet }); + return sheet; +} + +function uniquePoints(dependency: DependencyProgress | null): { score: number; max: number } | null { + const unique = dependency?.score?.uniqueChecks; + if (!unique || unique.passedPoints == null || unique.availablePoints == null) return null; + return { score: unique.passedPoints, max: unique.availablePoints }; +} + +// Attempt sub-resources + +function attemptState(directory: string, attemptId: string): CampaignAttemptState { + const { state } = dashboardCampaignState(directory); + const attempt = state.attempts.find(item => item.plan.id === attemptId); + if (!attempt) throw new Error('campaign attempt does not exist'); + return attempt; +} + +export interface AttemptCheckGrade { + id: string; + level: number | null; + round: number | null; + score: { score: number; max: number } | null; + error?: string; +} + +export interface AttemptCheck { + category?: 'feature' | 'production' | 'interface' | null; + key: string; + id: string; + description: string; + points: number; + feature: string; + outcome: string; + regressed: boolean; + history: string[]; + observations: Array<{ status: string; summary: string | null; expected: string | null; actual: string | null } | null>; +} + +export interface AttemptChecks { + attemptId: string; + stack: string; + grades: AttemptCheckGrade[]; + checks: AttemptCheck[]; +} + +function checkOutcome(evidence: unknown): string { + const status = evidence !== null && typeof evidence === 'object' && 'status' in evidence + ? (evidence as { status?: unknown }).status : null; + if (status === 'passed') return 'pass'; + if (status === 'failed') return 'fail'; + return 'not-run'; +} + +function checkObservation(value: unknown): AttemptCheck['observations'][number] { + if (!value || typeof value !== 'object') return null; + const evidence = value as Partial; + if (!evidence.status || !CHECK_EVIDENCE_STATUSES.includes(evidence.status)) return null; + const status = evidenceDisposition(evidence.status).label; + if (evidence.sensitivity?.length) return { status, summary: 'Sensitive evidence omitted.', expected: null, actual: null }; + const text = (item: unknown): string | null => { + if (item == null) return null; + const result = redactCredentials(typeof item === 'string' ? item : JSON.stringify(item, null, 2)); + return result.length <= 12_000 ? result : `${result.slice(0, 12_000)}\n[Truncated. Full evidence is in Files.]`; + }; + return { status, summary: text(evidence.summary), expected: text(evidence.expected), actual: text(evidence.observation) }; +} + +function gradeDirectories(executionDirectory: string): AttemptCheckGrade[] { + if (!existsSync(executionDirectory)) return []; + const progression = join(executionDirectory, 'progression'); + if (existsSync(progression)) { + return readdirSync(progression, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && PROGRESSION_ATTEMPT.test(entry.name)) + .map(entry => ({ id: `progression/${entry.name}`, + level: null, round: Number(PROGRESSION_ATTEMPT.exec(entry.name)?.[1] ?? 0), score: null })) + .sort((left, right) => (left.round ?? 0) - (right.round ?? 0)); + } + return readdirSync(executionDirectory, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && GRADE_DIRECTORY.test(entry.name)) + .map(entry => { + const match = GRADE_DIRECTORY.exec(entry.name); + const level = match?.[1] ?? match?.[2] ?? null; + return { id: entry.name, level: level === null ? null : Number(level), + round: match?.[3] === undefined ? 0 : Number(match[3]), score: null }; + }) + // The final `grading` directory has no level and comes after every level's grades. + .sort((left, right) => (left.level ?? Infinity) - (right.level ?? Infinity) + || (left.round ?? 0) - (right.round ?? 0)); +} + +// Per-check outcome and the history of every grade that reported it: the +// question "did this ever pass" has no other answer in the evidence. +export function attemptChecks(resultsRoot: string, key: string, attemptId: string): AttemptChecks { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + const execution = attempt.executions.at(-1) ?? null; + if (!execution) return { attemptId, stack: attempt.plan.stack, grades: [], checks: [] }; + const executionDirectory = contained(directory, execution.output, 'campaign execution'); + const grades = gradeDirectories(executionDirectory); + const { plan } = dashboardCampaignState(directory); + const metadata = new Map(plan.featureCatalog?.definition.nodes.flatMap(node => + node.gradingChecks.map(check => [check.id, check] as const)) ?? []); + const checks = new Map(); + grades.forEach((grade, index) => { + const path = join(executionDirectory, grade.id, ARTIFACT_FILE.gradeBundle); + if (!existsSync(path)) { + grade.error = 'grade bundle is missing'; + return; + } + let payload; + try { + payload = readArtifactPayload(path, { expectedKind: 'grade_bundle' }); + } catch (error) { + grade.error = redactCredentials(errorMessage(error)); + return; + } + grade.score = payload.totals?.score == null || payload.totals.max == null + ? null : { score: payload.totals.score, max: payload.totals.max }; + for (const suite of Object.values(payload.suites ?? {})) { + for (const feature of suite.features ?? []) { + for (const criterion of feature.criteria ?? []) { + const stableKey = criterion.stableKey ?? `${feature.name ?? ''}.${criterion.id ?? ''}`; + const entry = checks.get(stableKey) ?? { key: stableKey, id: criterion.id ?? stableKey, + description: criterionDescription(criterion), points: criterion.points ?? 0, + category: metadata.get(stableKey)?.category ?? null, + feature: feature.name ?? '', outcome: 'not-run', regressed: false, + history: grades.map(() => 'not-run'), observations: grades.map(() => null) }; + entry.history[index] = checkOutcome(criterion.evidence); + entry.observations[index] = checkObservation(criterion.evidence); + checks.set(stableKey, entry); + } + } + } + }); + for (const check of checks.values()) { + const conclusive = check.history.filter(outcome => outcome !== 'not-run'); + check.outcome = conclusive.at(-1) ?? 'not-run'; + check.regressed = conclusive.some((outcome, index) => + outcome === 'fail' && conclusive.slice(0, index).includes('pass')); + } + return { attemptId, stack: attempt.plan.stack, grades, checks: [...checks.values()] }; +} + +// The grade bundle names the criterion text `desc`. +function criterionDescription(criterion: object): string { + const record = criterion as { desc?: unknown; description?: unknown }; + if (typeof record.desc === 'string') return record.desc; + return typeof record.description === 'string' ? record.description : ''; +} + +export interface AttemptPackage { + attemptId: string; + stack: string; + executions: Array<{ + executionId: string; + ordinal: number; + status: string; + artifacts: DashboardArtifact[]; + visuals: DashboardArtifact[]; + truncated: boolean; + }>; +} + +export function attemptPackage(resultsRoot: string, key: string, + attemptId: string): AttemptPackage { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + return { + attemptId, + stack: attempt.plan.stack, + executions: attempt.executions.map(execution => { + const scanned = walkPublicExecutionArtifacts(directory, + contained(directory, execution.output, 'campaign execution')); + return { executionId: execution.id, ordinal: execution.ordinal, status: execution.status, + artifacts: scanned.artifacts, + visuals: scanned.artifacts.filter(artifact => artifact.kind === 'visual'), + truncated: scanned.truncated }; + }), + }; +} + +export interface AttemptLogSlice { + attemptId: string; + from: number; + offset: number; + size: number; + text: string; +} + +// Bytes after an offset, so a following view pays for growth rather than for +// the whole log on every poll. +export function attemptLogSlice(resultsRoot: string, key: string, attemptId: string, + fromOffset = 0): AttemptLogSlice { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + const execution = attempt.executions.at(-1) ?? null; + const path = execution + ? join(contained(directory, execution.output, 'campaign execution'), LOG_FILE) : null; + if (!path || !existsSync(path)) { + return { attemptId, from: fromOffset, offset: 0, size: 0, text: '' }; + } + const descriptor = openSync(path, 'r'); + try { + const size = fstatSync(descriptor).size; + // A rotated or truncated log invalidates the caller's offset. + const start = Math.min(Math.max(0, fromOffset), size); + const length = Math.min(size - start, MAX_LOG_BYTES); + const buffer = Buffer.alloc(length); + if (length) readSync(descriptor, buffer, 0, length, start); + return { attemptId, from: fromOffset, offset: start + length, size, + text: redactCredentials(buffer.toString('utf8')) }; + } finally { + closeSync(descriptor); + } +} + +// Campaign progression + +export interface ProgressionCatalogNode { + id: string; + title: string; + questline: string; + depth: number; + dependencies: string[]; +} + +export interface ProgressionStep { + sequence: number; + completedAt?: string | null; + completion?: number | null; + featureCompletion?: number | null; + action: 'build' | 'repair' | 'grant'; + targets: string[]; + // Node status after the event, index-aligned with `nodes`. + statuses: string[]; + score: number | null; + repairs: number; +} + +export interface ProgressionTrack { + liveCosts?: Array<{ completedAt: string; costUsd: number }>; + stack: string; + attemptId: string; + updatedAt: string; + steps: ProgressionStep[]; + costs?: Array<{ completedAt: string; cost: CostEvidence }>; +} + +export interface CampaignProgression { + key: string; + depths: number[]; + questlines: Array<{ id: string; title: string; nodes: string[] }>; + nodes: ProgressionCatalogNode[]; + stacks: ProgressionTrack[]; +} + +function progressionSnapshot(state: ProgressionState, nodeIds: readonly string[]): { + statuses: string[]; + score: number | null; + repairs: number; +} { + const average = progressionEngine.score(state).questlineAveragePercentage; + return { + statuses: nodeIds.map(id => state.nodes[id]?.status ?? 'locked'), + score: average == null ? null : Math.round(average * 10) / 10, + repairs: state.attempts.filter(attempt => attempt.repair !== undefined).length, + }; +} + +function progressionSteps(state: DependencyState, nodeIds: readonly string[], times: Map): ProgressionStep[] { + let replay = progressionEngine.initialize(state.definition); + return state.events.map(event => { + const action = progressionEngine.nextAction(replay); + if (event.type === 'repairs-granted') { + replay = progressionEngine.grantRepairs(replay, event.grant); + return { sequence: event.sequence, action: 'grant' as const, + targets: [...event.grant.nodeIds], ...progressionSnapshot(replay, nodeIds) }; + } + const targets = action.type === 'terminal' + ? [] : [...(action.prompt as DependencyPromptSelection).nodeIds]; + const repair = action.type === 'repair'; + replay = progressionEngine.recordResult(replay, event.result); + return { sequence: event.sequence, action: repair ? 'repair' as const : 'build' as const, + targets, ...progressionSnapshot(replay, nodeIds), + completedAt: event.result.evidence ? times.get(`${event.result.evidence.id}:${event.result.evidence.sha256}`) ?? null : null, + completion: scoreDependencyState(replay as DependencyState).completion.rate, + featureCompletion: dependencyCompletionBreakdown(replay as DependencyState).featureCompletion.rate }; + }); +} + +const progressionCache = new Map(); + +// The catalog subgraph the campaign runs, plus one node-status snapshot per +// progression event: the graph and its replay come from the same read. +export function campaignProgression(resultsRoot: string, key: string): CampaignProgression | null { + const directory = campaignDirectory(resultsRoot, key); + const { plan, state } = dashboardCampaignState(directory); + if (plan.definition.mode?.id !== 'dependency' || !plan.featureCatalog + || !plan.dependencyPolicy) return null; + const fingerprint = campaignFingerprint(directory, [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state], + [ARTIFACT_FILE.progressionState, ARTIFACT_FILE.run]); + const cached = progressionCache.get(directory); + if (cached?.fingerprint === fingerprint) return cached.view; + const progression = compileProgressionInput(dependencyRuntimeDefinition( + plan.featureCatalog, plan.dependencyPolicy)); + const definition = progression.definition; + const nodeIds = definition.nodes.map(node => node.id); + const owned = new Set(nodeIds); + const stacks: ProgressionTrack[] = []; + for (const attempt of state.attempts) { + const execution = attempt.executions.at(-1) ?? null; + if (!execution) continue; + const path = join(contained(directory, execution.output, 'campaign execution'), + ARTIFACT_FILE.progressionState); + if (!existsSync(path)) continue; + const stored = readProgressionState(path, { + progression, + featureCatalogIdentity: plan.featureCatalog.identity, + dependencyPolicyIdentity: plan.dependencyPolicy.identity, + owner: campaignProgressionOwner(plan, attempt.plan, { workspace: true }), + }); + const executionDirectory = contained(directory, execution.output, 'campaign execution'); + const times = new Map(); + for (const grade of gradeDirectories(executionDirectory)) { + const bundlePath = join(executionDirectory, grade.id, ARTIFACT_FILE.gradeBundle); + if (!existsSync(bundlePath)) continue; + try { + const bundle = readArtifact(bundlePath, { expectedKind: 'grade_bundle' }); + times.set(`${bundle.id}:${sha256(canonicalDefinitionJson(bundle))}`, bundle.timestamps.completedAt); + } catch { /* Missing or invalid evidence must not invent a chart timestamp. */ } + } + const costs: NonNullable = []; + try { + const run = readArtifact(join(executionDirectory, ARTIFACT_FILE.run), { expectedKind: 'benchmark_run' }); + if (run.attempt.parentId !== attempt.plan.id) throw new Error('Run belongs to another attempt'); + const checkpoints = (run.payload as { checkpoints?: RunCheckpoint[] }).checkpoints ?? []; + for (const checkpoint of checkpoints) { + try { + const evidencePath = contained(executionDirectory, checkpoint.evidence.path, 'cost checkpoint'); + if (sha256(readFileSync(evidencePath)) !== checkpoint.evidence.sha256) continue; + const grade = readArtifact(evidencePath, { expectedKind: 'grade_bundle' }); + if (grade.timestamps.completedAt && checkpoint.executionCost.status !== 'unknown') { + costs.push({ completedAt: grade.timestamps.completedAt, cost: checkpoint.executionCost }); + } + } catch { /* Missing evidence does not establish a timed cost. */ } + } + } catch { /* A run may not have saved its first checkpoint yet. */ } + stacks.push({ stack: attempt.plan.stack, attemptId: attempt.plan.id, + updatedAt: new Date(statSync(path).mtimeMs).toISOString(), costs, + steps: progressionSteps(stored.state as DependencyState, nodeIds, times) }); + } + const view: CampaignProgression = { + key: basename(resolve(directory)), + depths: [...new Set(definition.nodes.map(node => node.level))].sort((a, b) => a - b), + questlines: definition.questlines.map(questline => ({ id: questline.id, + title: questline.title, nodes: [...questline.nodes] })), + nodes: definition.nodes.map(node => ({ id: node.id, title: node.title, + questline: node.questline, depth: node.level, + dependencies: node.dependencies.filter(id => owned.has(id)) })), + stacks, + }; + progressionCache.set(directory, { fingerprint, view }); + return view; +} + +type LiveCosts = Map>>; +const liveCampaignCache = new Map(); + +function campaignLiveCosts(resultsRoot: string, key: string) { + const directory = campaignDirectory(resultsRoot, key); + const cached = liveCampaignCache.get(directory); + if (cached && (cached.pending || Date.now() - cached.at < 5000)) return cached.value; + const { state } = dashboardCampaignState(directory); + const entry = { at: Date.now(), pending: true, value: cached?.value ?? new Map() as LiveCosts }; + const refresh = async () => { + const costs = new Map>>(); + await Promise.all(state.attempts.map(async attempt => { + const execution = attempt.executions.at(-1); + // Restored executions can carry older transcript history. Keep their saved + // receipts until response identities are available across that boundary. + if (attempt.status !== 'running' || attempt.executions.length !== 1 || !execution?.startedAt + || !['claude-code', 'codex'].includes(attempt.plan.agentAdapter)) return; + try { + costs.set(attempt.plan.id, await liveTranscriptCost(contained(directory, execution.output, 'campaign execution'), + attempt.plan.agentAdapter, attempt.plan.pricing.rates, attempt.plan.model, execution.startedAt)); + } catch { /* Missing, conflicting, or unpriced usage leaves saved receipts visible. */ } + })); + entry.value = costs; + }; + if (liveCampaignCache.size >= 32) liveCampaignCache.delete(liveCampaignCache.keys().next().value!); + liveCampaignCache.set(directory, entry); + void refresh().catch(() => { entry.value = new Map(); }).finally(() => { + entry.pending = false; + entry.at = Date.now(); + }); + return entry.value; +} + +export function campaignLiveSheet(resultsRoot: string, key: string, options: ViewOptions = {}): CampaignSheet { + const sheet = structuredClone(campaignSheet(resultsRoot, key, options)); + const live = campaignLiveCosts(resultsRoot, key); + const directory = campaignDirectory(resultsRoot, key); + const { state } = dashboardCampaignState(directory); + for (const stack of sheet.stacks) { + for (const attempt of stack.attempts) { + const execution = state.attempts.find(entry => entry.plan.id === attempt.id)?.executions.at(-1); + if (execution) { + const path = join(contained(directory, execution.output, 'campaign execution'), LOG_FILE); + if (existsSync(path)) { + const updatedAt = new Date(statSync(path).mtimeMs).toISOString(); + if (attempt.logUpdatedAt !== updatedAt) { + attempt.logUpdatedAt = updatedAt; + const progress = parseRunProgress(readTextTail(path), { repairs: attempt.repairs.budget, + running: attempt.status === 'running', status: attempt.status, dependency: sheet.mode === 'dependency' }); + if (attempt.status === 'running' && !attempt.paused) attempt.phase = progress.phase; + attempt.climb = progress.series; + } + } + } + if (attempt.status !== 'running') continue; + const snapshot = live.get(attempt.id); + if (snapshot?.activityUpdatedAt) { + attempt.activityUpdatedAt = snapshot.activityUpdatedAt; + attempt.stalling = attemptStalling(attempt); + } + const total = snapshot?.costs.at(-1)?.costUsd; + attempt.liveSpend = liveCostTotal(attempt.status, total, attempt.spend.costUsd); + } + if (stack.attempts.some(attempt => attempt.liveSpend !== undefined) + && stack.attempts.every(attempt => attempt.liveSpend !== undefined || attempt.spend.status === 'exact')) { + stack.liveSpend = stack.attempts.reduce((sum, attempt) => sum + (attempt.liveSpend ?? attempt.spend.costUsd!), 0); + } + } + return sheet; +} + +// Transcript and log updates do not need another graph replay or evidence payload. +export function campaignLiveUpdate(resultsRoot: string, key: string, options: ViewOptions = {}) { + const sheet = campaignLiveSheet(resultsRoot, key, options); + const live = campaignLiveCosts(resultsRoot, key); + return { updatedAt: sheet.updatedAt, status: sheet.status, stacks: sheet.stacks.map(stack => ({ + stack: stack.stack, liveSpend: stack.liveSpend ?? null, + attempts: stack.attempts.filter(attempt => attempt.executionStartedAt !== null).map(attempt => ({ + id: attempt.id, phase: attempt.phase, logUpdatedAt: attempt.logUpdatedAt, + activityUpdatedAt: attempt.activityUpdatedAt ?? null, stalling: attempt.stalling, + climb: attempt.climb, liveSpend: attempt.liveSpend ?? null, + liveCosts: attempt.liveSpend === undefined ? [] : live.get(attempt.id)?.costs ?? [], + })), + })) }; +} +export type CampaignLiveUpdate = ReturnType; + +export function campaignLiveProgression(resultsRoot: string, key: string): CampaignProgression | null { + const progression = structuredClone(campaignProgression(resultsRoot, key)); + if (!progression) return null; + const live = campaignLiveCosts(resultsRoot, key); + const sheet = campaignSheet(resultsRoot, key); + const running = new Set(sheet.stacks.flatMap(stack => + stack.attempts.filter(attempt => attempt.status === 'running').map(attempt => attempt.id))); + for (const stack of sheet.stacks) for (const attempt of stack.attempts) { + const snapshot = live.get(attempt.id); + if (running.has(attempt.id) && snapshot?.costs.length + && !progression.stacks.some(track => track.attemptId === attempt.id)) { + progression.stacks.push({ stack: stack.stack, attemptId: attempt.id, + updatedAt: snapshot.activityUpdatedAt ?? sheet.updatedAt, steps: [], costs: [] }); + } + } + for (const track of progression.stacks) { + if (!running.has(track.attemptId)) continue; + const snapshot = live.get(track.attemptId); + if (snapshot?.costs.length && liveCostTotal('running', snapshot.costs.at(-1)!.costUsd, + track.costs?.at(-1)?.cost.costUsd ?? null) !== undefined) { + track.liveCosts = snapshot.costs; + } + } + return progression; +} + +export function attemptTranscript(resultsRoot: string, key: string, attemptId: string, + session: string, before?: number) { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + return readAttemptTranscript(attempt.executions.map((execution, index) => ({ + directory: contained(directory, execution.output, 'campaign execution'), + label: `Execution ${index + 1}`, + })), attempt.plan.agentAdapter, session, before); +} diff --git a/tools/stack-bench/dashboard/public/app.ts b/tools/stack-bench/dashboard/public/app.ts new file mode 100644 index 00000000000..de238bccdee --- /dev/null +++ b/tools/stack-bench/dashboard/public/app.ts @@ -0,0 +1,776 @@ +/// +/// +import type { readExecutionJob } from '../../src/campaigns/execution-jobs.js'; +import type { RunSetupCatalog, RunSetupRequest, RunSetupReview } from '../../src/campaigns/run-setup.js'; +import { initialRun, readRunForm, runSetupPage } from './views/run-setup.js'; +import type { TranscriptPage } from '../dashboard-transcript.js'; +import type { referenceRuns } from '../dashboard-reference-runs.js'; + + +// The client: real paths, one event stream, and keyed reconciliation so a +// refresh does not move what the pointer is on. Every view is a pure function +// of data; the only DOM work in the dashboard happens here. + +import type { AttemptChecks, AttemptPackage, CampaignLiveUpdate, CampaignProgression, CampaignSheet, OverviewEntry, OverviewPage } + from '../dashboard-views.js'; +import type { DashboardPlan } from '../dashboard-model.js'; +import type { readCampaignTimeBudget } from '../../src/campaigns/campaign-time-grant.js'; +import { type QuestlineView, campaignPage, replayTimeline, selectedProgression } from './views/campaign.js'; +import { type AttemptTab, attemptPage } from './views/attempt.js'; +import { type CampaignFilter, campaignsPage } from './views/campaigns.js'; +import { type Page, type RunForm, plansPage, topbar } + from './views/plans.js'; +import { duration, elapsed, esc } from './format.js'; + +const FALLBACK_MS = 15_000; +const TABS: readonly AttemptTab[] = ['checks', 'transcript', 'screenshots', 'files', 'log']; +const VIEWS: readonly QuestlineView[] = ['grid', 'graph', 'replay']; +const FILTERS: readonly CampaignFilter[] = ['all', 'attention', 'completed', 'ready']; + +interface Route { + key: string; + attempt: string; + plans: boolean; + newRun: boolean; + filter: CampaignFilter; + page: number; + view: QuestlineView; + chart: 'completion' | 'cost' | 'distribution'; + unit: 'checks' | 'features'; + step: number; + tab: AttemptTab; +} + +const state = { + overview: [] as OverviewEntry[], + overviewPage: null as OverviewPage | null, + overviewQuery: '', + references: { runs: [], error: null } as Awaited>, + plans: [] as DashboardPlan[], + overviewLoaded: false, + plansLoaded: false, + pendingJobs: new Map; dispatchError: string | null }>(), + setup: null as RunSetupCatalog | null, + setupRequest: null as RunSetupRequest | null, + setupReview: null as RunSetupReview | null, + canStart: false, + csrfToken: '', + readError: '', + form: { error: '' } as RunForm, + sheets: new Map(), + progression: new Map(), + hiddenChartRuns: new Map>(), + checks: new Map(), + evidence: new Map(), + timeBudgets: new Map>(), + timeGrantIds: new Map(), + timeGrantMinutes: '120', + transcript: { attempt: '', session: '', before: undefined as number | undefined, page: null as TranscriptPage | null }, + log: { attempt: '', text: '', offset: 0 }, +}; +let fallback = 0; +let events: EventSource | null = null; +let playing = 0; +let submitting = false; +let loading = false; +let loadVersion = 0; +let loadTask: Promise | null = null; +let loadController = new AbortController(); +let refreshPending = false; +let pendingNavigation = false; +let pendingKeys: Set | null = new Set(); +let pendingOverview = false; + +function route(): Route { + const url = new URL(location.href); + const parts = url.pathname.split('/').filter(Boolean); + const pick = (values: readonly Value[], name: string, fall: Value): Value => + values.find(value => value === url.searchParams.get(name)) ?? fall; + return { + key: parts[0] === 'c' ? parts[1] ?? '' : '', + attempt: parts[2] === 'a' ? parts[3] ?? '' : '', + plans: parts[0] === 'plans' || parts[0] === 'new', + newRun: parts[0] === 'new', + filter: pick(FILTERS, 'filter', 'all'), + page: /^\d+$/.test(url.searchParams.get('page') ?? '1') && Number.isSafeInteger(Number(url.searchParams.get('page') ?? 1)) + ? Math.max(1, Number(url.searchParams.get('page') ?? 1)) : 1, + view: pick(VIEWS, 'questlines', 'grid'), + chart: pick(['completion', 'cost', 'distribution'] as const, 'chart', 'completion'), + unit: pick(['checks', 'features'] as const, 'unit', 'features'), + step: Math.max(0, Number(url.searchParams.get('step') ?? 0)), + tab: pick(TABS, 'tab', 'checks'), + }; +} + +async function read(url: string): Promise { + const version = loadVersion; + try { + const response = await fetch(url, { headers: { accept: 'application/json' }, + signal: AbortSignal.any([loadController.signal, AbortSignal.timeout(30_000)]) }); + if (!response.ok) { + const failure = await response.json().catch(() => ({})) as { error?: string }; + if (version === loadVersion) state.readError = failure.error ?? `Request failed (${response.status}).`; + return null; + } + const payload = await response.json() as Payload; + return version === loadVersion ? payload : null; + } catch { + if (version === loadVersion) state.readError = 'The dashboard did not respond. Check its connection and try again.'; + return null; + } +} + +function attemptUrl(current: Route, suffix: string): string { + return `/api/campaigns/${encodeURIComponent(current.key)}` + + `/attempts/${encodeURIComponent(current.attempt)}/${suffix}`; +} + +async function readLog(current: Route): Promise { + const version = loadVersion; + if (state.log.attempt !== current.attempt) state.log = { attempt: current.attempt, text: '', offset: 0 }; + try { + const response = await fetch(attemptUrl(current, `log?from=${state.log.offset}`), + { signal: AbortSignal.any([loadController.signal, AbortSignal.timeout(30_000)]) }); + if (!response.ok) throw new Error('Log request failed'); + const text = await response.text(); + if (version !== loadVersion || state.log.attempt !== current.attempt) return; + state.log.text += text; + state.log.offset = Number(response.headers.get('x-stack-bench-log-offset') ?? state.log.offset); + } catch { + if (version === loadVersion) state.readError = 'Could not load the run log. Try again.'; + } +} + +function chrome(current: Route): string { + const sheet = state.sheets.get(current.key) ?? null; + const page: Page = current.plans ? 'plans' + : current.key && !current.attempt ? 'campaign' : 'campaigns'; + return topbar({ page, key: current.key, canStart: state.canStart, error: state.form.error, + reportFiles: sheet?.reportFiles, + controllerOwner: page === 'campaign' ? sheet?.controllerOwner : null, + resumable: state.canStart && page === 'campaign' && (sheet?.resumable ?? false) }); +} + +function page(current: Route): string { + const sheet = state.sheets.get(current.key) ?? null; + const pending = state.pendingJobs.get(current.key); + if (pending) return `

${esc(pending.pendingJob.job.key)}

${esc(pending.pendingJob.status)}

` + + `

${esc(pending.dispatchError ?? pending.pendingJob.error ?? pending.pendingJob.capacityWait?.reason ?? 'Waiting for the campaign to start. This page updates automatically.')}

` + + (state.canStart && ['queued', 'running'].includes(pending.pendingJob.status) + ? '
' : '') + + (state.canStart && pending.pendingJob.status === 'queued' && pending.dispatchError + ? '
' : '') + + (state.form.error ? `

${esc(state.form.error)}

` : '') + '
'; + if (current.newRun) return runSetupPage(state.setup, state.setupRequest, state.setupReview, state.form.error, state.canStart); + if (current.plans) { + return plansPage({ plans: state.plans, + loading: loading && !state.plansLoaded }); + } + if (!current.key) { + const running = (state.overviewPage?.running ?? []) + .map(key => state.sheets.get(key)) + .filter((entry): entry is CampaignSheet => entry !== undefined); + return campaignsPage({ campaigns: state.overview, sheets: running, filter: current.filter, + pagination: state.overviewPage ?? undefined, + references: state.references, + loading: loading && (!state.overviewLoaded || state.overviewQuery !== `${current.filter}:${current.page}`) }); + } + if (!sheet) return `
Campaigns / ` + + `${esc(current.key)}
`; + if (current.attempt) { + return attemptPage({ sheet, progression: state.progression.get(current.key) ?? null, + attemptId: current.attempt, tab: current.tab, + timeBudget: state.timeBudgets.get(current.attempt), canControl: state.canStart, + controlError: state.form.error, + transcript: state.transcript.attempt === current.attempt ? state.transcript.page : null, + checks: state.checks.get(current.attempt) ?? null, + evidence: state.evidence.get(current.attempt) ?? null, + log: state.log.attempt === current.attempt ? state.log.text : '' }); + } + return campaignPage({ sheet, progression: state.progression.get(current.key) ?? null, + view: current.view, step: current.step, chart: current.chart, unit: current.unit, + hiddenChartRuns: state.hiddenChartRuns.get(current.key) }); +} + +function sync(current: Element, next: Element): void { + for (const name of [...current.getAttributeNames()]) { + if (current.tagName === 'DETAILS' && name === 'open') continue; + if (!next.hasAttribute(name)) current.removeAttribute(name); + } + for (const name of next.getAttributeNames()) { + if (current.getAttribute(name) !== next.getAttribute(name)) { + current.setAttribute(name, next.getAttribute(name) ?? ''); + } + } +} + +// Replace only what changed, matching children by position and data-key, so a +// row under the pointer keeps its hover across a refetch. +function patch(current: Element, next: Element): void { + const mine = [...current.children]; + const theirs = [...next.children]; + if (mine.length !== theirs.length || current.childNodes.length !== mine.length + || next.childNodes.length !== theirs.length) { + current.replaceChildren(...next.childNodes); + return; + } + mine.forEach((child, index) => { + const other = theirs[index]!; + if (child.tagName !== other.tagName + || child.getAttribute('data-key') !== other.getAttribute('data-key')) { + child.replaceWith(other); + return; + } + if (child.outerHTML === other.outerHTML) return; + if (!child.children.length || !other.children.length) { + child.replaceWith(other); + return; + } + // Moving option nodes can change a native select's value during reconciliation. + const selected = other instanceof HTMLSelectElement ? other.value : null; + sync(child, other); + patch(child, other); + if (child instanceof HTMLSelectElement && selected !== null) child.value = selected; + }); +} + +function updateTimeTotal(field: HTMLInputElement): void { + const total = field.form?.querySelector('[data-time-base]'); + if (total) total.textContent = field.validity.valid + ? `Limit after request: ${duration((Number(total.dataset.timeBase) + field.valueAsNumber) * 60)}` + : 'Enter positive whole minutes.'; +} + +function render(): void { + const current = route(); + const root = document.body; + const next = document.createElement('body'); + const ready = current.plans ? state.plansLoaded : current.key + ? state.sheets.has(current.key) : state.overviewLoaded; + next.innerHTML = `${chrome(current)}
` + + (state.readError ? `` : '') + + (loading && !ready && current.key ? `

${current.attempt ? 'Run details' : 'Campaign'}

` + + '
Loading…
' : page(current)) + + '
'; + const transcript = root.querySelector('.transcript'); + const scroll = transcript?.scrollTop ?? 0; + const follow = !transcript || transcript.scrollHeight - scroll - transcript.clientHeight < 40; + const openTools = [...root.querySelectorAll('.transcript details[open]')].map(el => el.dataset.key); + patch(root, next); + const updated = root.querySelector('.transcript'); + if (updated) { + for (const tool of updated.querySelectorAll('details')) tool.open = openTools.includes(tool.dataset.key); + updated.scrollTop = follow ? updated.scrollHeight : scroll; + } + // Keep the time limit input across background refreshes. + for (const field of document.querySelectorAll('form[data-run] input')) { + if (field.name === 'minutes') { + field.value = state.timeGrantMinutes; + updateTimeTotal(field); + } + } + for (const form of document.querySelectorAll('form[data-run]')) { + form.setAttribute('aria-busy', String(submitting)); + if (submitting && form.dataset.run?.startsWith('setup-')) { + const submit = form.querySelector('button[type=submit]'); + if (submit) submit.textContent = form.dataset.run === 'setup-review' ? 'Preparing review…' : 'Starting…'; + } + for (const button of form.querySelectorAll('button[type=submit]')) { + button.disabled = submitting || (!state.canStart && form.dataset.run?.startsWith('setup-')) || (form.dataset.run === 'grant-time' + && (state.timeBudgets.get(current.attempt)?.grants.some(grant => grant.disposition === 'pending') ?? false)); + } + } +} + +function load(navigation = false, changedKey?: string, liveOnly = false): Promise { + refreshPending = true; + pendingNavigation ||= navigation; + pendingOverview ||= !liveOnly; + if (changedKey) pendingKeys?.add(changedKey); + else pendingKeys = null; + if (navigation) { + state.form = { error: '' }; + ++loadVersion; + loadController.abort(); + } + if (loadTask) return loadTask; + if (document.hidden && !navigation) return Promise.resolve(); + loadTask = (async () => { + while (refreshPending && (!document.hidden || pendingNavigation)) { + const showLoading = pendingNavigation; + const keys = pendingKeys; + const refreshOverview = pendingOverview; + refreshPending = pendingNavigation = false; + pendingOverview = false; + pendingKeys = new Set(); + const version = ++loadVersion; + loadController = new AbortController(); + loading = true; + state.readError = ''; + if (showLoading) render(); + try { + await loadData(version, keys, refreshOverview); + } catch { + if (version === loadVersion) state.readError = 'Could not load this page. Try again.'; + } finally { + if (version === loadVersion) { + loading = false; + render(); + } + } + } + })().finally(() => { loadTask = null; }); + return loadTask; +} + +async function loadData(version: number, changedKeys: Set | null, refreshOverview: boolean): Promise { + const current = route(); + if (refreshOverview && !current.key && !current.plans) { + const references = await read>>('/api/reference-runs'); + if (references && version === loadVersion) { state.references = references; render(); } + } + if (!refreshOverview) { + const keys = current.key ? [current.key] : [...state.sheets.keys()] + .filter(key => state.sheets.get(key)?.status === 'running' && (!changedKeys || changedKeys.has(key))); + await Promise.all(keys.map(async key => { + const sheet = state.sheets.get(key); + if (!sheet) return; + const update = await read(`/api/campaigns/${encodeURIComponent(key)}/live`); + if (!update || version !== loadVersion) return; + if (update.updatedAt !== sheet.updatedAt || update.status !== sheet.status) { + void load(false, key); // Evidence changed while a log refresh was in flight. + return; + } + const progression = state.progression.get(key); + for (const stack of sheet.stacks) { + const fresh = update.stacks.find(entry => entry.stack === stack.stack); + stack.liveSpend = fresh?.liveSpend ?? undefined; + for (const attempt of stack.attempts) { + const live = fresh?.attempts.find(entry => entry.id === attempt.id); + if (!live) continue; + const { liveCosts, liveSpend, ...fields } = live; + Object.assign(attempt, fields, { liveSpend: liveSpend ?? undefined }); + let track = progression?.stacks.find(entry => entry.attemptId === attempt.id); + if (!track && progression && liveCosts.length) { + track = { stack: stack.stack, attemptId: attempt.id, updatedAt: update.updatedAt, steps: [], costs: [] }; + progression.stacks.push(track); + } + if (track) track.liveCosts = liveCosts.length ? liveCosts : undefined; + } + } + })); + if (version !== loadVersion) return; + if (current.tab === 'transcript') await readTranscript(); + else if (current.attempt && current.tab === 'log') await readLog(current); + return; + } + const setupRequest = current.newRun && !state.setupRequest ? read('/api/run-setup').then(catalog => { + if (catalog && version === loadVersion) { + state.setup = catalog; state.setupRequest ??= initialRun(catalog); state.plansLoaded = true; render(); + } + }) : null; + const plansRequest = current.plans && !current.newRun ? read('/api/plans').then(plans => { + if (plans && version === loadVersion) { + state.plans = plans; + state.plansLoaded = true; + render(); + } + }) : null; + if (!state.csrfToken && (current.key || current.plans)) { + const session = await read<{ canStart: boolean; csrfToken: string }>('/api/session'); + if (version !== loadVersion) return; + if (session) Object.assign(state, session); + } + if (!current.key && !current.plans && (refreshOverview || !state.csrfToken)) { + const overview = await read(`/api/overview?filter=${current.filter}&page=${current.page}`); + if (version !== loadVersion) return; + if (overview) Object.assign(state, { overview: overview.campaigns, overviewLoaded: true, + overviewPage: overview, overviewQuery: `${current.filter}:${current.page}`, + canStart: overview.canStart, csrfToken: overview.csrfToken }); + render(); + } + if (current.plans) { + await Promise.all([plansRequest, setupRequest]); + if (version !== loadVersion) return; + render(); + return; + } + if (!current.key) { + const campaigns = (state.overviewPage?.running ?? []).filter(key => + !changedKeys || changedKeys.has(key) || !state.sheets.has(key)); + await Promise.all(campaigns.map(async key => { + const sheet = await read(`/api/campaigns/${encodeURIComponent(key)}`); + if (version !== loadVersion) return; + if (sheet) state.sheets.set(key, sheet); + render(); + })); + return; + } + const result = await read; dispatchError: string | null }>(`/api/campaigns/${encodeURIComponent(current.key)}`); + if (version !== loadVersion) return; + if (result && 'pendingJob' in result) { state.pendingJobs.set(current.key, result); render(); return; } + state.pendingJobs.delete(current.key); + const sheet = result; + if (sheet) state.sheets.set(current.key, sheet); + render(); + if (sheet?.mode === 'dependency') { + const progression = await read( + `/api/campaigns/${encodeURIComponent(current.key)}/progression`); + if (version !== loadVersion) return; + if (progression) state.progression.set(current.key, progression); + render(); + } + if (!current.attempt) return; + const timeBudget = await read>(attemptUrl(current, 'time')); + if (version !== loadVersion) return; + if (timeBudget) { + state.timeBudgets.set(current.attempt, timeBudget); + if (timeBudget.grants.some(grant => grant.request.grantId === state.timeGrantIds.get(current.attempt) + && grant.disposition !== 'pending')) state.timeGrantIds.delete(current.attempt); + } + if (current.tab === 'checks') { + const checks = await read(attemptUrl(current, 'checks')); + if (checks) state.checks.set(current.attempt, checks); + } else if (current.tab === 'screenshots' || current.tab === 'files') { + const evidence = await read(attemptUrl(current, 'package')); + if (evidence) state.evidence.set(current.attempt, evidence); + } else if (current.tab === 'transcript') { + await readTranscript(); + } else if (current.tab === 'log') { + await readLog(current); + } + render(); +} + +function go(href: string): void { + history.pushState(null, '', href); + void load(true); +} + +function stepTo(offset: number): void { + const current = route(); + const progression = state.progression.get(current.key) ?? null; + const sheet = state.sheets.get(current.key); + if (!progression || !sheet) return; + const total = replayTimeline(selectedProgression(progression, sheet)).length; + const next = Math.min(Math.max(0, current.step + offset), Math.max(0, total - 1)); + const url = new URL(location.href); + url.searchParams.set('step', String(next)); + history.replaceState(null, '', `${url.pathname}${url.search}`); + render(); +} + +function subscribe(): void { + if (document.hidden || events) return; + const source = events = new EventSource('/api/events'); + const changed = (event: MessageEvent): void => { + const current = route(); + const message = JSON.parse(event.data) as { key?: string; attemptId?: string }; + const ids = message.attemptId ? [message.attemptId] + : state.sheets.get(message.key ?? '')?.stacks.flatMap(stack => stack.attempts.map(attempt => attempt.id)) ?? []; + for (const id of ids) { + // Keep the visible tab stable until its replacement data arrives. + if (message.key === current.key && id === current.attempt) continue; + state.checks.delete(id); + state.evidence.delete(id); + } + if (current.plans || (current.key && message.key !== current.key)) return; + if (current.attempt && message.attemptId && message.attemptId !== current.attempt) return; + void load(false, message.key); + }; + source.addEventListener('campaign', changed); + source.addEventListener('reference', async () => { + const current = route(); + if (!current.key && !current.plans && !document.hidden) { + const references = await read>>('/api/reference-runs'); + if (references) { state.references = references; render(); } + } + }); + source.addEventListener('log', event => { + const current = route(); + const message = JSON.parse((event as MessageEvent).data) as { key: string }; + if (!current.plans && (!current.key || message.key === current.key)) void load(false, message.key, true); + }); + source.addEventListener('open', () => { + if (fallback) clearInterval(fallback); + fallback = 0; + void load(); + }); + // Recover missed campaign changes while the stream is down. + source.addEventListener('error', () => { + fallback ||= window.setInterval(() => void load(), FALLBACK_MS); + }); +} + +let helpClose = 0; +for (const type of ['pointerover', 'pointerout', 'focusin', 'focusout']) document.addEventListener(type, event => { + if (!(event.target instanceof Element)) return; + const series = event.target.closest('[data-chart-series]'); + if (!series) return; + const lines = [...series.closest('.page')?.querySelectorAll('[data-chart-series]') ?? []]; + const active = (type === 'pointerover' || type === 'focusin') + && lines.some(line => line.dataset.chartSeries === series.dataset.chartSeries); + for (const line of lines) { + line.classList.toggle('is-muted', active && line.dataset.chartSeries !== series.dataset.chartSeries); + line.classList.toggle('is-highlighted', active && line.dataset.chartSeries === series.dataset.chartSeries); + } +}); +for (const type of ['pointerover', 'focusin']) document.addEventListener(type, event => { + if (!(event.target instanceof Element)) return; + if (!event.target.closest('.metric-help, .metric-tooltip')) return; + clearTimeout(helpClose); + const trigger = event.target.closest('.metric-help'); + trigger?.click(); +}); +for (const type of ['pointerout', 'focusout']) document.addEventListener(type, event => { + if (!(event.target instanceof Element) + || !event.target.closest('.metric-help, .metric-tooltip')) return; + clearTimeout(helpClose); + helpClose = window.setTimeout(() => { + if (document.querySelector('.metric-help:hover, .metric-help:focus, .metric-tooltip:hover')) return; + document.querySelector('.metric-tooltip:popover-open')?.hidePopover(); + }, 150); +}); + +document.addEventListener('click', event => { + if (event.defaultPrevented || event.button !== 0 || event.metaKey || event.ctrlKey + || event.shiftKey || event.altKey) return; + const chartToggle = (event.target as Element | null)?.closest('[data-chart-run], [data-chart-stack]'); + if (chartToggle) { + const key = route().key; + const hidden = state.hiddenChartRuns.get(key) ?? new Set(); + const ids = chartToggle.dataset.chartRun !== undefined ? [chartToggle.dataset.chartRun] + : state.sheets.get(key)?.stacks.find(stack => stack.stack === chartToggle.dataset.chartStack) + ?.attempts.map(attempt => attempt.id) ?? []; + const hide = ids.some(id => !hidden.has(id)); + for (const id of ids) { if (hide) hidden.add(id); else hidden.delete(id); } + state.hiddenChartRuns.set(key, hidden); + render(); + return; + } + if ((event.target as Element | null)?.closest('[data-retry]')) { + void load(true); + return; + } + const shot = (event.target as Element | null)?.closest('[data-shot]'); + if (shot) { + const dialog = document.querySelector('.lightbox'); + const image = dialog?.querySelector('img'); + if (!dialog || !image) return; + image.src = shot.dataset.shot ?? ''; + image.alt = shot.dataset.shotName ?? ''; + dialog.showModal(); + return; + } + if (event.target instanceof HTMLDialogElement) event.target.close(); + const link = (event.target as Element | null)?.closest('a'); + const href = link?.getAttribute('href') ?? ''; + if (!href || href === '/checks' || href.startsWith('/api/') || !/^[/?]/.test(href)) return; + event.preventDefault(); + go(href.startsWith('?') ? `${location.pathname}${href}` : href); +}); + +// Controls require the browser token and same origin; the server re-reads the plan. +async function post(form: HTMLFormElement): Promise { + if (submitting) return; + const current = route(); + const data = new FormData(form); + const action = form.dataset.run; + // A response belongs to this form, even if the user leaves before it arrives. + const submittedForm = state.form = { error: '' }; + if (action === 'setup-review' || action === 'setup-start') { + if (action === 'setup-review') state.setupRequest = readRunForm(form, state.setup!); + submitting = true; render(); + try { + const response = await fetch(action === 'setup-review' ? '/api/runs/prepare' : '/api/runs', { + method: 'POST', headers: { 'content-type': 'application/json', 'x-stack-bench-token': state.csrfToken }, + body: JSON.stringify(action === 'setup-review' ? state.setupRequest + : { request: state.setupReview!.request, reviewId: state.setupReview!.reviewId }), + }); + const result = await response.json(); + if (state.form !== submittedForm) return; + if (!response.ok) { + submittedForm.error = result.error ?? `Request failed (${response.status})`; + if (response.status === 403) { state.csrfToken = ''; void load(); } + return; + } + if (action === 'setup-review') state.setupReview = result as RunSetupReview; + else { + state.setupReview = null; state.setupRequest = null; + go(`/c/${encodeURIComponent(result.campaignKey)}`); + } + } catch { submittedForm.error = 'Could not confirm the request. Retry with the same setup; it cannot create a second job.'; } + finally { submitting = false; render(); } + return; + } + + const resumeWithTime = action === 'grant-time' && form.dataset.resume === 'true'; + const existing = action === 'job-start' || action === 'job-cancel' || action === 'resume' || action === 'stop' || action === 'grant-time'; + if (!existing) return; + // Retain the ID after an uncertain response, so retry cannot add time twice. + if (action === 'grant-time' && !state.timeGrantIds.has(current.attempt)) { + state.timeGrantIds.set(current.attempt, crypto.randomUUID()); + } + const grantId = state.timeGrantIds.get(current.attempt); + submitting = true; + render(); + let timeAccepted = false; + try { + let response = await fetch(action === 'job-start' ? `/api/jobs/${current.key.slice(4)}/start` : action === 'job-cancel' ? `/api/jobs/${current.key.slice(4)}/cancel` : action === 'grant-time' ? attemptUrl(current, 'time') : `/api/campaigns/${encodeURIComponent(current.key)}/${action}`, { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-stack-bench-token': state.csrfToken }, + body: JSON.stringify(action === 'grant-time' ? { grantId, minutes: Number(data.get('minutes')) } + : action === 'stop' ? { owner: data.get('owner') } + : {}), + }); + if (response.ok && resumeWithTime) { + timeAccepted = true; + response = await fetch(`/api/campaigns/${encodeURIComponent(current.key)}/resume`, { + method: 'POST', headers: { 'content-type': 'application/json', 'x-stack-bench-token': state.csrfToken }, body: '{}', + }); + } + if (response.ok) { + return void load(); + } + const failure = await response.json().catch(() => ({})) as { error?: string }; + if (response.status === 403) { state.csrfToken = ''; void load(); } + submittedForm.error = + (timeAccepted ? 'Time was added, but resume failed. ' : '') + + (failure.error || `Request failed (HTTP ${response.status}). Check campaign status before retrying.`); + } catch { + submittedForm.error = (timeAccepted ? 'Time was added. Could not confirm resume. ' : 'Could not confirm the request. ') + + 'Check campaign status before retrying.'; + } finally { + submitting = false; + render(); + } +} + +document.addEventListener('submit', event => { + const form = event.target; + if (!(form instanceof HTMLFormElement) || !form.dataset.run) return; + event.preventDefault(); + void post(form); +}); + +// Keep typed settings across background refreshes. +document.addEventListener('input', event => { + const field = event.target as HTMLInputElement; + if (field.form?.dataset.run === 'setup-review') { + state.setupRequest = field.name === 'workload' ? initialRun(state.setup!, field.value) : readRunForm(field.form, state.setup!); + if (field.name === 'workload' || field.name === 'level') render(); + return; + } + + if (field.name === 'minutes') { + state.timeGrantMinutes = field.value; + updateTimeTotal(field); + } + +}); + +document.addEventListener('keydown', event => { + if (route().view !== 'replay') return; + if (event.target instanceof Element + && event.target.closest('input, textarea, select, button, summary, [contenteditable]')) return; + if (event.key === 'ArrowRight') stepTo(1); + else if (event.key === 'ArrowLeft') stepTo(-1); + else if (event.key === ' ') { + event.preventDefault(); + if (playing) { + clearInterval(playing); + playing = 0; + } else playing = window.setInterval(() => stepTo(1), 600); + return; + } else return; + if (playing) { + clearInterval(playing); + playing = 0; + } +}); + +window.setInterval(() => { + for (const clock of document.querySelectorAll('[data-started-at]')) { + clock.textContent = elapsed(clock.dataset.startedAt ?? null, null); + } +}, 1000); +window.addEventListener('popstate', () => void load(true)); +document.addEventListener('visibilitychange', () => { + if (document.hidden) { + // Hidden tabs must not consume the browser's limited HTTP connections. + events?.close(); + events = null; + clearInterval(fallback); + fallback = 0; + } else { + subscribe(); + void load(); + } +}); +subscribe(); +void load(true); + +let transcriptLoading = false; +let transcriptReload = false; +async function readTranscript(force = false): Promise { + const current = route(); + if (document.hidden || current.tab !== 'transcript' || !current.attempt) return; + if (transcriptLoading) { transcriptReload ||= force; return; } + if (state.transcript.attempt !== current.attempt) state.transcript = { + attempt: current.attempt, session: '', before: undefined, page: null }; + const pane = document.querySelector('.transcript'); + if (!force && state.transcript.page && pane + && pane.scrollHeight - pane.scrollTop - pane.clientHeight >= 40) return; + const selected = state.transcript; + transcriptLoading = true; + try { + const query = new URLSearchParams({ session: selected.session }); + if (selected.before !== undefined) query.set('before', String(selected.before)); + const page = await read(attemptUrl(current, `transcript?${query}`)); + if (page && state.transcript === selected) selected.page = page; + } finally { + transcriptLoading = false; + if (transcriptReload) { + transcriptReload = false; + await readTranscript(true); + } + } +} +setInterval(() => { + if (document.hidden) return; + const current = route(); + if (current.plans) return; + if (current.key && state.sheets.get(current.key)?.status === 'running') { + void load(false, current.key, true); + } else if (!current.key && state.overviewPage?.running.length) { + void load(false, undefined, true); + } else if (current.tab === 'transcript' && state.transcript.before === undefined) { + void readTranscript().then(render); + } +}, 5000); +document.addEventListener('change', event => { + const target = event.target; + if (target instanceof HTMLSelectElement && target.matches('[data-transcript-session]')) { + state.transcript = { ...state.transcript, session: target.value, before: undefined }; + void readTranscript(true).then(render); + } +}); +document.addEventListener('click', event => { + const target = event.target instanceof Element ? event.target.closest('[data-transcript-before], [data-transcript-latest]') : null; + if (!target) return; + state.transcript = { ...state.transcript, + session: state.transcript.page?.session ?? '', + before: target.hasAttribute('data-transcript-before') ? Number(target.dataset.transcriptBefore) : undefined }; + void readTranscript(true).then(() => { + render(); + const pane = document.querySelector('.transcript'); + if (pane && target.hasAttribute('data-transcript-latest')) pane.scrollTop = pane.scrollHeight; + }); +}); + +document.addEventListener('click', event => { + if (event.target instanceof Element && event.target.closest('[data-setup-edit]')) { + state.setupReview = null; state.form.error = ''; render(); + } +}); diff --git a/tools/stack-bench/dashboard/public/check-guide.ts b/tools/stack-bench/dashboard/public/check-guide.ts new file mode 100644 index 00000000000..3f721b9e5e7 --- /dev/null +++ b/tools/stack-bench/dashboard/public/check-guide.ts @@ -0,0 +1,31 @@ +/// +/// +export {}; +const search = document.querySelector('[data-guide-search]')!; +const others = document.querySelector('[data-guide-others]')!; +const checks = [...document.querySelectorAll('.guide-check')]; +const url = new URL(location.href); +search.value = url.searchParams.get('q') ?? ''; +others.checked = url.searchParams.get('scope') === 'all'; +function filter(): void { + let shown = 0; + for (const check of checks) { + check.hidden = (!others.checked && check.dataset.active !== 'true') + || !(check.dataset.search ?? '').includes(search.value.trim().toLowerCase()); + if (!check.hidden) shown++; + } + document.querySelector('[data-guide-count]')!.textContent = `${shown} ${shown === 1 ? 'check' : 'checks'} shown`; + document.querySelector('[data-guide-empty]')!.hidden = shown > 0; + if (search.value) url.searchParams.set('q', search.value); else url.searchParams.delete('q'); + if (others.checked) url.searchParams.set('scope', 'all'); else url.searchParams.delete('scope'); + history.replaceState(null, '', url); +} +search.addEventListener('input', filter); +others.addEventListener('change', filter); +const toggle = document.querySelector('[data-guide-toggle]')!; +toggle.addEventListener('click', () => { + const expand = toggle.textContent === 'Expand visible'; + for (const check of checks) check.open = expand && !check.hidden; + toggle.textContent = expand ? 'Collapse all' : 'Expand visible'; +}); +filter(); diff --git a/tools/stack-bench/dashboard/public/climb.ts b/tools/stack-bench/dashboard/public/climb.ts new file mode 100644 index 00000000000..4c1d7d72741 --- /dev/null +++ b/tools/stack-bench/dashboard/public/climb.ts @@ -0,0 +1,77 @@ +// The climb: one point per completed grade, unaided grades ringed, the current +// grade filled. Small in a lane or a sheet cell, large on the attempt page. + +import type { ClimbPoint } from '../dashboard-views.js'; +import { esc } from './format.js'; + +interface Plot { + x: number; + y: number; + point: ClimbPoint; +} + +function pointTitle(point: ClimbPoint): string { + return `${point.score} / ${point.max} points${point.unaided ? ' · First build at this level; earlier fixes and feedback retained' : ''}`; +} + +function plot(series: readonly ClimbPoint[], left: number, right: number, + top: number, bottom: number): Plot[] { + const span = Math.max(1, series.length - 1); + return series.map((point, index) => ({ + x: series.length === 1 ? (left + right) / 2 : left + (right - left) * index / span, + y: bottom - (bottom - top) * (point.max ? point.score / point.max : 0), + point, + })); +} + +function stepPath(plots: readonly Plot[]): string { + const head = plots[0]; + if (!head) return ''; + return plots.slice(1).reduce((path, item, index) => + `${path} L${item.x} ${plots[index]!.y} L${item.x} ${item.y}`, `M${head.x} ${head.y}`); +} + +// Full size: the same points with a band per depth or level, and a number at +// the first, the best and the current grade. +export function bigClimb(series: readonly ClimbPoint[], stage: (level: number) => string): string { + if (!series.length) return '

Awaiting first grade. The score history will appear here.

'; + const top = 10; + const bottom = 130; + const plots = plot(series, 100, 1010, top, bottom); + const bands: string[] = []; + let start = 0; + plots.forEach((item, index) => { + const next = plots[index + 1]; + if (next && next.point.level === item.point.level) return; + const level = item.point.level; + if (level !== null) { + const from = Math.max(60, plots[start]!.x - 40); + const width = Math.min(1050, item.x + 40) - from; + bands.push(`` + + `${esc(stage(level))}`); + } + start = index + 1; + }); + const line = stepPath(plots); + const first = plots[0]!; + const last = plots.at(-1)!; + const best = plots.reduce((top1, item) => item.y < top1.y ? item : top1, first); + const label = (item: Plot, tone: string): string => + `` + + `${Math.round(item.point.max ? 100 * item.point.score / item.point.max : 0)}`; + return `
Weighted score by completed grade. Each grade can cover a different scope.${bands.join('')}` + + [0, 50, 100].map(value => { + const y = bottom - (bottom - top) * value / 100; + return `` + + `${value}%`; + }).join('') + + `` + + `` + + plots.map(item => `${pointTitle(item.point)}`).join('') + + label(first, '#b6c0cf') + (best === first || best === last ? '' : label(best, '#b6c0cf')) + + (last === first ? '' : label(last, '#e6e9f0')) + '
'; +} diff --git a/tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt b/tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt new file mode 100644 index 00000000000..40589daa9de --- /dev/null +++ b/tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt @@ -0,0 +1,93 @@ +Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) Inter-Italic[opsz,wght].ttf: Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 b/tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 new file mode 100644 index 00000000000..d15208de03c Binary files /dev/null and b/tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 differ diff --git a/tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt b/tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt new file mode 100644 index 00000000000..046fc664900 --- /dev/null +++ b/tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt @@ -0,0 +1,93 @@ +Google Inc. + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 b/tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 new file mode 100644 index 00000000000..bc303f50c5d Binary files /dev/null and b/tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 differ diff --git a/tools/stack-bench/dashboard/public/format.ts b/tools/stack-bench/dashboard/public/format.ts new file mode 100644 index 00000000000..5b7c147f5df --- /dev/null +++ b/tools/stack-bench/dashboard/public/format.ts @@ -0,0 +1,123 @@ +// One spelling per value. Every figure the dashboard prints goes through here, +// so a percentage, a duration and a dash look the same on every page. + +import type { SheetAttempt } from '../dashboard-views.js'; +import type { CostEvidence } from '../../src/evidence/cost-proof.js'; +import { statusWord } from '../../src/evidence/status-words.js'; +import { outputSilentMinutes } from './metrics.js'; + +export { statusWord }; + +const SILENCE_MINUTES = 10; + +export const STACK_LABEL: Record = { spacetime: 'SpacetimeDB', + postgres: 'PostgreSQL', mongodb: 'MongoDB', convex: 'Convex' }; +export const DASH = '—'; + +export function esc(value: unknown): string { + return String(value ?? '').replace(/[&<>"']/g, character => + ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[character] ?? character); +} + +export function stackLabel(stack: string): string { + return STACK_LABEL[stack] ?? stack; +} + +export function metricLabel(label: string, description: string | undefined): string { + if (!description) return `${esc(label)}`; + const id = `help-${label.toLowerCase().replaceAll(' ', '-')}`; + return `` + + ``; +} + +export function pct(value: number | null | undefined): string { + return value == null ? DASH : `${Math.round(value)}%`; +} + +export function num(value: number | null | undefined): string { + return value == null ? DASH : String(Math.round(value)); +} + +// One value: the count and the total it is out of. +export function ratio(used: number | null | undefined, budget: number | null | undefined): string { + if (used == null) return DASH; + return budget == null ? String(used) : `${used} / ${budget}`; +} + +export function money(value: number | null | undefined): string { + if (value == null) return DASH; + return `$${value.toFixed(2)}`; +} + +export function spend(value: CostEvidence & { knownCostUsd?: number }, pending = false, liveSpend?: number): string { + return (liveSpend !== undefined + ? `~${money(liveSpend)}` + : value.status === 'unknown' ? value.knownCostUsd + ? `${money(value.knownCostUsd)} recorded` : 'Unknown' + : `${value.status === 'upper-bound' ? '≤' : ''}${money(value.costUsd)}`) + + (pending ? ' ' : ''); +} + +export function duration(seconds: number | null | undefined): string { + if (seconds == null) return DASH; + const minutes = Math.round(seconds / 60); + return minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + +// Wall time for the current execution, separate from the measured run duration. +export function elapsed(startedAt: string | null, completedAt: string | null, + now = Date.now()): string { + if (startedAt === null) return DASH; + const start = Date.parse(startedAt); + const end = completedAt === null ? now : Date.parse(completedAt); + if (!Number.isFinite(start) || !Number.isFinite(end)) return DASH; + const seconds = Math.floor(Math.max(0, end - start) / 1000); + const minutes = Math.floor(seconds / 60); + return `${minutes >= 60 ? `${Math.floor(minutes / 60)}h ` : ''}${minutes % 60}m ${seconds % 60}s`; +} + +export function executionClock(startedAt: string | null, completedAt: string | null): string { + return `${elapsed(startedAt, completedAt)}`; +} + +export function since(value: string | null | undefined, now = Date.now()): string { + if (!value) return DASH; + const minutes = Math.max(0, Math.floor((now - Date.parse(value)) / 60000)); + if (minutes < 60) return `${minutes}m`; + if (minutes < 60 * 48) return `${Math.floor(minutes / 60)}h`; + return `${Math.floor(minutes / 1440)}d`; +} + +export function phrase(attempt: SheetAttempt, now = Date.now()): string { + const parts = [attempt.phase]; + const silent = outputSilentMinutes(attempt, now); + if (silent >= SILENCE_MINUTES) parts.push(`no agent activity observed for ${silent}m`); + return parts.join(' · '); +} + +// depth 3 · 1× / L1–L3 · 3× +export function shape(mode: string, levels: readonly number[], repetitions: number): string { + const depth = levels.length ? Math.max(...levels) : 0; + const span = mode === 'dependency' ? `depth ${depth}` + : levels.length > 1 ? `L${Math.min(...levels)}–L${depth}` : `L${depth}`; + return `${span} · ${repetitions}×`; +} + +export function modelLabel(model?: string): string { + return ({ 'claude-sonnet-5': 'Sonnet 5', 'claude-fable-5-1': 'Fable 5.1', 'claude-opus-5': 'Opus 5', 'gpt-5.6-sol': 'Sol', 'gpt-6-astra': 'Astra' } as Record)[model ?? ''] ?? model ?? ''; +} + +// A finished attempt that does not count shows why instead of its completion. +export function excludedLabel(attempt: Pick): string | null { + if (!attempt.excluded || attempt.status === 'running' || attempt.status === 'pending') return null; + return attempt.status === 'completed' ? 'Excluded' : 'Incomplete'; +} + +export function completionLabel(attempt: Pick, + completion: Pick, 'passed' | 'selected'> | null = attempt.completion): string { + return excludedLabel(attempt) ?? (completion ? ratio(completion.passed, completion.selected) : DASH); +} + +export function runLabel(attempt: Pick, showRepetition = true): string { + return `${modelLabel(attempt.model)}${attempt.effort ? ` (${attempt.effort})` : ''}${showRepetition ? ` · Rep ${attempt.repetition}` : ''}`; +} diff --git a/tools/stack-bench/dashboard/public/graph.ts b/tools/stack-bench/dashboard/public/graph.ts new file mode 100644 index 00000000000..1c130c4a51a --- /dev/null +++ b/tools/stack-bench/dashboard/public/graph.ts @@ -0,0 +1,95 @@ +// One graph for the campaign: columns are depth, bands are questlines, edges +// are the catalog's own dependencies. Every stack builds the same catalog, so a +// node carries one dot per stack in fixed order. The renderer takes one +// node-status snapshot per stack, which is what the replay feeds it per step. + +import type { CampaignProgression } from '../dashboard-views.js'; +import { esc, stackLabel, statusWord } from './format.js'; + +export interface GraphStack { + stack: string; + statuses: readonly string[]; +} + +const DOT: Record = { passed: 'p', active: 'a', working: 'a', failed: 'f', + blocked: 'b', locked: 'o' }; +const DOT_START = 180; +const DOT_SPACING = 14; +const ROW = 30; + +interface Placed { + x: number; + y: number; + index: number; +} + +export function graph(view: CampaignProgression, stacks: readonly GraphStack[]): string { + if (!view.nodes.length) return '

No feature graph is available.

'; + const depths = view.depths; + const nodeWidth = DOT_START + Math.max(1, stacks.length) * DOT_SPACING; + const columnWidth = nodeWidth + 60; + const width = 150 + Math.max(1, depths.length) * columnWidth - 40; + const placed = new Map(); + const bands: string[] = []; + let top = 20; + for (const questline of view.questlines) { + const nodes = view.nodes.filter(node => node.questline === questline.id); + if (!nodes.length) continue; + const used = new Map(); + let rows = 0; + for (const node of nodes) { + const row = used.get(node.depth) ?? 0; + used.set(node.depth, row + 1); + rows = Math.max(rows, row + 1); + placed.set(node.id, { x: 150 + Math.max(0, depths.indexOf(node.depth)) * columnWidth, + y: top + 8 + row * ROW, index: view.nodes.indexOf(node) }); + } + const height = rows * ROW + 16; + bands.push(`${esc(questline.title.length > 19 ? `${questline.title.slice(0, 18)}…` : questline.title)}${esc(questline.title)}`); + top += height; + bands.push(``); + } + const failed = (index: number): boolean => + stacks.some(entry => entry.statuses[index] === 'failed'); + const blocked = (index: number): boolean => + stacks.some(entry => entry.statuses[index] === 'blocked'); + const edges = view.nodes.flatMap(node => { + const target = placed.get(node.id); + if (!target) return []; + return node.dependencies.flatMap(id => { + const source = placed.get(id); + if (!source) return []; + const cut = failed(source.index) || blocked(target.index); + return [``]; + }); + }); + const nodes = view.nodes.map(node => { + const at = placed.get(node.id); + if (!at) return ''; + const dots = stacks.map((entry, column) => + `` + + (entry.statuses[at.index] === 'passed' + ? `` : '')).join(''); + const hover = stacks.map(entry => + `${stackLabel(entry.stack)} ${statusWord(entry.statuses[at.index] ?? 'locked')}`).join(' · '); + return `${esc(`${node.title} · ${hover}`)}` + + `` + + `${esc(node.title.length > 22 + ? `${node.title.slice(0, 21)}…` : node.title)}${dots}`; + }); + const columns = depths.map((depth, index) => + `depth ${depth}`).join(''); + const order = stacks.map(entry => stackLabel(entry.stack)).join(', '); + const description = view.nodes.map((node, index) => `${node.title}: ${stacks.map(entry => + `${stackLabel(entry.stack)} ${statusWord(entry.statuses[index] ?? 'locked')}`).join(', ')}`).join('. '); + const key = [['p', 'Passed'], ['a', 'Active'], ['f', 'Failed'], ['b', 'Blocked'], ['o', 'Locked']] + .map(([tone, label]) => `${label}`).join(''); + return `
${stacks.length > 1 ? `

Feature dots, left to right: ${esc(order)}.

` : ''}${key}
` + + `
` + + `` + + `Feature dependencies and stack status${esc(description)}` + + `${bands.join('')}${columns}${edges.join('')}${nodes.join('')}
`; +} diff --git a/tools/stack-bench/dashboard/public/index.html b/tools/stack-bench/dashboard/public/index.html new file mode 100644 index 00000000000..e94a45b9a35 --- /dev/null +++ b/tools/stack-bench/dashboard/public/index.html @@ -0,0 +1,15 @@ + + + + + + + + Stack Bench + + + + + +
Loading Stack Bench…
+ diff --git a/tools/stack-bench/dashboard/public/metrics.ts b/tools/stack-bench/dashboard/public/metrics.ts new file mode 100644 index 00000000000..8804322b895 --- /dev/null +++ b/tools/stack-bench/dashboard/public/metrics.ts @@ -0,0 +1,203 @@ +import type { CampaignRunLevelResult, CampaignRunResult, DependencyProgress } + from '../../src/campaigns/campaign-inspection.js'; +import type { CostEvidence } from '../../src/evidence/cost-proof.js'; +import type { CheckCompletion } from '../../src/evidence/check-completion.js'; + +// The dashboard's vocabulary in one place: First builds, Score, Repairs, Regressions, +// Stalling and Excluded are defined here and nowhere else, so the server-rendered +// sheet and the browser read the same numbers from the same evidence. + +const EXCLUDED_OUTCOMES = new Set(['harness_failure', 'inconclusive', 'ungraded', 'contaminated']); +const SILENCE_MINUTES = 10; + +export interface MetricExecution { + outcome: string | null; + reason: string | null; +} + +export interface MetricAttempt { + id: string; + stack: string; + status: string; + repetition?: number; + logUpdatedAt?: string | null; + activityUpdatedAt?: string | null; + paused?: boolean; + execution: MetricExecution | null; + result: CampaignRunResult | null; + dependency: DependencyProgress | null; + spend?: CostEvidence; + measuredCost?: CostEvidence; + measuredDurationSec?: number | null; + completion?: CheckCompletion | null; + comparisonKey?: string; +} + +export interface AttemptMetrics { + first: number | null; + final: number; + repairs: number; + spend: number | null; + duration: number | null; + scope: string; + abortedFirst: number; + raw: { + first: { score: number; max: number } | null; + final: { score: number; max: number } | null; + }; +} + +export interface ComparisonEntry { + stack: string; + runs: Array<{ attempt: Attempt; metrics: AttemptMetrics }>; + excluded: Array<{ attempt: Attempt; reason: string }>; + pending: number; + spendSoFar: number | null; + abortedFirst: number; +} + +export type ComparisonRow = ComparisonEntry & { + n: number; scopes: string[]; first: number | null; final: number | null; + repairs: number | null; spend: number | null; duration: number | null; + costPerValidRun: number | null; + firstRange: { min: number; max: number } | null; + spendRange: { min: number; max: number } | null; + durationRange: { min: number; max: number } | null; +}; + +export function median(values: readonly number[]): number | null { + if (!values.length) return null; + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2; +} + +export function attemptSpend(attempt: MetricAttempt): number | null { + return attempt.spend?.status === 'exact' ? attempt.spend.costUsd : null; +} + +// An ungraded first build has no score; it is not a zero. +export function attemptMetrics(attempt: MetricAttempt): AttemptMetrics | null { + const run = attempt.result; + if (!run || run.unreadable) return null; + const dependency = attempt.dependency; + if (dependency) { + const score = dependency.score; + const unique = score?.uniqueChecks; + if (score?.status !== 'final' || unique?.percentage == null) return null; + const available = unique.availablePoints ?? 0; + return { + first: run.firstBuildRate ?? null, + // Final completion counts each selected graph point once. First builds + // instead sum the cumulative scored scope at each depth. + final: unique.percentage / 100, + repairs: dependency.history?.repairAttempts ?? 0, + spend: attempt.measuredCost?.status === 'exact' ? attempt.measuredCost.costUsd : null, + duration: attempt.measuredDurationSec ?? null, + scope: `${attempt.comparisonKey ?? ''}:dependency:${dependency.nodes.length}:${available}`, + abortedFirst: 0, + raw: { first: null, final: unique.passedPoints == null + ? null : { score: unique.passedPoints, max: available } }, + }; + } + type FinalLevel = CampaignRunLevelResult & { finalScore: { score: number; max: number } }; + type ScoredLevel = FinalLevel & { firstScore: { score: number; max: number } }; + const levels = (run.levels ?? []) + .filter((level): level is FinalLevel => level.finalScore !== null); + if (!levels.length) return null; + const sum = (list: readonly Level[], + pick: (level: Level) => number): number => list.reduce((total, item) => total + pick(item), 0); + const scored = levels.filter((level): level is ScoredLevel => + level.firstScore !== null && level.firstAbort === null); + const abortedFirst = levels.filter(level => level.firstAbort).length; + const firstMax = sum(scored, level => level.firstScore.max); + const finalMax = sum(levels, level => level.finalScore.max) + (run.unreachedPoints ?? 0); + return { + first: run.firstBuildRate ?? null, + final: sum(levels, level => level.finalScore.score) / finalMax, + repairs: sum(levels, level => level.used ?? 0), + spend: attempt.measuredCost?.status === 'exact' ? attempt.measuredCost.costUsd : null, + duration: attempt.measuredDurationSec ?? null, + scope: `${attempt.comparisonKey ?? ''}:sequential`, + abortedFirst, + // Do not show a partial first-build sum when the complete rate is unknown. + raw: { first: run.firstBuildRate != null && firstMax ? { score: sum(scored, l => l.firstScore.score), max: firstMax + (run.unreachedPoints ?? 0) } : null, + final: { score: sum(levels, l => l.finalScore.score), max: finalMax } }, + }; +} + +export function attemptExcluded(attempt: MetricAttempt): string | null { + const outcome = attempt.execution?.outcome ?? attempt.result?.outcome; + if (attempt.status === 'invalid') return attempt.execution?.reason ?? outcome ?? 'excluded'; + if (attempt.result?.unreadable && attempt.status !== 'running') return `Result validation failed: ${attempt.result.unreadable}`; + // 'ungraded' on an attempt still running means "not yet", not "thrown out". + if (outcome && EXCLUDED_OUTCOMES.has(outcome) && attempt.status === 'completed') return outcome; + return null; +} + +// Compare results only when they share the same recorded test plan. +export function compareCampaign(campaign: { + attempts?: readonly Attempt[]; +}): { rows: Array>; usable: Array>; + priced: Array>; burn: Map; + mixedScope: boolean; comparable: boolean } { + const byStack = new Map>(); + const unknownSpend = new Set(); + for (const attempt of campaign.attempts ?? []) { + const entry = byStack.get(attempt.stack) + ?? { stack: attempt.stack, runs: [], excluded: [], pending: 0, spendSoFar: null, abortedFirst: 0 }; + byStack.set(attempt.stack, entry); + // Excluded attempts still contribute to actual spend. + const incurred = attemptSpend(attempt); + if (incurred === null) unknownSpend.add(attempt.stack); + else entry.spendSoFar = (entry.spendSoFar ?? 0) + incurred; + const reason = attemptExcluded(attempt); + if (reason) { entry.excluded.push({ attempt, reason }); continue; } + const metrics = attempt.status === 'completed' ? attemptMetrics(attempt) : null; + if (metrics) { + entry.runs.push({ attempt, metrics }); + entry.abortedFirst += metrics.abortedFirst; + } else entry.pending += 1; + } + const rows = [...byStack.values()] + .map(entry => { + const pick = (key: 'first' | 'final' | 'repairs' | 'spend' | 'duration'): number[] => + entry.runs.map(run => run.metrics[key]).filter((value): value is number => value !== null); + const range = (values: readonly number[]): { min: number; max: number } | null => + values.length ? { min: Math.min(...values), max: Math.max(...values) } : null; + const spend = pick('spend'); + const duration = pick('duration'); + const first = pick('first'); + const scopes = [...new Set(entry.runs.map(run => run.metrics.scope))].sort(); + return { ...entry, spendSoFar: unknownSpend.has(entry.stack) ? null : entry.spendSoFar, + n: entry.runs.length, scopes, + first: scopes.length === 1 ? median(first) : null, firstRange: range(first), + final: scopes.length === 1 ? median(pick('final')) : null, + repairs: scopes.length === 1 ? median(pick('repairs')) : null, + spend: scopes.length === 1 ? median(spend) : null, spendRange: scopes.length === 1 ? range(spend) : null, + costPerValidRun: scopes.length === 1 && spend.length > 0 && spend.length === entry.runs.length + ? spend.reduce((total, cost) => total + cost, 0) / spend.length : null, + duration: scopes.length === 1 ? median(duration) : null, + durationRange: scopes.length === 1 ? range(duration) : null }; + }); + const usable = rows.filter(row => row.n > 0); + const scopes = new Set(usable.flatMap(row => row.scopes)); + const priced = usable.filter(row => row.spend != null); + return { rows, usable, priced, + burn: new Map(rows.map(entry => [entry.stack, entry.spendSoFar])), + mixedScope: scopes.size > 1, + comparable: priced.length > 1 && scopes.size === 1 }; +} + +export function outputSilentMinutes(attempt: Pick, now = Date.now()): number { + if (attempt.status !== 'running' || attempt.paused || !attempt.activityUpdatedAt) return 0; + const updated = Date.parse(attempt.activityUpdatedAt); + return Number.isFinite(updated) ? Math.max(0, Math.floor((now - updated) / 60000)) : 0; +} + +// Flag observed agent inactivity, never infer it from controller output or scores. +export function attemptStalling(attempt: Pick, + now = Date.now()): boolean { + return outputSilentMinutes(attempt, now) >= SILENCE_MINUTES; +} diff --git a/tools/stack-bench/dashboard/public/progress-chart.ts b/tools/stack-bench/dashboard/public/progress-chart.ts new file mode 100644 index 00000000000..086788215b7 --- /dev/null +++ b/tools/stack-bench/dashboard/public/progress-chart.ts @@ -0,0 +1,135 @@ +import type { CampaignProgression, CampaignSheet } from '../dashboard-views.js'; +import { duration, esc, runLabel, stackLabel } from './format.js'; + +export function progressChart(sheet: CampaignSheet, progression: CampaignProgression | null, + metric: 'completion' | 'cost' | 'distribution' = 'completion', view = 'grid', hidden: ReadonlySet = new Set(), unit: 'checks' | 'features' = 'features'): string { + const tracks = metric === 'distribution' ? sheet.stacks.flatMap(stack => stack.attempts.flatMap(attempt => { + const rate = unit === 'features' ? attempt.featureCompletion?.rate : attempt.completion?.rate; + return attempt.status === 'completed' && !attempt.excluded && rate != null && Number.isFinite(rate) + ? [{ stack: stack.stack, attempt, points: [{ elapsed: 0, value: rate * 100, upper: false }] }] : []; + })) : sheet.stacks.flatMap(stack => stack.attempts.map(attempt => + progression?.stacks.find(track => track.attemptId === attempt.id) + ?? { stack: stack.stack, attemptId: attempt.id, steps: [], costs: [], liveCosts: undefined })).flatMap(track => { + const attempt = sheet.stacks.find(stack => stack.stack === track.stack)?.attempts + .find(candidate => candidate.id === track.attemptId); + const start = Date.parse(attempt?.executionStartedAt ?? ''); + if (!attempt || !Number.isFinite(start)) return []; + const observations = (metric === 'cost' ? (track.liveCosts?.map(point => ({ + completedAt: point.completedAt, value: point.costUsd, upper: false, + })) ?? (track.costs ?? []).map(point => ({ + completedAt: point.completedAt, value: point.cost.costUsd, upper: point.cost.status === 'upper-bound', + }))) : track.steps.map(step => ({ completedAt: step.completedAt, + value: (unit === 'features' ? step.featureCompletion : step.completion) == null ? null + : (unit === 'features' ? step.featureCompletion! : step.completion!) * 100, upper: false }))).flatMap(step => { + const elapsed = (Date.parse(step.completedAt ?? '') - start) / 1000; + return Number.isFinite(elapsed) && elapsed >= 0 && step.value != null + && Number.isFinite(step.value) && step.value >= 0 + ? [{ elapsed, value: step.value, upper: step.upper }] : []; + }).sort((a, b) => a.elapsed - b.elapsed); + if (metric === 'cost' && attempt.executionCost) { + const liveTotal = track.liveCosts?.at(-1)?.costUsd; + const total = liveTotal ?? attempt.executionCost.costUsd; + const end = Date.parse(attempt.executionCompletedAt ?? attempt.activityUpdatedAt ?? attempt.logUpdatedAt ?? ''); + const elapsed = (end - start) / 1000; + if (total != null && Number.isFinite(total) && Number.isFinite(elapsed) && elapsed >= 0) { + // The sheet owns the total; histories can arrive in a different refresh. + while (observations.length && observations.at(-1)!.elapsed >= elapsed) observations.pop(); + observations.push({ elapsed, value: total, upper: liveTotal === undefined && attempt.executionCost.status === 'upper-bound' }); + } + } + const points = [{ elapsed: 0, value: 0, upper: false }, ...observations]; + return observations.length ? [{ stack: track.stack, attempt, points }] : []; + }); + const unitDescription = unit === 'features' + ? 'Features fully passed out of all selected features. A feature passes only when all its selected checks pass, including production guarantees.' + : 'Accepted checks passed out of all selected checks.'; + const label = metric === 'distribution' ? 'Completion distribution' : metric === 'cost' ? 'Cost' : 'Completion'; + const description = metric === 'distribution' + ? `One point per eligible completed run, grouped by stack. ${unitDescription} Running and excluded runs are not plotted.` + : metric === 'cost' + ? 'Current-execution cost, including repairs and excluded runs. Earlier executions remain in Total spend; Cost per valid run also includes explicit resume history. Live estimates use reported response usage; final receipts replace estimates. Other runs show saved grade checkpoints. Subscription costs use the pinned API-equivalent price snapshot, not invoice charges. Unknown costs are not plotted; upper bounds are labelled. Time starts at the current execution. Lines connect observations; intermediate values are not measured.' + : `${unitDescription} Each point is a saved grade. Zero marks run start. Each line is one repetition; elapsed time starts at that run. Excluded runs are labelled. Lines can fall after regressions. Intermediate values are not measured.`; + const heading = `

${label}${metric === 'distribution' ? '' : ' over time'}

' + + '' + + '
'; + const valueLabel = (value: number, upper = false, decimals = 1) => metric === 'cost' + ? `${upper ? '≤' : ''}$${value.toFixed(2)}` : `${value.toFixed(decimals)}%`; + const brandColors: Record = { spacetime: '#4cf490', mongodb: '#b45af2', postgres: '#336791' }; + const color = (stack: string) => brandColors[stack] + ?? `hsl(${Array.from(stack).reduce((hash, char) => (hash * 31 + char.charCodeAt(0)) % 360, 0)},65%,65%)`; + const marker = (repetition: number, x: number, y: number, title = '') => { + const shape = (repetition - 1) % 3; + return shape === 1 ? `${title}` + : shape === 2 ? `${title}` + : `${title}`; + }; + const controls = `
` + sheet.stacks.map(stack => { + const shown = stack.attempts.filter(attempt => !hidden.has(attempt.id)).length; + return `
` + + `` + + '
' + stack.attempts.map(attempt => { + const point = tracks.find(track => track.attempt.id === attempt.id)?.points.at(-1); + const label = runLabel(attempt, sheet.repetitions > 1) + + (point ? ` · ${metric === 'cost' && attempt.liveSpend !== undefined ? '~' : ''}${valueLabel(point.value, point.upper, 0)}` : ''); + const status = attempt.excluded || attempt.status === 'invalid' ? ' · Excluded' : ''; + return ``; + }).join('') + '
'; + }).join('') + '
'; + const visible = tracks.filter(track => !hidden.has(track.attempt.id)); + if (metric === 'distribution') { + const axisLeft = 150; + const position = (value: number) => axisLeft + (910 - axisLeft) * value / 100; + const rowHeight = Math.max(70, ...sheet.stacks.map(stack => stack.attempts.length * 20 + 24)); + const bottom = sheet.stacks.length * rowHeight + 24; + const ticks = [0, 25, 50, 75, 100].map(value => + `` + + `${value}%`).join(''); + const rows = sheet.stacks.map((stack, index) => { + const center = 24 + rowHeight * (index + 0.5); + const points = visible.filter(track => track.stack === stack.stack); + return `${esc(stackLabel(stack.stack))}` + + points.map(track => { + const value = track.points[0]!.value; + const at = center + (stack.attempts.findIndex(a => a.id === track.attempt.id) - (stack.attempts.length - 1) / 2) * 20; + const label = `${stackLabel(stack.stack)} · ${runLabel(track.attempt)}: ${valueLabel(value)}${track.attempt.excluded ? ' · Excluded' : ''}`; + return `` + + `${esc(label)}` + + marker(track.attempt.repetition, position(value), at) + + `${esc(runLabel(track.attempt))}` + + `${valueLabel(value, false, 0)}`; + }).join(''); + }).join(''); + return `
${heading}${controls}
` + + `${description}${ticks}${rows}
`; + } + if (!visible.length) return `
${heading}${controls}

${tracks.length > 0 && tracks.every(track => hidden.has(track.attempt.id)) ? 'Select a run to show its progress.' : metric === 'cost' ? 'Awaiting first timed cost receipt.' : 'Awaiting first timed grade.'}

`; + const ceiling = metric === 'cost' ? Math.max(0.01, ...tracks.flatMap(track => track.points.map(point => point.value))) : 100; + const maximum = Math.max(60, ...tracks.flatMap(track => track.points.map(point => point.elapsed))); + const left = metric === 'cost' ? 80 : 48; + const x = (seconds: number) => left + (948 - left) * seconds / maximum; + const y = (value: number) => 190 - 160 * value / ceiling; + const grid = [0, 0.25, 0.5, 0.75, 1].map(part => part * ceiling).map(value => + `${valueLabel(value, false, 0)}`).join(''); + const ticks = [0, 0.25, 0.5, 0.75, 1].map(part => + `${esc(part ? duration(part * maximum) : '0')}`).join(''); + const lines = visible.map(({ stack, attempt, points }) => { + // Saved observations are not continuous measurements. Stop at the last receipt. + let path = ''; + const marks = points.map((point, index) => { + path += index ? ` L${x(point.elapsed)} ${y(point.value)}` : `M${x(point.elapsed)} ${y(point.value)}`; + return marker(attempt.repetition, x(point.elapsed), y(point.value), `${esc(stackLabel(stack))} · ${esc(runLabel(attempt))}: ${valueLabel(point.value, point.upper)} at ${esc(duration(point.elapsed))}${index === 0 ? (metric === 'cost' ? ' · Run start; no recorded cost' : ' · Run start; no checks graded') : ''}${attempt.excluded ? ' · Excluded' : ''}`); + }).join(''); + return `${marks}`; + }).join(''); + return `
${heading}${controls}
` + + `${description}${grid}${ticks}${lines}Elapsed run time
` + + '
'; +} diff --git a/tools/stack-bench/dashboard/public/spacetimedb-mark.svg b/tools/stack-bench/dashboard/public/spacetimedb-mark.svg new file mode 100644 index 00000000000..f7957efa1ed --- /dev/null +++ b/tools/stack-bench/dashboard/public/spacetimedb-mark.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/tools/stack-bench/dashboard/public/styles.css b/tools/stack-bench/dashboard/public/styles.css new file mode 100644 index 00000000000..c0445b6ab89 --- /dev/null +++ b/tools/stack-bench/dashboard/public/styles.css @@ -0,0 +1,417 @@ +/* Stack Bench dashboard: compact tables, sentence-case labels, and consistent spacing. + * Brand colors identify stacks in charts. Status: pulsing green active, static green checks passed, + * red failed, yellow warning, gray pending. Completion alone is not success. */ + +@font-face { + font-family: 'Inter Variable'; + font-style: normal; + font-display: swap; + font-weight: 100 900; + src: url(/fonts/inter-latin-variable.woff2) format('woff2-variations'); +} +@font-face { + font-family: 'Source Code Pro Variable'; + font-style: normal; + font-display: swap; + font-weight: 200 900; + src: url(/fonts/source-code-pro-latin-variable.woff2) format('woff2-variations'); +} + +:root { + --green: #4cf490; --green-25: #4cf49040; --green-10: #4cf4901a; + --active: var(--green); --active-ring: var(--green-25); + --yellow: #fbdc8e; --yellow-25: #fbdc8e40; --yellow-10: #fbdc8e1a; + --red: #ff4c4c; + --n1: #e6e9f0; --n2: #ced3e0; --n3: #b6c0cf; --n4: #8d98a5; --n5: #363840; --n7: #050505; + --shade1: #162d38; --shade4: #121e24; --shade5: #0f191f; --shade6: #0e161a; + --shade7: #0b1114; --shade8: #0b0e12; + --sans: 'Inter Variable', Inter, ui-sans-serif, system-ui, -apple-system, 'Segoe UI', sans-serif; + --mono: 'Source Code Pro Variable', 'Source Code Pro', ui-monospace, SFMono-Regular, Consolas, monospace; + color-scheme: dark; +} + +* { box-sizing: border-box; } +body { margin: 0; background: var(--shade7); color: var(--n3); font: 14px/1.5 var(--sans); } +a { color: var(--n1); } +:focus-visible { outline: 2px solid var(--green); outline-offset: 2px; } +::selection { background: var(--green-25); } + +.topbar { display: flex; align-items: center; gap: 22px; height: 48px; padding: 0 24px; border-bottom: 1px solid var(--shade4); } +.brand { display: flex; align-items: center; gap: 10px; color: var(--n1); text-decoration: none; } +.brand b { font: 600 12px/1 var(--mono); letter-spacing: .1em; } +.btn { display: inline-flex; align-items: center; min-height: 36px; padding: 0 14px; border-radius: 4px; border: 1px solid var(--shade1); color: var(--n1); background: transparent; font: 500 13px/1 var(--sans); cursor: pointer; list-style: none; } +.nav { display: flex; gap: 2px; } +.nav a { padding: 8px 10px; border-radius: 4px; color: var(--n4); text-decoration: none; font: 500 12.5px var(--sans); } +.nav a.on { color: var(--n1); background: var(--shade5); } +.btn.primary { background: var(--green); border-color: var(--green); color: var(--n7); text-decoration: none; } +.tools { display: flex; align-items: center; gap: 10px; margin-left: auto; } +.files { position: relative; z-index: 10; } +.files summary::-webkit-details-marker { display: none; } +.files div { position: absolute; right: 0; top: 36px; display: grid; gap: 2px; padding: 8px 10px; background: var(--shade6); border: 1px solid var(--shade1); border-radius: 4px; } +.files div a { padding: 6px 2px; white-space: nowrap; color: var(--n2); font: 12px var(--mono); text-decoration: none; } +.page { max-width: 1600px; margin-inline: auto; padding: 22px 24px 40px; } +.crumbs { color: var(--n4); font: 12px var(--mono); margin-bottom: 8px; } +.crumbs a { color: var(--n4); text-decoration: none; } +.crumbs b { color: var(--n2); font-weight: 500; } +.title { flex-wrap: wrap; display: flex; align-items: center; gap: 12px; margin-bottom: 16px; } +.title h2 { margin: 0; color: var(--n1); font: 600 25px/28px var(--sans); letter-spacing: -.01em; } +.title h2 span { color: var(--n4); font-weight: 400; } +.label { color: var(--n4); font: 500 12px/1.4 var(--sans); } +.state { color: var(--n4); font: 500 13px var(--sans); white-space: nowrap; } +.state.run { color: var(--active); } +.state.done { color: var(--n2); } +.state.warn { color: var(--yellow); } +.state.idle { color: var(--n4); } + +/* live lanes: stack, score, climb, phase */ +.live { border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade6); margin-bottom: 18px; } +.live-head { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; min-height: 44px; padding: 8px 16px; border-bottom: 1px solid var(--shade4); } +.live-head b { color: var(--n1); font-size: 15px; font-weight: 600; } +.lane { display: grid; grid-template-columns: minmax(220px, 1.6fr) minmax(80px, .4fr) minmax(100px, .5fr) minmax(180px, 1fr); gap: 20px; align-items: center; padding: 14px 16px; border-top: 1px solid var(--shade4); } +.lane > * { min-width: 0; } +.lane-repetition { white-space: nowrap; } +.lane-variant { margin-top: 5px; font-size: 12px; overflow-wrap: anywhere; color: var(--n2); } +.lane-metric { display: grid; gap: 6px; font-variant-numeric: tabular-nums; } +.lane-label { font-size: 12px; color: var(--n2); } +.lane:first-of-type { border-top: 0; } +.lane .who { color: var(--n1); font-weight: 600; } +.lane .big { color: var(--n1); font: 600 26px/1 var(--sans); letter-spacing: -.02em; } +.lane .phase { font-size: 13px; overflow-wrap: anywhere; } +.lane .phase.warn { color: var(--yellow); } + +.reference-run { margin-bottom: 16px; border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade6); } +.reference-run > summary { display: grid; grid-template-columns: minmax(0, 1fr) auto auto; align-items: center; gap: 24px; padding: 14px 16px; cursor: pointer; list-style: none; } +.reference-run > summary::-webkit-details-marker { display: none; } +.reference-run > summary:hover { background: var(--shade5); } +.reference-identity { display: flex; flex-wrap: wrap; align-items: center; gap: 4px 12px; min-width: 0; } +.reference-identity b { color: var(--n1); font-weight: 600; overflow-wrap: anywhere; } +.reference-identity b::first-letter, .reference-identity .state { text-transform: capitalize; } +.reference-run small { display: block; color: var(--n4); font-size: 12px; font-weight: 400; } +.reference-identity small { flex-basis: 100%; } +.reference-score { white-space: nowrap; font-variant-numeric: tabular-nums; } +.reference-score strong { color: var(--n1); font-weight: 600; } +.reference-toggle { display: flex; align-items: center; gap: 10px; color: var(--n4); font-size: 12px; } +.reference-toggle span { font-size: 20px; line-height: 1; } +.reference-run[open] .reference-toggle span { transform: rotate(90deg); } +.reference-run > .log { margin: 0; border-top: 1px solid var(--shade4); border-radius: 0; } +@media (max-width: 600px) { + .reference-run > summary { grid-template-columns: minmax(0, 1fr) auto; gap: 12px; } + .reference-score { grid-column: 1; grid-row: 2; white-space: normal; } + .reference-toggle { grid-column: 2; grid-row: 1 / 3; } +} + +/* campaigns table */ +.tablewrap { border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade6); overflow: hidden; } +.toolbar { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; min-height: 44px; padding: 6px 12px; border-bottom: 1px solid var(--shade4); } +.chip { height: 24px; padding: 0 10px; border-radius: 4px; border: 1px solid var(--shade1); color: var(--n3); font: 500 12px/22px var(--sans); text-decoration: none; } +.chip.on { background: var(--shade1); color: var(--n1); } +.chip.sm { height: 24px; line-height: 22px; font-size: 12px; padding: 0 8px; } +.wrap { overflow-x: auto; } +table.runs { width: 100%; border-collapse: collapse; font-size: 13px; } +table.runs th, table.runs td { padding: 0 14px; height: 40px; text-align: left; vertical-align: middle; border-bottom: 1px solid var(--shade4); white-space: nowrap; } +table.runs thead th { color: var(--n4); font: 600 12px/1.4 var(--sans); height: 36px; } +table.runs tbody tr:last-child td { border-bottom: 0; } +table.runs tbody tr:hover td { background: var(--shade5); } +table.runs td.name a { color: var(--n1); font-weight: 600; text-decoration: none; } +table.runs td.shape { color: var(--n4); font: 12px var(--mono); } +table.runs td.stack { text-align: right; font: 13px var(--mono); font-variant-numeric: tabular-nums; color: var(--n1); width: 128px; } +table.runs th.stack:first-of-type, table.runs td.stack:first-of-type { border-left: 1px solid var(--shade4); } +table.runs td.stack.na { color: var(--n4); } +table.runs td.stack u { text-decoration-color: var(--green); text-underline-offset: 5px; text-decoration-thickness: 2px; } +table.runs th.when, table.runs td.when { color: var(--n4); font: 12px var(--mono); text-align: right; } +table.runs thead th.stack { text-align: right; } +table.runs thead th.when { color: var(--n4); font: 600 12px/1.4 var(--sans); } + +/* saved plans and shared controls */ +table.plans th.stack, table.plans td.stack { width: auto; } +table.plans td.name { color: var(--n1); font-weight: 600; min-width: 240px; max-width: 360px; white-space: normal; padding-block: 10px; overflow-wrap: anywhere; } +.secret input, .transcript-controls select, .transcript-controls button { height: 30px; padding: 0 10px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade7); color: var(--n1); font: 13px var(--mono); } +.secret { display: flex; align-items: center; gap: 8px; } +.secret input { width: 168px; } +.err { color: var(--red); font: 12.5px var(--sans); align-self: center; } + +/* campaign sheet: stacks across, facts down */ +.page > h3 { margin: 28px 0 12px; font-size: 15px; font-weight: 600; color: var(--n1); } +.facts { display: flex; flex-wrap: wrap; gap: 1px; margin: 0 0 16px; background: var(--shade4); border: 1px solid var(--shade4); border-radius: 4px; overflow: hidden; } +.facts div { flex: 1 1 168px; align-content: start; display: grid; gap: 5px; min-width: 0; padding: 9px 12px; background: var(--shade6); } +.facts b { color: var(--n2); font: 500 12px/1.5 var(--mono); overflow-wrap: anywhere; } +.sheet { width: 100%; border-collapse: collapse; background: var(--shade6); } +.sheet th, .sheet td { min-width: 185px; padding: 8px 16px; text-align: left; vertical-align: middle; border: 1px solid var(--shade4); } +.sheet th:first-child { min-width: 190px; width: 190px; } +.sheet .k { color: var(--n4); font: 500 13px/1.4 var(--sans); } +.sheet .h { height: 48px; color: var(--n1); font-size: 15px; font-weight: 600; } +.sheet .h a { text-decoration: none; } +.sheet .v { color: var(--n1); font: 13px var(--mono); font-variant-numeric: tabular-nums; } +.sheet .v i, .checks .group i { color: var(--n4); font-style: normal; margin-left: 6px; } +.sheet .big { color: var(--n1); font: 600 30px/1 var(--sans); letter-spacing: -.02em; } +.sheet .q { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; min-height: 36px; padding-block: 6px; } +.sheet .q.k { display: table-cell; text-transform: none; letter-spacing: 0; font: 12.5px var(--sans); color: var(--n3); } +.sheet .q .pct { margin-left: auto; color: var(--n4); font: 11.5px var(--mono); font-variant-numeric: tabular-nums; } +.sheet .q .pct.full { color: var(--green); } + +.evhead { display: flex; flex-wrap: wrap; gap: 0; padding: 0; } +.evhead .ev { display: grid; gap: 5px; padding: 8px 16px; border-left: 1px solid var(--shade4); min-height: 48px; min-width: 130px; } +.evhead .ev:first-child { border-left: 0; } +.dot { width: 9px; height: 9px; border-radius: 50%; background: var(--shade1); flex: 0 0 auto; } +.dot.p { position: relative; background: var(--green); } +.dot.p::after { content: ""; position: absolute; left: 3px; top: 1px; width: 2px; height: 4px; border: solid var(--shade7); border-width: 0 1.5px 1.5px 0; transform: rotate(45deg); } +.dot.a { background: var(--active); box-shadow: 0 0 0 2px var(--active-ring); } +.dot.f { background: var(--red); } +.dot.b { background: transparent; border: 1.5px solid var(--red); } +.dot.o { background: transparent; border: 1.5px solid var(--n4); } + +/* graph */ +.dag { display: block; height: auto; } +.dag .band { font: 11px var(--mono); fill: var(--n4); } +.dag .col { font: 500 10.5px var(--mono); fill: var(--n4); letter-spacing: .08em; text-transform: uppercase; } +.dag .sep { stroke: var(--shade4); } +.dag .e { fill: none; stroke: var(--shade1); stroke-width: 1.1; opacity: .8; } +.dag .e.cut { stroke: var(--red); stroke-dasharray: 3 4; opacity: .7; } +.dag .n rect { fill: var(--shade5); stroke: var(--shade1); } +.dag .n text { font: 12px var(--mono); fill: var(--n2); } +.dag .d { fill: var(--shade1); } +.dag .d.p { fill: var(--green); } +.dag .d.a { fill: var(--active); } +.dag .d.f { fill: var(--red); } +.dag .d.b { fill: none; stroke: var(--red); stroke-width: 1.4; } +.dag .d.o { fill: none; stroke: var(--n4); stroke-width: 1.4; } + +/* replay */ +.replay { min-height: 40px; padding: 6px 16px; } +.strip { display: block; width: 100%; height: 28px; } +.strip .st { fill: var(--n4); } +.strip .st.b, .strip .st.r { fill: var(--active); } +.strip .st.g { fill: var(--n5); } +.strip .st.f { fill: var(--red); } +.strip .st.on { stroke: var(--yellow-25); stroke-width: 3; } +.strip .st.dim { opacity: .3; } +.strip .cur { stroke: var(--n1); stroke-width: 1; } + +/* attempt */ +.figs { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 1px; margin: 4px 0 20px; border: 1px solid var(--shade4); border-radius: 4px; overflow: hidden; background: var(--shade4); } +.figs > div { display: grid; align-content: start; gap: 6px; min-width: 0; padding: 12px 14px; background: var(--shade6); } +.metric-label { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; min-height: 24px; } +.figs b { color: var(--n1); font: 600 20px/1.4 var(--sans); font-variant-numeric: tabular-nums; overflow-wrap: anywhere; } +.figs b.now { font: 500 14px/1.5 var(--sans); } +.figs b.now.warn { color: var(--yellow); } +.issue { margin: 0 0 20px; padding: 12px 14px; border-left: 2px solid var(--yellow); background: var(--yellow-10); } +.issue p { margin: 6px 0 0; color: var(--n2); } +.bigclimb { display: block; width: 100%; min-width: 1060px; height: auto; aspect-ratio: 1060 / 170; margin-bottom: 12px; } +.bigclimb text { font: 12px var(--mono); fill: var(--n4); } +.bigclimb .l { stroke: var(--green); stroke-width: 2; fill: none; stroke-linejoin: round; } +.bigclimb .a { fill: var(--green-10); } +.bigclimb .g { stroke: var(--shade4); } +.bigclimb .band { fill: var(--shade6); } +.bigclimb .ev { fill: var(--n1); } +.bigclimb .ev.first { fill: var(--shade7); stroke: var(--n3); stroke-width: 1.5; } +.bigclimb .ev.now { fill: var(--n1); } +.tabs { display: flex; gap: 2px; border-bottom: 1px solid var(--shade4); } +.tabs a { padding: 10px 12px; color: var(--n4); font: 500 13px var(--sans); border-bottom: 2px solid transparent; margin-bottom: -1px; text-decoration: none; } +.tabs a.on { color: var(--n1); border-bottom-color: var(--green); } +.tabs a i { color: var(--n4); font: 11.5px var(--mono); font-style: normal; margin-left: 6px; } +.checks { width: 100%; border-collapse: collapse; font-size: 13px; } +.checks th, .checks td { height: 34px; padding: 0 14px; text-align: left; border-bottom: 1px solid var(--shade4); white-space: nowrap; } +.checks thead th { color: var(--n4); font: 600 12px/1.4 var(--sans); height: 36px; } +.checks td.k { font-family: var(--mono); color: var(--n2); } +.checks td.d { color: var(--n3); white-space: normal; } +.checks td.h { color: var(--n4); font: 12px var(--mono); letter-spacing: .14em; } +.checks .h .p { color: var(--green); } +.checks .h .f { color: var(--red); } +.checks .h .x { color: var(--n4); } +.checks tr.group td { color: var(--n1); font-weight: 600; background: var(--shade6); } +.check-evidence { padding: 8px 0; min-width: 240px; } +.checks td:not(.d) { vertical-align: top; padding-top: 10px; } +.checks td.k { max-width: 180px; white-space: normal; overflow-wrap: anywhere; } +.check-evidence summary { cursor: pointer; } +.check-evidence summary:focus-visible { outline: 2px solid var(--n2); outline-offset: 3px; } +.check-evidence section { margin: 12px 0; padding-top: 12px; border-top: 1px solid var(--shade4); } +.check-evidence strong { font-size: 12px; } +.check-evidence p { margin: 8px 0; overflow-wrap: anywhere; } +.check-evidence pre { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 240px; overflow: auto; font: 12px/1.5 var(--mono); } +.grade-key { flex-wrap: wrap; display: flex; gap: 18px; padding: 10px 14px; color: var(--n4); font: 12px/1.6 var(--sans); } +.grade-key .p { color: var(--green); } +.grade-key .f { color: var(--red); } +.grade-key .x { color: var(--n4); } +.shots { display: grid; grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); gap: 12px; padding: 14px 0; } +.shots button { padding: 0; border: 0; background: none; cursor: zoom-in; } +.shots img { width: 100%; border: 1px solid var(--shade4); border-radius: 4px; } +.lightbox { width: min(94vw, 1500px); max-height: 94vh; padding: 42px 12px 12px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade8); } +.lightbox::backdrop { background: #000c; } +.lightbox form { position: absolute; top: 8px; right: 10px; } +.lightbox button { border: 0; background: none; color: var(--n2); cursor: pointer; } +.lightbox img { display: block; max-width: 100%; max-height: calc(94vh - 54px); margin: auto; } +.files-list { display: grid; gap: 6px; padding: 14px 0; } +.files-list a { color: var(--n2); font: 12px var(--mono); text-decoration: none; } +.log { margin: 14px 0 0; padding: 14px 16px; max-height: 520px; overflow: auto; background: var(--shade8); color: var(--n3); font: 12px/1.7 var(--mono); border-radius: 4px; } + +/* Local scrolling keeps labels readable without widening the page. */ +.crumbs, .title h2, .files-list a { overflow-wrap: anywhere; } +.sheet-scroll, .chart-scroll, .graph-scroll { max-width: 100%; overflow-x: auto; } +.chart-empty, .summary-note, .chart-caption { color: var(--n3); font: 12px/1.5 var(--sans); } +.chart-caption { margin: 0 0 16px; } +.graph-key { display: flex; flex-wrap: wrap; gap: 8px 18px; padding: 8px; font-size: 12px; } +.graph-key span { display: inline-flex; align-items: center; gap: 6px; } +.graph-key p { flex-basis: 100%; margin: 0; } +/* Secondary help stays out of the layout and above scrolling containers. */ +.metric-help { padding: 4px 0; border: 0; background: transparent; text-align: left; cursor: help; } +.metric-help:hover, .metric-help:focus-visible { color: var(--n1); } +.metric-tooltip { position: fixed; inset: auto; position-area: top span-right; position-try-fallbacks: flip-block, flip-inline; margin: 6px; width: min(260px, calc(100vw - 24px)); padding: 10px 12px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade5); color: var(--n2); font: 12px/1.5 var(--sans); letter-spacing: normal; text-transform: none; overflow-wrap: anywhere; } +.btn:hover, .chip:hover, .nav a:hover, .tabs a:hover { color: var(--n1); background: var(--shade1); } +.btn.primary:hover { color: var(--n7); background: var(--green); filter: brightness(1.08); } +.btn:disabled { opacity: .55; cursor: not-allowed; } +.btn:disabled:hover { filter: none; } +@media (max-width: 900px) { + .topbar { height: auto; min-height: 48px; flex-wrap: wrap; padding: 10px 16px; gap: 12px; } + .tools { flex-wrap: wrap; margin-left: 0; } + .page { padding: 18px 16px 32px; } + .lane { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; } + .lane-identity, .lane .phase { grid-column: 1 / -1; } + .figs { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .title h2 { font-size: 22px; } + .tabs { flex-wrap: wrap; } + .sheet-scroll::before { content: 'Scroll horizontally to compare stacks'; display: block; padding: 6px 0; color: var(--n4); font-size: 12px; } +} +@media (max-width: 420px) { + .facts div { flex-basis: calc(50% - 1px); } + .page { padding-inline: 12px; } + .secret { flex-wrap: wrap; } + .secret input { max-width: 100%; } +} + +.feature-progress { margin-block: 28px; } +.section-heading { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 12px; } +.section-heading h3 { margin: 0; color: var(--n1); font-size: 15px; font-weight: 600; } +.section-heading nav { display: flex; gap: 6px; } + +.explore { position: relative; } +.explore summary { cursor: pointer; color: var(--n3); font-size: 12px; padding: 6px 0; } +.explore nav { position: absolute; right: 0; z-index: 2; padding: 10px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade5); } +table.attempt-list { table-layout: fixed; min-width: 960px; } +table.attempt-list th:first-child { width: 16%; } +table.attempt-list th:nth-child(2) { width: 18%; } +table.attempt-list th:nth-child(3), table.attempt-list th:nth-child(4) { width: 10%; } +table.attempt-list th:nth-last-child(2) { width: 10%; } +table.attempt-list td { white-space: normal; overflow-wrap: anywhere; } +table.attempt-list .run-name { display: block; padding-block: 8px; color: var(--n1); text-decoration: none; } +table.attempt-list .run-effort { font-weight: 400; color: var(--n4); } +table.attempt-list .run-name:hover { text-decoration: underline; } +table.attempt-list .run-status { padding-block: 10px; } +.run-status summary { cursor: pointer; color: var(--yellow); } +.loading { display: flex; align-items: center; gap: 12px; min-height: 96px; color: var(--n2); font-size: 14px; } +.loading::before { content: ''; width: 16px; height: 16px; flex-shrink: 0; border: 2px solid var(--shade1); border-top-color: var(--n2); border-radius: 50%; animation: loading-spin .8s linear infinite; } +@keyframes loading-spin { to { transform: rotate(360deg); } } +main[aria-busy="true"]:not(:has(.loading))::before { content: ''; position: fixed; top: 0; left: 0; width: 30%; height: 2px; z-index: 10; background: var(--n2); animation: loading-progress 1.4s ease-in-out infinite; } +@keyframes loading-progress { from { transform: translateX(-100%); } to { transform: translateX(334%); } } +@media (prefers-reduced-motion: reduce) { main[aria-busy="true"]:not(:has(.loading))::before, .loading::before { animation: none; } } +[data-started-at] { font-variant-numeric: tabular-nums; white-space: nowrap; } +.time-grant { display: flex; flex-wrap: wrap; align-items: center; gap: .75rem; margin: 1rem 0; } +.time-grant label { display: flex; align-items: center; gap: .5rem; } +.time-grant input[type="number"] { width: 6rem; } + +.progress-heading { margin: 24px 0 8px; } +.progress-chart { display: block; width: 100%; min-width: 520px; max-height: 300px; } +.progress-chart text { fill: var(--n3); font: 12px var(--sans); } +.progress-grid { stroke: var(--n3); opacity: .15; } +.progress-panel { margin-bottom: 24px; } +.progress-controls { display: grid; grid-template-columns: max-content minmax(0, 1fr); gap: 6px 8px; margin: 12px 0; } +.progress-stack { display: grid; grid-template-columns: subgrid; grid-column: 1 / -1; align-items: start; } +.chart-runs { display: flex; flex-wrap: wrap; gap: 4px; min-width: 0; } +.progress-controls .chart-stack-toggle { max-width: 12rem; overflow-wrap: anywhere; } +.progress-controls .chart-run-toggle { min-width: 7.5rem; font-variant-numeric: tabular-nums; } +.progress-controls button { display: inline-flex; align-items: center; gap: 6px; min-height: 32px; padding: 4px 8px; border: 1px solid transparent; border-radius: 4px; background: transparent; color: var(--n2); font: 12px var(--sans); cursor: pointer; } +.progress-controls button:hover { background: var(--shade1); } +.progress-controls button:focus-visible { outline: 2px solid var(--n2); outline-offset: 2px; } +.progress-controls button[aria-pressed="false"] { color: var(--n4); } +.progress-controls button[aria-pressed="false"] svg, .progress-controls button[aria-pressed="false"] .chart-swatch { opacity: .25; } +.progress-controls .chart-run-toggle[aria-pressed="true"] { border-color: var(--shade1); background: var(--shade5); } +.chart-swatch { flex-shrink: 0; } +.chart-run-toggle svg { flex-shrink: 0; } +.progress-series.is-muted { opacity: .15; } +.progress-series.is-highlighted .progress-line { stroke-width: 3; } +.distribution-run-label { display: none; pointer-events: none; paint-order: stroke; stroke: var(--shade8); stroke-width: 4px; } +.progress-series.is-highlighted .distribution-run-label { display: block; } + +.transcript-controls { margin-top: 16px; display: flex; flex-wrap: wrap; align-items: center; gap: 12px; margin-bottom: 12px; } +.transcript-controls label { min-width: 0; flex: 1; } +.transcript-controls select { max-width: 100%; width: min(100%, 560px); } +.transcript { max-height: 65vh; overflow: auto; overflow-anchor: none; border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade8); padding: 16px; } +.transcript article, .transcript details { margin-bottom: 16px; } +.transcript pre { white-space: pre-wrap; overflow-wrap: anywhere; margin: 8px 0; font-size: 12px; line-height: 1.6; } +.transcript strong, .transcript summary { font-size: 12px; color: var(--n3); } + +.transcript-controls button { cursor: pointer; } +.transcript-controls button:hover { border-color: var(--green); } + +@media (max-width: 600px) { + .progress-chart { min-width: 640px; } + .progress-chart text { font-size: 16px; } +} + +.spend-pending { display: inline-block; width: 6px; height: 6px; vertical-align: middle; margin-left: 3px; } +.dot.a, .dag .d.a { animation: activity-pulse 2.8s ease-in-out infinite; } +@keyframes activity-pulse { 50% { opacity: .4; } } +@media (prefers-reduced-motion: reduce) { .dot.a, .dag .d.a { animation: none; } } + +.dag .passed-check { fill: none; stroke: var(--shade7); stroke-width: 1.2; } + +.chart-options { display: flex; flex-wrap: wrap; gap: 16px; align-items: center; } + +.chip[aria-disabled="true"] { opacity: .4; pointer-events: none; } + +.attempt-list tr.is-highlighted td, .chart-run-toggle.is-highlighted { background: var(--shade1); } + +/* Run setup uses native controls and one review step. */ +.setup{max-width:900px}.setup-form{display:grid;gap:24px}.setup-fields{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:16px}.setup label{display:flex;gap:8px}.setup-fields>label,.setup-actions>label{flex-direction:column}.setup label>span,.setup legend{color:var(--muted,#9eabb9);font-size:13px}.setup input:not([type=checkbox]),.setup select{min-width:0;background:#101c22;color:inherit;border:1px solid #29404b;border-radius:4px;padding:10px;font:inherit}.setup fieldset{border:1px solid #24383f;border-radius:5px;padding:16px}.setup-choices{display:flex;flex-wrap:wrap;gap:20px}.setup-model{display:flex;align-items:center;justify-content:space-between;gap:16px;padding:7px 0}.setup-model select{width:130px}.setup-actions{display:flex;align-items:end;justify-content:flex-end;gap:12px}.setup details>div{margin-top:16px}.setup-review{display:grid;grid-template-columns:160px 1fr;gap:14px}.setup-review dt{color:#9eabb9}.setup-review dd{margin:0}.setup .warning{border-left:3px solid #eabc65;padding:12px;background:#211e15}.setup pre{overflow:auto;max-height:260px}.setup .summary-note{margin:20px 0}@media(max-width:600px){.setup-fields{grid-template-columns:1fr}.setup-actions{align-items:stretch;flex-direction:column}.setup-review{grid-template-columns:1fr;gap:6px}.setup-review dd{margin-bottom:12px}} + +/* Definition guide: neutral panels; color labels distinguish work from assertions. */ +.guide { max-width: 1160px; } +.guide h1 { margin: 0; color: var(--n1); font-size: 28px; letter-spacing: -.02em; } +.guide-intro { margin: 8px 0 20px; font-size: 16px; color: var(--n2); } +.guide-help { margin-bottom: 20px; } +.guide summary { cursor: pointer; } +.guide-help > summary, .guide-evidence > summary { padding: 10px 0; color: var(--n2); font-weight: 500; } +.guide-key { overflow-wrap: anywhere; font: 12px/1.7 var(--mono); color: var(--n4); } +.guide-toolbar { position: sticky; top: 0; z-index: 2; background: var(--shade7); padding: 12px 0; border-bottom: 1px solid var(--shade1); } +.guide-search { display: grid; gap: 6px; color: var(--n2); font-weight: 500; } +.guide-search input { width: 100%; padding: 12px 14px; border: 1px solid var(--shade1); border-radius: 5px; background: var(--shade6); color: var(--n1); font: inherit; } +.guide-search input::placeholder { color: var(--n4); } +.guide-controls { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; margin-top: 12px; } +.guide-controls label { display: flex; align-items: center; gap: 6px; margin-right: auto; } +.guide-controls input { accent-color: var(--green); } +.guide-controls [role=status] { color: var(--n4); font-size: 12px; } +.guide-kind { font: 600 11px/1.6 var(--sans); letter-spacing: .015em; } +.guide-kind.action { color: #9ecbff; } +.guide-kind.assert { color: #87deb0; } +.guide-kind.wait { color: var(--yellow); } +.guide-kind.repeat { color: #c3b4fa; } +.guide-check { margin: 10px 0; border: 1px solid var(--shade1); border-radius: 6px; background: var(--shade6); overflow: hidden; } +.guide-check[hidden] { display: none; } +.guide-check > summary { display: flex; align-items: baseline; gap: 14px; padding: 17px 18px; color: var(--n1); font-weight: 500; list-style: none; } +.guide-check > summary::-webkit-details-marker { display: none; } +.guide-check > summary::before { content: '\203A'; color: var(--n4); font-size: 20px; line-height: 1; } +.guide-check[open] > summary::before { transform: rotate(90deg); } +.guide-check > summary:hover { background: var(--shade5); } +.guide-id { color: var(--n4); font: 12px var(--mono); min-width: 40px; } +.guide-scope { margin-left: auto; white-space: nowrap; color: var(--n4); font-size: 11px; } +.guide-body { padding: 0 22px 20px; border-top: 1px solid var(--shade1); } +.guide-body h2 { margin: 24px 0 10px; font-size: 15px; color: var(--n1); } +.guide-step { display: grid; grid-template-columns: 110px minmax(0, 1fr); gap: 16px; padding: 10px 0; border-bottom: 1px solid var(--shade4); line-height: 1.7; overflow-wrap: anywhere; } +.guide-step .guide-kind { padding-top: 3px; } +.guide-indent-1 { padding-left: 18px; } +.guide-indent-2 { padding-left: 36px; } +.guide-indent-3 { padding-left: 54px; } +.guide-indent-4 { padding-left: 72px; } +.guide-note { color: var(--n4); } +.guide-evidence { border-top: 1px solid var(--shade1); } +.guide-evidence li { margin: 8px 0; overflow-wrap: anywhere; } +.guide pre { padding: 16px; background: var(--shade8); border-radius: 4px; white-space: pre-wrap; overflow-wrap: anywhere; color: var(--n2); font: 12px/1.7 var(--mono); } +@media (max-width: 640px) { + body:has(.guide) .topbar { height: auto; min-height: 48px; padding: 12px 16px; flex-wrap: wrap; gap: 12px; } + .guide { padding-inline: 16px; } + .guide-toolbar { position: static; } + .guide-check > summary { gap: 10px; padding: 14px 12px; } + .guide-scope { display: none; } + .guide-body { padding-inline: 14px; } + .guide-step { grid-template-columns: minmax(0, 1fr); gap: 2px; } + .guide pre { padding: 10px; } +} diff --git a/tools/stack-bench/dashboard/public/views/attempt.ts b/tools/stack-bench/dashboard/public/views/attempt.ts new file mode 100644 index 00000000000..7ebb647574a --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/attempt.ts @@ -0,0 +1,207 @@ +import type { TranscriptPage } from '../../dashboard-transcript.js'; +// One attempt: figures, feature dependencies, and the evidence behind tabs. +// Each tab is a link, so what is open survives a reload and a back button. + +import type { AttemptCheck, AttemptChecks, AttemptPackage, CampaignProgression, CampaignSheet, SheetAttempt, SheetStack } + from '../../dashboard-views.js'; +import type { readCampaignTimeBudget } from '../../../src/campaigns/campaign-time-grant.js'; +import { graph } from '../graph.js'; +import { bigClimb } from '../climb.js'; +import { DASH, completionLabel, duration, executionClock, esc, metricLabel, spend, pct, phrase, ratio, stackLabel } from '../format.js'; + +export type AttemptTab = 'checks' | 'screenshots' | 'files' | 'log' | 'transcript'; + +export interface AttemptPageInput { + sheet: CampaignSheet; + progression?: CampaignProgression | null; + attemptId: string; + tab: AttemptTab; + checks: AttemptChecks | null; + evidence: AttemptPackage | null; + log: string; + transcript?: TranscriptPage | null; + timeBudget?: ReturnType | null; + canControl?: boolean; + controlError?: string; +} + +const GLYPH: Record = { pass: '✓', + fail: '✕', 'not-run': '·' }; + +function locate(sheet: CampaignSheet, attemptId: string): { + stack: SheetStack; + attempt: SheetAttempt; +} | null { + for (const stack of sheet.stacks) { + const attempt = stack.attempts.find(item => item.id === attemptId); + if (attempt) return { stack, attempt }; + } + return null; +} + +const CATEGORY = { feature: 'Feature', production: 'Production', interface: 'Interface', unknown: 'Unclassified' }; + +function checksTable(checks: AttemptChecks | null): string { + const errors = checks?.grades.filter(grade => grade.error).map(grade => + `

${esc(grade.id)}: ${esc(grade.error)}

`).join('') ?? ''; + if (!checks?.checks.length) return errors + '

No check results are recorded yet. Check the log for current work or an execution error.

'; + const features = new Map(); + for (const check of checks.checks) { + features.set(check.feature, [...features.get(check.feature) ?? [], check]); + } + const groups = [...features.entries()].map(([feature, items]) => { + return `${esc(feature)}` + + items.map(check => `${esc(check.id)}` + + `
` + + `${esc(check.description || check.id)}` + + check.observations.map((observation, index) => { + const grade = checks.grades[index]; + const label = `Grade ${index + 1}${grade?.level == null ? '' : ` · L${grade.level}`}`; + return `
${esc(label)} · ${esc(observation?.status ?? 'NO RESULT')}` + + (observation?.summary ? `

${esc(observation.summary)}

` : '') + + (observation?.expected != null ? `
Expected
${esc(observation.expected)}
` : '') + + (observation?.actual != null ? `
Observed
${esc(observation.actual)}
` : '') + + (!observation ? `

${esc(grade?.error ?? 'This check has no recorded result in this grade.')}

` + : observation.expected == null && observation.actual == null && !observation.summary + ? '

No observation details were recorded.

' : '') + + '
'; + }).join('') + '
' + + `${CATEGORY[check.category ?? 'unknown']}` + + `${check.history.map((outcome, index) => `${GLYPH[outcome] ?? GLYPH['not-run']}`).join('')}` + + '').join(''); + }).join(''); + return errors + '
Recorded grades, oldest first. Expand a check for evidence. ' + + '✓ Pass✕ Fail' + + '· No pass/fail result
' + + '
' + + `${groups}
CheckRequirementCategoryGrades
`; +} + +function artifacts(evidence: AttemptPackage | null, key: string, visual: boolean): string { + const items = (evidence?.executions ?? []).flatMap(execution => + visual ? execution.visuals : execution.artifacts.filter(item => item.kind !== 'visual')); + const link = (id: string): string => + `/api/campaigns/${encodeURIComponent(key)}/artifacts/${encodeURIComponent(id)}`; + if (!items.length) return `

No ${visual ? 'screenshots' : 'files'} are available for this attempt.

`; + if (visual) { + return `
${items.map(item => { + const source = link(item.id); + return ``; + }).join('')}
` + + '
'; + } + return `
${items.map(item => + `${esc(item.path)}`).join('')}
`; +} + +export function attemptPage({ sheet, attemptId, tab, checks, evidence, log, transcript, + progression, timeBudget, canControl = false, controlError = '' }: AttemptPageInput): string { + const found = locate(sheet, attemptId); + const crumbs = (tail: string): string => `
Campaigns / ` + + `${esc(sheet.title)} / ` + + `${esc(tail)}
`; + if (!found) { + return `
${crumbs(attemptId)}` + + '

Attempt not found

'; + } + const { stack, attempt } = found; + const clock = timeBudget?.observedAt + ? executionClock(new Date(Date.parse(timeBudget.observedAt) - timeBudget.consumedMs).toISOString(), + attempt.status === 'running' ? null : timeBudget.observedAt) + : executionClock(attempt.executionStartedAt, attempt.executionCompletedAt); + const latestGrant = timeBudget?.grants.slice().sort((a, b) => + a.request.requestedAt.localeCompare(b.request.requestedAt)).at(-1); + const pending = timeBudget?.grants.some(grant => grant.disposition === 'pending') ?? false; + const resumeWithTime = attempt.status !== 'running' && timeBudget?.continuation?.eligible === true; + const timeControls = canControl && timeBudget && (resumeWithTime || (attempt.status === 'running' && timeBudget.liveGrantSupported)) + ? `
` + + '' + + `` + + `Limit after request: ${duration((timeBudget.effectiveMinutes + 120) * 60)}` + + `${resumeWithTime ? 'Continues from the verified checkpoint.' : 'Keeps the agent running.'} Cost and repair limits stay fixed.
` + : canControl && attempt.status === 'running' && timeBudget?.liveGrantSupported === false + ? '

This controller does not support live time extensions.

' + : canControl && timeBudget?.continuation?.reason + ? `

Cannot resume: ${esc(timeBudget.continuation.reason)}

` : ''; + const grantStatus = latestGrant ? `

${esc( + latestGrant.disposition === 'pending' ? 'Time request pending. The limit has not changed yet.' + : latestGrant.disposition === 'accepted' ? `Time added. Limit: ${duration(timeBudget!.effectiveMinutes * 60)}.` + : `Time request rejected: ${latestGrant.reason ?? 'See the grant evidence.'}`)}

` : ''; + const name = `${stackLabel(stack.stack)} rep ${attempt.repetition}`; + const counts: Record = { + checks: checks ? String(checks.checks.length) : '', + screenshots: evidence + ? String(evidence.executions.reduce((total, item) => total + item.visuals.length, 0)) : '', + files: evidence ? String(evidence.executions.reduce((total, item) => + total + item.artifacts.filter(entry => entry.kind !== 'visual').length, 0)) : '', + transcript: attempt.status === 'running' ? 'live' : '', + log: attempt.status === 'running' ? 'live' : '', + }; + const tabs = (['checks', 'transcript', 'screenshots', 'files', 'log'] as const).map(entry => + `` + + `${entry[0]!.toUpperCase()}${entry.slice(1)}` + + `${counts[entry] ? `${esc(counts[entry])}` : ''}`).join(''); + const help: Record = { + 'Checks passed': 'Accepted checks passed / selected, including checks not reached.', + 'Weighted score': 'Earned points / available points.', + 'First build': 'First build at each level. Earlier fixes and feedback are retained.', + Repairs: 'Completed repairs / allowance. Per-feature limits apply.', + Elapsed: 'Consumed time across executions / effective time limit. Includes coding, grading, repairs, and host sleep. Time between executions is excluded.' + + (timeBudget ? ` Original limit: ${duration(timeBudget.originalMinutes * 60)}. Accepted extensions: ${timeBudget.extensionCount}.` : ''), + Time: 'Recorded attempt duration.', + Spend: 'Saved receipts or live usage at pinned prices. ~ estimate; ≤ upper bound.', + }; + const figure = (label: string, text: string, tone = ''): string => + `
${metricLabel(label, help[label])}
${text}
`; + const panel = tab === 'transcript' ? transcriptPanel(transcript) + : tab === 'checks' ? checksTable(checks) + : tab === 'log' ? (log ? `
${esc(log)}
` : '

No log output is recorded yet.

') + : artifacts(evidence, sheet.key, tab === 'screenshots'); + const issue = attempt.excluded + ? `
Why this run was excluded` + + `

${esc(attempt.excluded)}

` : ''; + const categories = Object.entries(attempt.checkCategories ?? {}).filter(([, value]) => value.selected > 0); + const categorySummary = categories.some(([category]) => category !== 'unknown') ? '
' + categories.map(([category, value]) => + figure(CATEGORY[category as keyof typeof CATEGORY], ratio(value.passed, value.selected))).join('') + '
' : ''; + const track = progression?.stacks.find(entry => entry.attemptId === attemptId); + const history = sheet.mode === 'dependency' + ? '

Feature dependencies

' + (progression && track + ? graph(progression, [{ stack: track.stack, + statuses: track.steps.at(-1)?.statuses ?? progression.nodes.map(() => 'locked') }]) + : '

Feature graph unavailable.

') + : `

Grade history

${bigClimb(attempt.climb, level => `L${level}`)}`; + return `
${crumbs(name)}` + + `

${esc(stackLabel(stack.stack))} ` + + `rep ${attempt.repetition}

` + + + `
${figure('Checks passed', completionLabel(attempt))}` + + figure('Spend', spend(attempt.spend, attempt.spendPending, attempt.liveSpend)) + + figure('Status', esc(phrase(attempt)), attempt.stalling ? 'now warn' : 'now') + + figure('Weighted score', pct(attempt.score)) + + figure('First build', pct(attempt.unaided)) + + figure('Repairs', ratio(attempt.repairs.used, attempt.repairs.budget)) + + figure('Elapsed', attempt.status === 'running' || attempt.executionCompletedAt + ? clock + + ` / ${duration((timeBudget?.effectiveMinutes ?? sheet.facts.timeLimitMinutes) * 60)}` : DASH) + + figure('Time', duration(attempt.timeSec)) + + `
${timeControls}${grantStatus}${controlError ? `` : ''}${issue}${history}` + + `
${tabs}
${tab === 'checks' ? categorySummary : ''}${panel}
`; +} + +function transcriptPanel(page?: TranscriptPage | null): string { + if (!page) return '
Loading transcript�
'; + if (!page.sessions.length) return '

No transcript is available for this run yet.

'; + return '
' + + (page.before === null ? '' : ``) + + '
' + + (page.skipped ? '

Some malformed transcript records could not be displayed.

' : '') + + '
' + + page.messages.map(message => message.tool + ? `
${esc(message.role)}
${esc(message.text)}
` + : `
${esc(message.role)}
${esc(message.text)}
`).join('') + + '
'; +} diff --git a/tools/stack-bench/dashboard/public/views/campaign.ts b/tools/stack-bench/dashboard/public/views/campaign.ts new file mode 100644 index 00000000000..439b881d672 --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/campaign.ts @@ -0,0 +1,259 @@ +// Results compare stacks. Runs expose individual evidence. Feature views use +// the same selected attempt per stack, separate from aggregate results. + +import type { CampaignProgression, CampaignSheet, ProgressionStep, SheetAttempt, SheetStack } + from '../../dashboard-views.js'; +import { DASH, completionLabel, modelLabel, duration, executionClock, esc, metricLabel, spend, num, pct, phrase, ratio, stackLabel, statusWord } from '../format.js'; +import { progressChart } from '../progress-chart.js'; +import { graph } from '../graph.js'; + +export type QuestlineView = 'grid' | 'graph' | 'replay'; + +export interface CampaignPageInput { + sheet: CampaignSheet; + progression: CampaignProgression | null; + view: QuestlineView; + chart?: 'completion' | 'cost' | 'distribution'; + unit?: 'checks' | 'features'; + hiddenChartRuns?: ReadonlySet; + step: number; +} + +export interface ReplayEvent { + stack: string; + ordinal: number; + step: ProgressionStep; +} + +const DOT: Record = { passed: 'p', active: 'a', working: 'a', failed: 'f', + blocked: 'b', locked: 'o' }; + +function short(value: string | null): string { + return value ? value.slice(0, 12) : DASH; +} + +function latest(stack: SheetStack): SheetAttempt | null { + return stack.attempts.find(attempt => attempt.id === stack.selectedAttemptId) ?? null; +} + +function facts(sheet: CampaignSheet): string { + const fact = sheet.facts; + const dependency = sheet.mode === 'dependency'; + const depth = sheet.levels.length ? Math.max(...sheet.levels) : 0; + const cells: Array<[string, string, string]> = [['Mode', fact.mode, '']]; + const limits = fact.repairLimits; + const repairBudget = [ + limits.perFeature === undefined ? '' : `${limits.perFeature} per feature`, + limits.perDepth === undefined ? '' : `${limits.perDepth.count} per depth${limits.perDepth.carry ? " (carry forward)" : ""}`, + limits.total === undefined ? '' : `${limits.total} total`, + ].filter(Boolean).join(' · '); + cells.push(dependency ? ['Depth', String(depth), ''] + : ['Levels', sheet.levels.map(level => `L${level}`).join('–'), '']); + if (dependency) { + cells.push(['Work', fact.workSelection ?? DASH, ''], + ['Repair', fact.repairSelection ?? DASH, ''], + ['Repair budget', repairBudget || 'No count limit', 'Limits apply to each attempt. When limits overlap, the tightest remaining limit applies.']); + } else { + cells.push(['Repair budget', repairBudget || 'No count limit', 'Limits apply to each attempt.']); + } + cells.push(['Repetitions', String(sheet.repetitions), ''], + ['Agent', fact.agent ?? DASH, ''], ['Model', fact.model ?? DASH, ''], + ['Guidance', fact.guidance ?? DASH, ''], + ['Production quality', fact.productionQuality === null ? 'Mixed' + : fact.productionQuality ? 'Requested' : 'Not requested', 'Whether the prompt explicitly requests a production-quality app.'], + ['Recipe', [...new Set(fact.recipes.map(recipe => + [recipe.id, short(recipe.contentSha256)].filter(Boolean).join(' ')))].join(' · ') || DASH, ''], + ['Time limit', `${fact.timeLimitMinutes} min`, ''], + ['Spend limit', fact.spendLimitUsd === null ? DASH + : `$${fact.spendLimitUsd} per attempt`, ''], + ['Controller', short(fact.controllerImage), ''], ['Plan', short(fact.planSha256), '']); + if (sheet.mixedScope) cells.push(['Scope', 'mixed', 'attempts do not share one test plan']); + const continued = sheet.stacks.filter(stack => stack.continued).length; + if (continued) cells.push(['Continued', String(continued), '']); + + return `
${cells.map(([label, value, hover]) => + `
${esc(label)}` + + `${esc(value)}
`).join('')}
`; +} + +function questlineRows(sheet: CampaignSheet, stacks: readonly SheetStack[]): string { + const lead = stacks.find(stack => stack.questlines?.length)?.questlines ?? []; + const rows = lead.map(questline => { + const cells = stacks.map(stack => { + const owned = stack.questlines?.find(entry => entry.id === questline.id) ?? null; + const dots = (owned?.nodes ?? []).map(node => + ``).join(''); + const score = owned?.score ?? null; + return `
${dots}` + + `${pct(score)}
`; + }).join(''); + return `${esc(questline.title)}${cells}`; + }).join(''); + if (sheet.mode !== 'dependency') return ''; + return rows || `Feature progress appears after the first recorded grade.`; +} + +function levelRows(stacks: readonly SheetStack[]): string { + const levels = stacks.find(stack => stack.levels?.length)?.levels ?? []; + return levels.map(level => ['unaided', 'score'].map(kind => { + const cells = stacks.map(stack => { + const owned = stack.levels?.find(entry => entry.level === level.level) ?? null; + const points = kind === 'unaided' ? owned?.unaided ?? null : owned?.score ?? null; + return `
${points + ? ratio(points.score, points.max) : DASH}
`; + }).join(''); + return `L${level.level} ${kind === 'unaided' ? 'first build' : 'score'}${cells}`; + }).join('')).join(''); +} + +export function selectedProgression(progression: CampaignProgression, sheet: CampaignSheet): CampaignProgression { + return { ...progression, stacks: sheet.stacks.flatMap(stack => progression.stacks.filter(track => + stack.stack === track.stack && stack.selectedAttemptId === track.attemptId)) }; +} + +export function replayTimeline(progression: CampaignProgression): ReplayEvent[] { + const tracks = progression.stacks; + const depth = Math.max(0, ...tracks.map(track => track.steps.length)); + const events: ReplayEvent[] = []; + for (let ordinal = 0; ordinal < depth; ordinal += 1) { + for (const track of tracks) { + const step = track.steps[ordinal]; + if (step) events.push({ stack: track.stack, ordinal, step }); + } + } + return events; +} + +function marker(step: ProgressionStep, failed: boolean): string { + if (failed) return 'f'; + if (step.action === 'repair') return 'r'; + return step.action === 'grant' ? 'g' : 'b'; +} + +function replay(progression: CampaignProgression, cursor: number): string { + const events = replayTimeline(progression); + cursor = Math.min(Math.max(0, cursor), Math.max(0, events.length - 1)); + const span = Math.max(1, events.length - 1); + const selected = events[cursor] ?? events.at(-1) ?? null; + const failedAt = (step: ProgressionStep): boolean => step.targets.some(target => + step.statuses[progression.nodes.findIndex(node => node.id === target)] === 'failed'); + const title = (id: string): string => + progression.nodes.find(node => node.id === id)?.title ?? id; + const head = selected ? [['Step', ratio(cursor + 1, events.length)], + ['Stack', esc(stackLabel(selected.stack))], ['Action', esc(selected.step.action)], + ['Feature', selected.step.targets.length === 1 + ? esc(title(selected.step.targets[0] ?? '')) : `${selected.step.targets.length} features`], + ['Score', pct(selected.step.score)], ['Repairs', num(selected.step.repairs)]] + .map(([label, value]) => `
${label}` + + `${value}
`).join('') : ''; + // Drawn as one SVG per stack: the dashboard's policy allows no inline style, + // and a marker's position is geometry, not decoration. + const at = (index: number): number => 20 + 960 * index / span; + const rows = progression.stacks.map(track => { + const marks = events.map((event, index) => event.stack !== track.stack ? '' : + ``).join(''); + return `${esc(stackLabel(track.stack))}` + + '' + + `${marks}`; + }).join(''); + const snapshot = progression.stacks.map(track => { + const step = events.filter((event, index) => + event.stack === track.stack && index <= cursor).at(-1)?.step ?? null; + return { stack: track.stack, + statuses: step?.statuses ?? progression.nodes.map(() => 'locked') }; + }); + return `
${head}
` + + graph(progression, snapshot) + + `
${rows}
`; +} + +function board({ sheet, progression, view, step }: CampaignPageInput, + stacks: readonly SheetStack[]): string { + const chips = (['grid', 'graph', 'replay'] as const).map(entry => + `` + + `${entry[0]!.toUpperCase()}${entry.slice(1)}`).join(''); + const heading = stacks.map(stack => `${esc(stackLabel(stack.stack))}`).join(''); + const grid = (rows: string): string => `
${heading}${rows}
Feature
`; + let content: string; + if (sheet.mode !== 'dependency') content = grid(levelRows(stacks)); + else if (view === 'grid' || !progression) content = grid(questlineRows(sheet, stacks)); + else if (view === 'graph') { + const selected = selectedProgression(progression, sheet); + const snapshot = selected.stacks.map(track => ({ stack: track.stack, + statuses: track.steps.at(-1)?.statuses ?? selected.nodes.map(() => 'locked') })); + content = graph(selected, snapshot); + } else content = replay(selectedProgression(progression, sheet), step); + return '
' + + '

Feature progress

' + + (sheet.mode === 'dependency' ? `
Explore · ${esc(view)}
` : '') + + '
' + + content + '
'; +} + +export function campaignPage(input: CampaignPageInput): string { + const sheet = input.sheet; + const stacks = sheet.stacks; + const showRepairs = stacks.some(stack => stack.attempts.some(attempt => + attempt.repairs.budget > 0 || attempt.repairs.used > 0)); + const cell = (render: (stack: SheetStack) => string): string => + stacks.map(stack => `${render(stack)}`).join(''); + const help: Record = { + 'Checks passed': 'Median percentage of selected checks passed, across valid completed runs.', + 'Weighted score': 'Median score weighted by check points, across valid completed runs.', + 'First builds': 'Summed first-build points across levels. Earlier fixes and feedback are retained; this is not an unaided run.', + Regressions: 'Median count of previously passing checks that later failed, across valid completed runs.', + 'Valid runs': 'Completed runs with usable evidence; not necessarily all checks passed.', + Excluded: 'Invalid or incomplete evidence. Spend is retained in Total spend.', + 'Active time': 'Median measured-run time, excluding recorded provider waits and operator pauses. Run Elapsed shows wall time.', + 'Cost per valid run': 'Mean exact measured-run cost, including explicit resume history. Independent failed retries remain in Total spend.', + 'Total spend': 'All runs, including excluded. ~ estimate; ≤ upper bound. Pinned prices.', + }; + const row = (label: string, render: (stack: SheetStack) => string): string => + `${metricLabel(label, help[label])}${cell(render)}`; + const value = (text: string): string => `
${text}
`; + const heads = stacks.map(stack => { + const attempt = latest(stack); + const label = esc(stackLabel(stack.stack)); + return `${attempt + ? `` + + `${label}` : label}`; + }).join(''); + const repetitions = row('Valid runs', stack => value(ratio(stack.n, stack.attempts.length))) + + row('Excluded', stack => + value(num(stack.attempts.filter(attempt => attempt.excluded).length))); + return `
Campaigns / ` + + `${esc(sheet.key)}
` + + `

${esc(sheet.title)}

` + + `${sheet.provisional ? 'Provisional' : esc(statusWord(sheet.status))}
${facts(sheet)}` + + '

Results

' + + `
${heads}` + + row('Checks passed', stack => `
${pct(stack.completionRate === null ? null : 100 * stack.completionRate)}
`) + + row('Cost per valid run', stack => value(stack.costPerValidRun === null ? (stack.n ? 'Unknown' : 'Awaiting valid runs') : `$${stack.costPerValidRun.toFixed(2)}`)) + + row('Weighted score', stack => value(pct(stack.score))) + + (showRepairs ? row('First builds', stack => value(pct(stack.unaided))) : '') + + row('Regressions', stack => value(num(stack.regressions))) + + row('Active time', stack => value(duration(stack.timeSec))) + + repetitions + + row('Total spend', stack => value(spend(stack.spend, stack.spendPending, stack.liveSpend))) + + '
Metric
' + + (sheet.mode === 'dependency' ? progressChart(sheet, input.progression, input.chart, input.view, input.hiddenChartRuns, input.unit) : '') + + '

Runs

' + + `
${showRepairs ? '' : ''}` + + stacks.flatMap(stack => stack.attempts.map(attempt => { + const href = `/c/${encodeURIComponent(sheet.key)}/a/${encodeURIComponent(attempt.id)}`; + const effort = attempt.effort ? ` (${attempt.effort})` : ''; + return `` + + `` + + `` + + `` + + (showRepairs ? `` : '') + + `` + + ``; + })).join('') + + '
RunModelFeatures passedChecks passedSpendRepairsElapsedStatus
${esc(stackLabel(stack.stack))} ${attempt.repetition}${esc(modelLabel(attempt.model))}${esc(effort)}${completionLabel(attempt, attempt.featureCompletion ?? null)}${completionLabel(attempt)}${spend(attempt.spend, attempt.spendPending, attempt.liveSpend)}${ratio(attempt.repairs.used, attempt.repairs.budget)}${attempt.status === 'running' || attempt.executionCompletedAt ? executionClock(attempt.executionStartedAt, attempt.executionCompletedAt) : DASH}${attempt.excluded + ? `
Excluded · show reason

${esc(attempt.excluded)}

` + : esc(phrase(attempt))}
' + board(input, stacks) + '
'; +} diff --git a/tools/stack-bench/dashboard/public/views/campaigns.ts b/tools/stack-bench/dashboard/public/views/campaigns.ts new file mode 100644 index 00000000000..de062746e0b --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/campaigns.ts @@ -0,0 +1,115 @@ +// Show every active attempt before the campaign history. + +import type { CampaignSheet, OverviewCampaign, OverviewEntry, OverviewPage, SheetAttempt } + from '../../dashboard-views.js'; +import { DASH, esc, excludedLabel, pct, phrase, shape, since, spend, stackLabel, statusWord } from '../format.js'; +import type { ReferenceRun } from '../../dashboard-reference-runs.js'; + +import type { CampaignFilter } from '../../dashboard-views.js'; +export type { CampaignFilter } from '../../dashboard-views.js'; + +const FILTERS: Array<{ id: CampaignFilter; label: string }> = [{ id: 'all', label: 'All' }, + { id: 'attention', label: 'Needs attention' }, { id: 'completed', label: 'Completed' }, + { id: 'ready', label: 'Ready' }]; + +function readable(campaign: OverviewEntry): campaign is OverviewCampaign { + return 'scores' in campaign; +} + +function matches(campaign: OverviewEntry, filter: CampaignFilter): boolean { + if (filter === 'all') return true; + if (filter === 'attention') { + return campaign.status === 'attention-required' || campaign.status === 'unreadable'; + } + if (filter === 'completed') return campaign.status === 'completed'; + return campaign.status === 'prepared'; +} + +function lane(sheet: CampaignSheet, stack: string, attempt: SheetAttempt): string { + const warn = attempt.stalling; + return `
` + + `
` + + `${esc(stackLabel(stack))} · Rep ${attempt.repetition}` + + `
${esc(attempt.variant)}
` + + `
Completion` + + `${excludedLabel(attempt) ?? pct(attempt.completion?.rate == null ? null : 100 * attempt.completion.rate)}
` + + `
Cost` + + `${spend(attempt.spend, attempt.spendPending, attempt.liveSpend)}
` + + `${esc(phrase(attempt))}
`; +} + +function live(sheet: CampaignSheet): string { + const lanes = sheet.stacks.flatMap(owner => owner.attempts + .filter(item => item.status === 'running') + .map(attempt => lane(sheet, owner.stack, attempt))); + if (!lanes.length) return ''; + return `
` + + `${esc(sheet.title)}
${lanes.join('')}
`; +} + +function stackCell(campaign: OverviewEntry, stack: string): string { + const score = readable(campaign) ? campaign.scores[stack] ?? null : null; + if (score === null) return `${DASH}`; + return `` + + `${pct(score)}`; +} + +function tone(status: string): string { + if (status === 'running') return 'run'; + if (status === 'completed') return 'done'; + if (status === 'attention-required' || status === 'unreadable') return 'warn'; + return 'idle'; +} + +function row(campaign: OverviewEntry, stacks: readonly string[]): string { + const summary = readable(campaign) ? campaign : null; + return `` + + `${esc(campaign.title)}` + + `${summary + ? esc(shape(summary.mode, summary.levels, summary.repetitions)) : DASH}` + + `${esc(statusWord(campaign.status))}` + + stacks.map(stack => stackCell(campaign, stack)).join('') + + `${summary ? esc(since(summary.updatedAt)) : DASH}`; +} + +export function campaignsPage({ campaigns, sheets, filter, loading = false, references, pagination }: { + campaigns: readonly OverviewEntry[]; + sheets: readonly CampaignSheet[]; + filter: CampaignFilter; + loading?: boolean; + references?: { runs: ReferenceRun[]; error: string | null }; + pagination?: Pick; +}): string { + const stacks = [...new Set([ + ...campaigns.flatMap(campaign => readable(campaign) ? Object.keys(campaign.scores) : []), + ...sheets.flatMap(sheet => sheet.stacks.map(entry => entry.stack)), + ])]; + const shown = campaigns.filter(campaign => matches(campaign, filter)); + const chips = FILTERS.map(entry => + `` + + `${entry.label}${loading ? '' : ` ${pagination?.counts[entry.id] ?? campaigns.filter(campaign => matches(campaign, entry.id)).length}`}`).join(''); + const body = loading ? `
Loading campaigns…
` + : shown.length ? shown.map(campaign => row(campaign, stacks)).join('') + : `No campaigns match this filter.`; + const pager = pagination && !loading ? `' : ''; + const validations = references?.runs.map(run => `
` + + `${esc(run.title)}` + + `${esc(run.status)}` + + `No model calls${run.points + ? `${run.points.passed}${run.points.planned === null ? '' : `/${run.points.planned}`} points passed` + + (run.points.measured !== run.points.planned ? `${run.points.measured} measured${run.points.planned === null ? ' · total unavailable' : ''}` : '') + : run.status === 'running' ? 'Awaiting results' : 'No complete grade recorded'}` + + `Log ` + + `
${esc(run.log || 'No log recorded.')}
`).join('') ?? ''; + return `

Campaigns

${sheets.map(live).join('')}` + + validations + + (references?.error ? `

${esc(references.error)}

` : '') + + `
${chips}
` + + '' + + stacks.map(stack => ``).join('') + + `${body}
CampaignScopeStatus${esc(stackLabel(stack))}Updated
${pager}
`; +} diff --git a/tools/stack-bench/dashboard/public/views/plans.ts b/tools/stack-bench/dashboard/public/views/plans.ts new file mode 100644 index 00000000000..1511ed19633 --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/plans.ts @@ -0,0 +1,93 @@ +// Saved plan summaries and shared navigation. New runs use the setup page. + +import type { DashboardPlan } from '../../dashboard-model.js'; +import { DASH, duration, esc, money, num } from '../format.js'; + +export type Page = 'campaigns' | 'plans' | 'campaign' | 'check-guide'; + +export interface RunForm { + error: string; +} + +const HEADS: Array<[string, string]> = [['Plan', 'name'], ['Mode', 'shape'], ['Shape', 'shape'], + ['Stacks', 'stack'], ['Attempts', 'stack'], ['Parallel', 'stack'], ['Repairs', 'stack'], + ['Time limit', 'stack'], ['Attempt cap', 'stack'], ['Campaign cap', 'stack'], ['State', 'state']]; + +export function runName(planId: string, now: Date): string { + const pad = (value: number): string => String(value).padStart(2, '0'); + const stamp = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}` + + `-${pad(now.getHours())}${pad(now.getMinutes())}`; + return `${planId}-${stamp}`.toLowerCase().replace(/[^a-z0-9.-]+/g, '-') + .replace(/^[^a-z0-9]+/, '').slice(0, 120); +} + +export function topbar({ page, key, canStart, resumable, controllerOwner, error, reportFiles = [] }: { + page: Page; key: string; canStart: boolean; resumable: boolean; controllerOwner?: string | null; error: string; + reportFiles?: string[]; +}): string { + const artifact = (path: string): string => `/api/campaigns/${encodeURIComponent(key)}/artifacts/` + + btoa(path).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); + const files = page === 'campaign' + ? '
Files
' + + `planstate` + + (reportFiles.includes('report/report.html') ? `report` : '') + + (reportFiles.includes('report/export-manifest.json') ? `export manifest` : '') + + '
' : ''; + const resume = resumable + ? '
' + + '' + + (error ? `${esc(error)}` : '') + '
' : ''; + const stop = canStart && controllerOwner + ? `
` + + '' + + (error ? `${esc(error)}` : '') + '
' : ''; + const nav = (on: boolean, label: string, href: string): string => + `${label}`; + return '
' + + 'STACK BENCH' + + `
${stop}${resume}${files}` + + 'New run
'; +} + +function shapeOf(plan: DashboardPlan): string { + const levels = plan.levels ?? []; + if (!levels.length) return DASH; + const depth = Math.max(...levels); + if (plan.mode === 'dependency') return `depth ${depth}`; + return levels.length > 1 ? `L${Math.min(...levels)}–L${depth}` : `L${depth}`; +} + +function planRow(plan: DashboardPlan): string { + const budgets = plan.budgets ?? null; + const stacks = plan.stacks ?? []; + const cell = (value: string, hover = ''): string => + `${value}`; + return `` + + `${esc(plan.title)}` + + `${esc(plan.mode ?? DASH)}` + + `${esc(shapeOf(plan))}` + + cell(stacks.length ? num(stacks.length) : DASH, stacks.join(' · ')) + + cell(num(plan.attempts)) + cell(num(plan.parallelism)) + + cell(plan.repairBudget === undefined ? DASH : num(plan.repairBudget)) + + cell(budgets ? duration(budgets.attemptTimeoutMinutes * 60) : DASH) + + cell(budgets ? money(budgets.maxCostUsdPerAttempt) : DASH) + + cell(budgets?.maxCostUsdPerAttempt != null && plan.attempts != null + ? money(budgets.maxCostUsdPerAttempt * plan.attempts) : DASH, + 'Maximum across planned attempts; each attempt cap includes its retries') + + `${esc(plan.state)}`; +} + +export function plansPage({ plans, loading = false }: { + plans: readonly DashboardPlan[]; loading?: boolean; +}): string { + return `

Saved plans

` + + '

Plans record the exact configuration behind a run. Use New run to select its settings.

' + + '
' + + HEADS.map(([label, kind]) => ``).join('') + + `${loading + ? `` + : plans.length ? plans.map(planRow).join('') + : ``}
${label}
Loading plans…
No plans
`; +} diff --git a/tools/stack-bench/dashboard/public/views/run-setup.ts b/tools/stack-bench/dashboard/public/views/run-setup.ts new file mode 100644 index 00000000000..1ad7433cf17 --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/run-setup.ts @@ -0,0 +1,122 @@ +import type { RunSetupCatalog, RunSetupRequest, RunSetupReview } from '../../../src/campaigns/run-setup.js'; +import { esc, money, modelLabel, stackLabel } from '../format.js'; +import { runName } from './plans.js'; + +const guidanceLabel = (id: string) => ({ neutral: 'Standard skills', + 'neutral-no-sdk': 'No SDK skills or dev workflow', + 'neutral-dev': 'Standard skills + dev workflow', + 'neutral-dev-no-sdk': 'Dev workflow without SDK skills' } as Record)[id] ?? id; + +export function initialRun(catalog: RunSetupCatalog, id?: string): RunSetupRequest | null { + const w = catalog.workloads.find(w => w.id === id) + ?? catalog.workloads.find(w => w.mode === 'dependency' && w.workSelection === 'progressive') + ?? catalog.workloads[0]; + if (!w) return null; + return { key: runName(w.track, new Date()) + '-' + crypto.randomUUID().slice(0, 8), + workload: w.id, workloadSha256: w.sha256, level: Math.max(...w.levels), stacks: [...w.stacks], + agents: [{ index: 0, effort: w.agents[0]!.effort ?? 'medium' }], + conditions: [(w.conditions.find(c => c.guidance === 'neutral-dev') + ?? w.conditions.find(c => c.guidance === 'neutral') ?? w.conditions[0])!.id], ...w.defaults, + productionQuality: true, maxCostUsd: w.defaults.maxCostUsd ?? 0, credentials: {} }; +} + +export function selectGuidance(conditions: RunSetupCatalog['workloads'][number]['conditions'], sdk: string, dev: string): string[] { + return conditions.filter(c => c.sdkSkills === (sdk === 'on') + && c.devWorkflow === (dev === 'on')).map(c => c.id); +} + +export function readRunForm(form: HTMLFormElement, catalog: RunSetupCatalog): RunSetupRequest { + const data = new FormData(form); + return { key: String(data.get('key')), workload: String(data.get('workload')), workloadSha256: String(data.get('workloadSha256')), + level: Number(data.get('level')), stacks: data.getAll('stack').map(String), + agents: data.getAll('agent').map(index => ({ index: Number(index), + effort: String(data.get(`effort-${index}`)) as RunSetupRequest['agents'][number]['effort'] })), + conditions: data.has('sdkSkills') ? selectGuidance(catalog.workloads.find(w => w.id === data.get('workload'))!.conditions, + String(data.get('sdkSkills')), String(data.get('devWorkflow'))) : data.getAll('condition').map(String), repetitions: Number(data.get('repetitions')), + productionQuality: data.has('productionQuality'), + parallelism: Number(data.get('parallelism')), repairs: Number(data.get('repairs')), + timeoutMinutes: Number(data.get('timeoutMinutes')), maxCostUsd: Number(data.get('maxCostUsd')), + pauseAfterDepth: data.get('pauseAfterDepth') ? Number(data.get('pauseAfterDepth')) : null, + credentials: { adapters: Object.fromEntries([...data].filter(([key, value]) => + key.startsWith('credential-') && value).map(([key, value]) => [key.slice(11), String(value)])) } }; +} + +export function runSetupPage(catalog: RunSetupCatalog | null, request: RunSetupRequest | null, + review: RunSetupReview | null, error: string, canStart: boolean): string { + const field = (label: string, input: string) => ``; + const option = (value: string | number, label: string, selected: boolean) => + ``; + const integer = (name: string, value: number, min = 1) => ``; + const alert = error ? `` : ''; + const head = '

New run

'; + if (!catalog) return head + '

Loading setup…

' + alert + '
'; + if (!request || !catalog.workloads.length) return head + '

No runnable workloads are configured. Run appliance setup to install the workload presets.

' + + catalog.errors.map(error => `

${esc(error)}

`).join('') + ''; + const w = catalog.workloads.find(w => w.id === request.workload)!; + const delivery = w.mode === 'dependency' + ? ({ progressive: 'Progressive dependency graph', feature: 'One ready feature at a time', + 'all-at-once': 'Full graph in one build' }[w.workSelection as string] ?? w.workSelection) + : 'Sequential levels'; + const splitGuidance = w.conditions.length === 4 + && new Set(w.conditions.map(c => `${c.sdkSkills}:${c.devWorkflow}`)).size === 4; + const guidanceChoice = (key: 'sdkSkills' | 'devWorkflow', label: string) => { + const value = w.conditions.find(c => request.conditions.includes(c.id))?.[key] ? 'on' : 'off'; + return field(label, ``); + }; + const model = (index: number) => w.agents[index]!; + if (review) { + const rows = [ + ['Workload', `${w.title} · L${request.level}`], + ['Work delivery', delivery], + ['Stacks', request.stacks.map(stackLabel).join(', ')], + ['Models', request.agents.map(a => `${modelLabel(model(a.index).model)} (${a.effort})`).join(', ')], + ['Guidance', request.conditions.map(id => guidanceLabel(w.conditions.find(c => c.id === id)!.guidance)).join(', ')], + ['Production-quality app', request.productionQuality ? 'Requested' : 'Not requested'], + ['Runs', `${review.attempts} attempts · ${request.repetitions} per combination · ${review.parallelism} concurrent`], + ['Repairs', `${request.repairs} per attempt`], + ['Limits', `${request.timeoutMinutes} minutes and ${money(request.maxCostUsd)} per attempt`], + ['Total cost cap', money(review.maxCostUsd)], + ['Pause', request.pauseAfterDepth ? `After L${request.pauseAfterDepth}` : 'None'], + ['Account', review.authentication.map(a => `${a.adapter}: ${a.profile ? `${a.profile.id} (${a.profile.mode})` : `appliance default (${a.source})`}`).join(', ')], + ]; + return head + '

Review run

' + rows.map(([key, value]) => + `
${esc(key!)}
${esc(value!)}
`).join('') + '
' + + (review.qualification === 'pending' ? '

Grading qualification is pending. Results will be provisional.

' : '') + + '

Cost caps use recorded token pricing. Subscription usage is not an invoice charge.

' + + `
Recorded pricing and runtime
${esc(JSON.stringify({ pricing: review.pricing, runtime: review.runtime }, null, 2))}
` + + `
` + + '
' + alert + ''; + } + return head + (canStart ? '' : '

This dashboard is read-only. Start the appliance to run a study.

') + + `
` + + '
' + + field('Workload', ``) + + field('Target', ``) + + `

${esc(String(delivery))}

` + + '
Stacks
' + + w.stacks.map(id => ``).join('') + + '
Models and reasoning' + + w.agents.map((agent, index) => `
` + + `
`).join('') + + '
App requirement' + + `` + + '

Build a production-quality application suitable for real users, not a prototype or demo.

' + + '
SpacetimeDB guidance' + + (splitGuidance ? '
' + guidanceChoice('sdkSkills', 'SDK skills') + + guidanceChoice('devWorkflow', 'Dev workflow') : '
' + w.conditions.map(c => ``).join('')) + + '
' + + field('Repetitions per combination', integer('repetitions', request.repetitions)) + + field('Concurrent attempts', integer('parallelism', request.parallelism)) + + field('Repairs per attempt', integer('repairs', request.repairs, 0)) + + field('Minutes per attempt', integer('timeoutMinutes', request.timeoutMinutes)) + + field('Cost cap per attempt (USD)', ``) + + '
Pause, run name and accounts
' + + field('Pause', `') + + field('Run name', ``) + + [...new Set(w.agents.map(a => a.adapter))].map(adapter => field(esc(adapter), `')).join('') + + '
' + + '
' + alert + '
'; +} diff --git a/tools/stack-bench/docker-compose.yaml b/tools/stack-bench/docker-compose.yaml new file mode 100644 index 00000000000..7c3675d51ed --- /dev/null +++ b/tools/stack-bench/docker-compose.yaml @@ -0,0 +1,49 @@ +# Databases for the Postgres and MongoDB backends. +# +# Development-only ports, container names and volumes keep this stack separate +# from the appliance. The SpacetimeDB +# backend needs no service here; run `spacetime start` for it. +# +# docker compose -f tools/stack-bench/docker-compose.yaml up -d +# +name: stack-bench + +services: + postgres: + image: postgres:16@sha256:219341e4cedb06c8634f80af40851da3425b41b76603fd890272f58e37e139f7 + container_name: stack-bench-dev-postgres + ports: + - "127.0.0.1:6532:5432" + environment: + POSTGRES_USER: appuser + POSTGRES_PASSWORD: local-app-password + POSTGRES_DB: app + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U appuser -d app"] + interval: 5s + timeout: 5s + retries: 12 + + # One local replica-set member exposes native transactions and change streams. + # This provides neither failover nor a multi-node availability test. + mongodb: + image: mongo:7@sha256:554a9bb1ec6e00c40ba078a41974a834d1a9a8ab1772645b69142afecc87f082 + command: ["mongod", "--replSet", "rs0", "--bind_ip_all"] + container_name: stack-bench-dev-mongodb + ports: + - "127.0.0.1:6537:27017" + volumes: + - mongodata:/data/db + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--eval", "try { rs.status(); } catch (e) { if (e.code !== 94) throw e; rs.initiate({_id:'rs0',members:[{_id:0,host:'localhost:27017'}]}); } if (!db.hello().isWritablePrimary) quit(1)"] + interval: 5s + timeout: 5s + retries: 12 + +volumes: + pgdata: + name: stack-bench-dev-pgdata + mongodata: + name: stack-bench-dev-mongodata diff --git a/tools/stack-bench/docs/README.md b/tools/stack-bench/docs/README.md new file mode 100644 index 00000000000..993047d5d17 --- /dev/null +++ b/tools/stack-bench/docs/README.md @@ -0,0 +1,53 @@ +# Stack Bench documentation + +Use the root [README](../README.md) for the product summary and +[Getting started](../GETTING-STARTED.md) for a first run. + +## Run Stack Bench + +- [Appliance operation](../appliance/README.md): configure and run campaigns +- [Dashboard](../dashboard/README.md): web interface and run-setup interface +- [Execution jobs](execution-jobs.md): submit work, assign hosts, and integrate a task queue +- [Credential profiles](credential-profiles.md): select and attribute account/API-key use +- [Recovery](../appliance/RECOVERY.md): interrupted runs and retained resources +- [Release](../appliance/RELEASE.md): assemble and verify a release + +## Understand the method + +- [Prompting method](prompting.md): prompt inputs, guidance profiles, + specification treatments, and repair requests +- [Grader](../grader/README.md): outcomes, scoring, and check validation +- [Grading coverage](grading-coverage.md): what checks observe, their limits, + and failure review +- [Check categories](check-categories.md): categories and counting units +- [Study method](study-method.md): collecting and reporting a defensible comparison +- [Reference apps](../reference-apps/README.md): grading fixtures and qualification + +## Understand the system + +- [System design](system-design.md): ownership, terms, data flow, and operator loop +- [Appliance design](../appliance/DESIGN.md): security and container boundaries + +## Develop and author + +- [Development](development.md): local dependencies, source checks, and agent adapters +- [Authoring](authoring.md): add features, checks, prompts, and rules through their existing owners +- [Ecommerce composition](../tracks/ecommerce/composition/README.md): packs, + recipes, calibration, and specification treatment +- [Ecommerce levels](../tracks/ecommerce/LEVELS.md): sequential levels and dependency depths +- [Chat levels](../tracks/chat/LEVELS.md): current chat scope + +## Visuals + +- [Dependency graph](dependency-graph.html): generated ecommerce feature graph +- [Technical guide](technical-guide.html): current run path +- [Presentation](stack-bench.html): product presentation and illustrative checks +- [How it works](how-it-works.html): system map with a guided tour of a campaign and its qualification + +These illustrations do not carry qualification status; use the current +definition and evidence. `dependency-graph.html` is generated from the current +graph with `npm run graph`. Do not edit it by hand. + +Markdown files under `backends/`, `conditions/`, `tracks/*/prompts`, and +`tracks/*/contracts` are executable benchmark inputs. They stay with their +owners and are not general documentation. diff --git a/tools/stack-bench/docs/authoring.md b/tools/stack-bench/docs/authoring.md new file mode 100644 index 00000000000..32f5ab06790 --- /dev/null +++ b/tools/stack-bench/docs/authoring.md @@ -0,0 +1,127 @@ +# Author a benchmark change + +Product text belongs in `tracks/`. Shared actions belong in `src/actions/`. Stack operations belong in `src/stacks/`. A normal feature or rule change does not need runtime, report, or dashboard edits. + +Build once with `npm run build`. The commands below then use the compiled tools. Keep before and after outputs outside the authored definition directories. Do not copy old hashes into new evidence. + +## Add a normal feature + +Use the existing customer profile as a worked example. Its complete path is: + +- `tracks/ecommerce/prompts/modular/customer-profile.md`: product request. +- `tracks/ecommerce/contracts/customer-profile.md`: stable application interface. +- `tracks/ecommerce/scenarios/progression-customer-profile.json`: observations and assertions. +- `tracks/ecommerce/composition/packs/progression-customer-profile.json`: feature, dependencies, and selected criteria. +- `tracks/ecommerce/composition/recipes/progression-catalog.json`: available packs. +- `tracks/ecommerce/progression/ecommerce.json`: graph ownership and dependencies. + +For a new delivery-note feature, follow that path with new IDs. Ask for “A customer can save and view a delivery note.” Use a text field, save button, and summary hook. Use existing `signUp`, `click`, `fill`, and `expect` actions. Put the sample note in the scenario, not the product request. Keep one positive criterion for saving and viewing the note. Do not award several points for several selectors that prove the same behavior. + +Create a feature pack with `moduleType: "feature"`. Declare its account dependency. Add the pack to the current recipe and give one graph node ownership of its grading group. Match the other graph nodes' `featureRefs`, `gradingGroups`, and `dependencies` format. Add the behavior to all reference stacks. + +```sh +node dist/commands/composition-cli.js pack validate tracks/ecommerce/composition/packs/progression-customer-profile.json --track ecommerce +node dist/commands/composition-cli.js recipe validate tracks/ecommerce/composition/recipes/progression-catalog.json --track ecommerce +node dist/commands/composition-cli.js recipe show tracks/ecommerce/composition/recipes/progression-catalog.json --track ecommerce +node dist/commands/check-scenarios.js --track ecommerce --recipe progression-catalog.json +node dist/commands/check-composition.js +``` + +The example commands validate the existing profile path. Substitute the new pack path for the first command. The recipe output records selected check IDs, task fragments, and identities. Review the task text for every stack and selected depth. The dependency prompt contract test covers fresh builds and repair text through all current depths: + +```sh +node --test dist/tests/dependency-neutral-prompt.contract.js +``` + +## Add an expected production check + +The profile scenario also shows the negative case: another customer must not see the saved address. For the delivery note, save a note as one customer, open a separate account, and prove the note is absent. Use independent actors. Do not let a prior criterion's pass be the only evidence for setup. + +Put the privacy criterion in a `moduleType: "specification"` pack with a product justification: private delivery instructions belong to their owner. Select it for scoring through the node's grading groups. The primary product request remains the normal feature request. A condition that explicitly supplies safeguards is a separate treatment. Do not add probe strings or negative-test instructions to general stack guidance. The intended SpacetimeDB skills remain enabled. + +Prefer an authoritative fresh read after a write or replay. A blocked button alone does not prove server authorization. A request timeout, status 0, missing route, or server error does not prove correct rejection. For a successful authenticated replay, prove that the stored effect occurred once. Unauthorized replay must still be refused. + +Add a mutant that exposes the other customer's note while keeping sign-up and saving functional. Declare the exact scenario and stable check ID. Run the existing anchor and syntax tests. Then obtain current baseline, null-control, and targeted mutation evidence before describing the new check as qualified. Static tests do not establish a mutation kill. + +## Match the observation to the claim + +For bounded populations, scenario setup and criterion steps can use +`{ "repeat": 10, "steps": [...] }` or +`{ "forEach": ["Beta", "Alpha"], "steps": [...] }`. +`forEach` replaces whole string values equal to `"{value}"` with each listed +string. Both forms compile to ordinary actions with individual evidence. +They permit at most 1,000 repetitions and 10,000 expanded steps per list. +Nested expansions, expressions and runtime loop state are not supported. + +The catalog-volume scenario also saves `seeded-catalog` for the raw-evidence +audit. The audit compares the final stored population with individual write +receipts and the original fixture. Per-write observations alone cannot prove +that later writes preserved earlier products. Keep this final snapshot. + +- **Session persistence:** establish a session, reload, and observe the signed-in user without + `signIn`, `signUp`, or `ensureSignedIn` between the reload and the observation. +- **Reload persistence:** save data, reload, and observe it. Signing in again can isolate data + retention from session behavior. Browser storage survives reloads, so this alone does not + establish server persistence. +- **Hosted login refusal:** observe the provider's actual error, then use `reload` with + `application: true` to return to the trusted app URL in the same page. This preserves + app session storage. Check the app's signed-out state there. Once a protected operation + exists, probe it with `callAction` and `authentication: "optional"`: this sends any real + actor credentials, but permits a signed-out actor with none. Using `"none"` would discard + an illicit session and could hide an app that grants access despite provider refusal. + Prove the same operation works for an authorized actor and verify no unauthorized effect. +- **Server persistence:** use an independent client without copied application storage, or + suitable server evidence. Surviving a backend restart is a separate claim. +- **Restart survival:** first prove the saved state or ordinary scheduled operation works. + Restart the owned runtime without reseeding its data, then use a fresh client to verify + the result. Record which process restarted. A runtime-control failure is not an app + defect. This does not establish power-loss, storage corruption, or database crash recovery. +- **Shared live updates:** establish the observer's initial state, change data through another + actor, then observe without reload or re-navigation during the measured interval. Setup + reloads are valid. The initiating client's optimistic update is not sufficient evidence. +- **Autonomous execution:** closing clients is insufficient if the next request runs overdue + work. Use an observation that cannot trigger the work and a targeted negative control. + Otherwise state the narrower behavior measured. +- **Absence:** establish readiness first. `waitUntilAbsent` tests eventual disappearance; + `expect` with `absent: true` tests continued absence over its bounded `within` interval. + Neither establishes permanent absence. Same-actor observations can validly test deletion + or filtering; use an independent observer when the claim requires one. +- **Navigation:** reach the destination through disclosed controls. An optional control can + be absent; the required destination cannot. Do not assume a toggle is idempotent or require + a catalog round trip when the current view is already known. +- **Timing:** prefer completion signals. Keep elapsed-time waits when time is the behavior + under test. Explain unavoidable fixed waits in the scenario. Budget the full execution + path, including parallel branches, rather than only the longest individual wait. +- **Contention:** establish a successful serial operation, use independent actors, classify + every request, and reconcile stored state after the burst. A timeout is an unknown + business outcome until reconciled. Request overlap does not prove server execution + overlap or sustained capacity. A race mutation must preserve the serial operation. +- **Disclosure:** review the actual compiled request at the relevant step, including retained + contracts. Requested features may state timing or safeguards. Specification packs can + measure expected production behavior without requesting it. Disclose necessary interface + facts, but avoid layout restrictions and exact adversarial inputs. + +When a valid interface reveals a driver assumption, fix the shared action or the scenario's +entry sequence. Extend the closest executor test with the smallest alternate interface that +demonstrates the failure. Assert the destination or result so a no-op cannot pass. These +fixtures validate driver behavior; they do not qualify an application's business behavior. + +## Change a requirement or weight + +For a changed delivery-note length limit, edit the feature request, interface only if needed, and scenario assertions together. Keep the test input in the scenario. Preserve a stable ID only while the criterion still means the same thing. Give a materially different behavior a new ID and remove the retired selection. + +For a weight change, edit the criterion's `points` in its scenario. Check the compiled selection and graph score. Do not duplicate the weight in a report or UI. Check completion remains passed checks divided by the fixed selected check count; weighted score remains a separate measure. A zero-point control is excluded from scored check completion. + +Save `recipe show` output before and after the edit. Compare the selected checks, point totals, task fragments, and meaning/execution/content hashes. Use `recipe diff --track ecommerce` when comparing two authored recipes already in the track. Do not keep temporary duplicate IDs in the pack catalog. Re-run the matching scenario, composition, prompt, and mutation-definition checks. Regenerate the graph with `npm run graph` if the graph changed. + +## Add a study condition + +Copy the shape of `conditions/guidance/neutral.json`, choose a new ID, and register it in `conditions/catalog.json`. Change only the treatment you intend to compare. Preserve application interface selection and recorded skill identities. Select the condition in a campaign definition and compile the campaign with the existing campaign command. Do not add an agent-adapter branch for a guidance change. + +Declare repetitions, retry policy, budgets, and analysis before running. `analysis.spendThresholdsUsd` selects cost checkpoints; `analysis.completionTargets` selects target completion rates in `[0, 1]`. Reports use recorded grades only. Missing costs stay unknown and upper bounds remain marked. Cohorts retain stack, model, mode, level scope, skills, pricing, and definition identity, so a changed condition is not silently pooled with earlier results. + +## Review and qualification + +Check the rendered request and scoring scope separately. A prompt change, scenario change, reference fix, weight change, or runtime change makes evidence with the old identity stale. Current schemas derive qualification from identity-bound evidence; do not add retired `draft` or status fields to recipe or reference records. Pending qualification keeps scores provisional and blocks verified publication. It does not erase stored runs. + +Before release, require a baseline pass, a nonfunctional control, and a targeted failing mutant for the intended verified scope. Record fresh-build results separately from post-feedback repairs. Report blocked and unmeasured checks as part of the full scope. See [the current coverage review](grading-coverage.md) for known gaps. diff --git a/tools/stack-bench/docs/check-categories.md b/tools/stack-bench/docs/check-categories.md new file mode 100644 index 00000000000..46ff14710d0 --- /dev/null +++ b/tools/stack-bench/docs/check-categories.md @@ -0,0 +1,27 @@ +# Check categories + +A category describes the property a criterion tests. It does not describe how the grader reaches the app. + +- **Feature:** a requested product operation or result, such as finding an item or creating a ticket. +- **Production:** an access boundary, ownership rule, consistency property, durability, concurrent correctness, deduplication, or synchronization property. +- **Interface:** presentation or a test hook without a separate product or production assertion. The reservation countdown is the current example. +- **Unknown:** the frozen definition does not contain category metadata. Old results keep this label. Current source must not classify historical results retroactively. + +Read counts from the frozen campaign's selected criteria, not from every criterion +in a scenario file. A depth-limited campaign selects only part of the graph. +Zero-point setup controls remain evidence but do not enter scored check completion. +The [compiled recipe](../tracks/ecommerce/composition/README.md#authoring-commands) +owns category and point metadata; this document does not maintain a second count. + +`category` is authored on the scenario criterion and copied into the compiled recipe and feature catalog. `role` stays separate. Roles control progression and prerequisites; categories only split reports. Neither point weights nor dependency gates change. + +A mixed criterion is production when passing it requires a production property. For example, a refund check that also tests duplicate refusal is production. This coarse label does not separate which assertion failed. Split such a criterion only through a reviewed definition change with new qualification evidence. + +Categories do not prove that a guarantee was supplied without being asked. That requires an audit of the exact delivered request, contracts, and selected skills against the [request boundaries](prompting.md#request-boundaries). A production requirement can still be explicitly disclosed. + +Feature completion counts only fully passed dependency nodes. A node with passing feature checks but unfinished guarantees is not complete. Check completion counts accepted positive-point criteria. Both use the full selected target, including blocked and unmeasured work. + +The dashboard's Features/Checks choice changes the counting unit. It is not a +filter for the Feature category. A feature node can contain checks from several +categories. Weighted score is a third measure: passed points divided by selected +points. Keep all three denominators explicit in exported comparisons. diff --git a/tools/stack-bench/docs/credential-profiles.md b/tools/stack-bench/docs/credential-profiles.md new file mode 100644 index 00000000000..cc78d460b36 --- /dev/null +++ b/tools/stack-bench/docs/credential-profiles.md @@ -0,0 +1,53 @@ +# Named execution credentials + +A job selects credentials by profile ID. Selection order is attempt ID, adapter ID, +then default. Selection is explicit; the runner does not rotate accounts. + +Set `STACK_BENCH_CREDENTIAL_PROFILES_FILE` to an absolute path in trusted controller +storage. The file maps profile IDs to provider, mode, secret file, and version: + +```json +{ + "claude-work": { + "provider": "anthropic", + "mode": "subscription-token", + "secretFile": "/state/secrets/claude-work", + "version": "v1" + }, + "openai-api": { + "provider": "openai", + "mode": "api-key", + "secretFile": "/state/secrets/openai-api", + "version": "v1" + } +} +``` + +The example paths must be replaced with paths available inside the controller. +Use protected secret files. Do not put credential values in a job or campaign. + +Execution credential references have this form: + +```json +{ + "default": "claude-work", + "adapters": { "codex": "openai-api" }, + "attempts": { "an-exact-attempt-id": "claude-work" } +} +``` + +Profiles must match the selected adapter's provider. `anthropic` accepts API keys +or subscription tokens. `openai` accepts API keys or `subscription-token` mode; +that mode reads the existing Codex ChatGPT account login JSON file. `openrouter` +accepts API keys only. Normal provider preflight still validates authentication. + +Execution evidence stores only the profile ID, version, provider, and mode. +Secret contents and file paths are not attribution fields. Before each provider +invocation, the worker checks that its selected profile and secret have not changed +since admission. Update the profile version when deliberately replacing a secret. +Do not overwrite a secret used by an active attempt. Use a new profile for new work. + +Existing environment-based credentials continue to work when no named assignment +is selected. Profile selection clears conflicting credentials for that provider +and generic API-key overrides. It preserves other providers' credentials for +mixed-adapter jobs. diff --git a/tools/stack-bench/docs/dependency-graph.html b/tools/stack-bench/docs/dependency-graph.html new file mode 100644 index 00000000000..6612b67659f --- /dev/null +++ b/tools/stack-bench/docs/dependency-graph.html @@ -0,0 +1,1377 @@ + + + + + +Stack Bench | Dependency Graph + + + +
+
+
+

Ecommerce dependency graph

+

Each row is a questline. Select a feature to see what it needs and what it unlocks.

+
+
+ +
+
+ Qualified feature pack + Draft feature pack +
+
+ +
+
+ +
+
+
+
+ + + + diff --git a/tools/stack-bench/docs/development.md b/tools/stack-bench/docs/development.md new file mode 100644 index 00000000000..5445603ef65 --- /dev/null +++ b/tools/stack-bench/docs/development.md @@ -0,0 +1,303 @@ +# Stack Bench development + +This guide covers local source development. Use the +[appliance guide](../appliance/README.md) for runner configuration, credentials, +preflight, campaigns, and paid model work. + +## Requirements + +- Node.js 22 or newer +- Docker Engine with Compose v2 +- Chromium installed through the pinned Playwright dependency +- Linux for campaign and resource-lock tests; use a Docker development container + when the host is not Linux + +Install the locked dependencies and browser: + +```bash +cd tools/stack-bench +npm ci +npm run bootstrap:browsers +``` + +Build the local coding image: + +```bash +docker build --platform linux/amd64 -t stack-bench-build:local container +``` + +For real stack execution, use the [appliance build and setup](../appliance/README.md). +That build produces the CLI, server, and SDK from the branch. It needs no host +Rust build or ignored binaries. The appliance resolves local image tags to +immutable image IDs; published bundles use verified digest references. + +## Source checks + +Run the smallest check that covers the change: + +| Change | Check | +|---|---| +| TypeScript | `npm run typecheck` and the focused compiled test | +| Unit tests | `npm test` | +| Dashboard read model, routes, and pages | `npm run test:dashboard` | +| Repository contracts | `npm run test:contracts` | +| Mutation definitions and anchors | `npm run test:mutation-definitions` | +| Browser, process, and Docker integration | `npm run test:integration` | +| Prompt composition | `npm run check:prompts` | +| Track scenarios | `npm run check:scenarios` | +| Packs and recipes | `npm run check:composition` | +| Calibration | `npm run check:calibration` | +| Dependency graph | `npm run graph` | + +After a shared runtime, composition, grading, campaign, or release change is +stable, run the integrated source gate once: + +```bash +npm run lint +npm run typecheck +npm run test:all +``` + +What each test command runs, after its build: + +| Command | Files | Notes | +|---|---|---| +| `npm test` | `tests/*.test.ts` | Unit tier; use while changing code | +| `npm run test:dashboard` | `tests/dashboard/*.test.ts` | Writes thirty campaigns of fixture evidence | +| `npm run test:contracts` | `tests/*.contract.ts` | Tracks, prompts, references, repository policy, campaign definitions | +| `npm run test:all` | The three rows above | Not integration or mutation-definition tests | +| `npm run test:mutation-definitions` | `tests/*.mutation.ts` | Model-free | +| `npm run test:integration` | `tests/*.integration.ts` | Serial; can own browsers, processes, ports, and Docker | +| (manual) | `tests/dashboard/*.browser.ts` | Needs `STACK_BENCH_BROWSER_TEST_URL` pointing at an isolated appliance | + +Docker and qualification checks remain separate. +Campaign and lock tests exercise native Linux `flock`. A Windows host cannot +run those tests directly. Portable compiler and definition tests still run +locally. For a clean branch, the existing controller Dockerfile's `source` +target contains the source, development dependencies, and compiled tests: + +```sh +# From the repository root; this target does not build the Rust binaries. +docker build --platform linux/amd64 --target source -f tools/stack-bench/appliance/Controller.Dockerfile -t stack-bench-source-tests:local . +docker run --rm --init --network none stack-bench-source-tests:local npm run test:all +``` + +The release source build requires a clean normal Git clone. During development, +use a Linux container with the current edited checkout and locked dependencies. +Mutation-definition tests are model-free. Run them when reference source, grading +checks, or mutation manifests change. They do not run during ordinary unit work. + +Documentation-only changes need link and formatting checks, not the harness. +Run Docker checks only when the changed code affects their boundary. Run +targeted mutations while developing checks and the complete mutation set only +for a release candidate. Integration files run sequentially because they can +own browsers, processes, ports, and Docker resources. + +A passing check stays valid until one of its inputs changes. Do not rerun it for +reassurance. Add a test only when it protects a distinct invariant that an +existing test does not cover. Pending qualification marks campaign scores as +provisional; it blocks publishing verified comparisons, not campaign execution. + +For a pack with an unmeasured runtime budget, use `qualify-reference --timing-only` +to collect clean-reference timings before `pack-budget recommend`. This mode +cannot run mutations. Its artifacts are diagnostic and cannot qualify a release. +After setting the measured budget, run ordinary qualification on the frozen +candidate. Timing collection does not replace that gate. + +## Optional contention diagnostic + +`tracks/ecommerce/scenarios/diagnostic-checkout-contention.json` is a separate, +zero-point diagnostic. It does not run in scored campaigns. On a reset, +disposable reference app copy with its authenticated lease environment, use the existing +grader entry point: + +```sh +node dist/grader/grade.js --backend --app --url --level 2 --spec tracks/ecommerce/scenarios/diagnostic-checkout-contention.json --out +``` + +This standalone command deliberately omits `--track`. Both diagnostics include their named action mappings. Output is unbound to a recipe and has zero scored points; it cannot establish campaign completion. Backend reads still require the authenticated backend lease. + +This starts no coding agent. It changes application data, so do not point it at a +campaign app or its retained database. Prepare each stack with the same runtime +and resources. The probe requires the declared stock data interface and two +sessions of each fresh test account. It sends 1, 4, 16, and 64 parallel checkout +requests, with three fresh-account cohorts per width. These are request counts, +not distinct client counts or sustained throughput. + +The checkout diagnostic currently requires the verified reference schema. It does +not guess the schema of a generated app. Saved apps need a separate audited mapping. +Schema fingerprints are recorded with each stored-state observation. Unavailable +or malformed reads stay unmeasured, rather than becoming empty data or app failures. + +Each cohort records stored state before adding the item, after preparing the cart, +and after checkout. It requires one order with the correct owner, line quantities, +prices and warehouse allocations, one booked payment, the exact stock change, +and an empty cart with no remaining reservations. Existing orders and payments +must remain intact. PostgreSQL stores payment fields on the order; MongoDB and +SpacetimeDB use separate payment records. The shared assertion accepts either +stock reservation during cart preparation or stock consumption at checkout. +Rejecting all requests cannot pass. Retained action evidence contains each +request's timing, response status, and transport error or timeout. Timings cover +client dispatch through response, not server overlap or commit latency. Inspect +these observations separately from correctness. This draft diagnostic still +needs matching reference and targeted-defect qualification. + +Use `diagnostic-purchase-contention.json` with the same command to test competing +affordable purchases. It uses the declared `data-buy-input` interface and two +fresh customer accounts per cohort. Stock is reset to 128 in East and zero in +West before each cohort. Every request must be accepted, each account must show +its exact order count, and stored stock must decrease by the request count. +This diagnoses lost updates under bursts. It does not replace the separate +scarce-stock overselling check or measure sustainable throughput. + +The draft `diagnostic-checkout-application-crash.json` and +`diagnostic-checkout-database-crash.json` scenarios also have zero points and are +not selected by campaigns. They require a disposable, owned lease and a +`--restart-spec` with the backend, app path, port and probe. Select **one feature** +with `--feature` on freshly reset reference data for each trial. Do not run the +whole file on shared data: an earlier cart reservation can expire during a later +trial. SpacetimeDB uses only the database scenario because its application logic +and database share one process boundary. + +These probes kill owned processes with SIGKILL and restart them without resetting +storage. They record request outcomes, signal times and recovered business state. +Unconfirmed checkout effects may be absent or complete; partial effects and lost +confirmed state fail. SpacetimeDB calls use its native confirmed WebSocket protocol. +A separate checkout tests recovery progress. Application-crash recovery first +waits up to 70 seconds for the old database connections and transactions to end. +It records read-only observations and does not kill sessions or change timeouts. +If an HTTP call disconnects without proof that database work ended, stored-state +comparisons stay inconclusive until a fresh grade on reset data. A proven app +recovery failure still fails the recovery check when the crash window is valid. +A client timeout does not prove that server work stopped. Missed fault windows also remain +inconclusive. These are process-crash tests, not power-loss tests or proof of a +crash at a particular instruction inside a transaction. The diagnostics remain +draft and outside scored campaigns. + +Run independent diagnostic cases through the reference command inside the Linux +appliance. Set `STACK_BENCH_CONTROLLER_IMAGE_ID` and `STACK_BENCH_IMAGE` to immutable +image IDs. No model credentials or paid calls are needed. + +```sh +node dist/src/references/reference-live.js --diagnostic-plan /evidence/diagnostics.json --out /evidence/result.json +``` + +The plan selects existing zero-point scenarios and imported references: + +```json +{ + "schemaVersion": 1, + "groups": [{ + "backend": "postgres", "track": "ecommerce", "level": 3, + "recipe": "ecommerce.progression-catalog", + "scenario": "/workspace/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-database-crash.json", + "features": [9800, 9803], "repetitions": 10 + }] +} +``` + +Each feature gets its own leased worker, source copy, database and ports. The +worker builds once and resets data between repetitions. Host resource admission +controls startup. Use `--diagnostic-workers 1` for the same execution path in +serial, or set a per-command concurrency limit. There is no additional host cap. + +For a disposable candidate, add `source: { "path": "...", "sha256": "..." }`. +Its dependency files and deployment metadata must match the imported reference. +Declare exact criterion IDs in `expectedFailures` for defect controls. Source +paths and scenario paths are relative to the plan. Candidate source is copied; +the supplied tree is never edited. + +For an accepted saved L3 app, use `saved` instead of `source`: + +```json +{ + "run": "/evidence/attempt/run.json", "runSha256": "", + "checkpoint": 11, "source": "/evidence/accepted-source", + "reader": { "path": "/evidence/reader.json", "sha256": "" } +} +``` + +This path requires the final accepted checkpoint, its source and selection hashes, +and the original build image, run index, and database or module address. It installs +the app's own dependencies and does not deploy a reference app. Saved SpacetimeDB +apps require `STACK_BENCH_RELEASE_DEPS_VOLUME`, initialized from the original backend +image with the existing `appliance/dependency-volume` command. The runner verifies +the mounted SDK and native binaries against that backend image's manifest before +app startup. This volume supplies stack artifacts, not the app's `node_modules`. +Each trusted reader JSON contains `sourceSha256` and a reviewed mapping: + +- PostgreSQL: `sql` uses `:'account'` and `:'item'` in a read-only, repeatable-read transaction. +- MongoDB: `script` reads through `store` in an aborted snapshot transaction. It receives `account`, `item`, `key`, and `minor` helpers. +- SpacetimeDB: `tables` selects one native subscription snapshot; `convert(tables, account, item)` maps its rows. + +Each mapping returns `accountMatches`, `itemMatches`, and `state`. Both counts must +equal one. Reads require the owned container. Mapping programs are trusted operator +code, never supplied by the tested app. SpacetimeDB connection hooks can change +state before a fresh subscription; disclose this limit when the app has such hooks. + +Saved order-only state includes allocations and orphan counts. Map separate refund +records when present. Use `refundedMinor: null` when no order refund amount is stored. +These mappings cover selected accounting fields, not the entire database. They cannot +claim payment or reservation coverage. They support checkout and crash recovery; +direct-purchase histories and cancellation require separate qualified mappings. +Every new source needs a reviewed reader and deliberate defect controls before +running the audit. Saved-app failures are measured results, not expected-control +failures. All results remain zero-point diagnostics and leave prior scores intact. + +The output retains planned, started, collected, interrupted and unstarted trials. +Collected includes inconclusive results; it does not mean qualified. Raw grades +retain setup, action and assertion timing. Worker audits add deployment, reset, +grade and cleanup time. Unexpected failures stop new trials. Active trials finish; +explicit cancellation stops owned processes and releases their leases. +`--diagnostic-resume /evidence/previous.json` with a new output resumes only whole +groups that were never dispatched, under the same plan and images. Interrupted +executions stay visible and require an explicit new study to rerun. + +## Agent adapter contract + +Register an agent in `src/agents/agent-adapters.ts`. The existing registry accepts +a Node entry point; a new provider does not need another runner or result format. +Use `AgentRequest` from `src/agents/agent-adapter-contract.ts` and +`ValidatedAgentResult` from `src/agents/agent-result-contract.ts` as the protocol. +The runner sends arguments without a shell. The final non-empty stdout +line must contain one result JSON object. Earlier lines can contain logs. + +The request carries the selected model, mode, app directory, visible task and +guidance. Preserve them exactly. Do not expose grading definitions to the agent. +Declare the modes, credentials, network destinations and cost limits the adapter +actually supports. Registering an entry point requires rebuilding the release; +there is no runtime plugin loader. Adapter identities bind its entry-point bytes +and declared settings, including grading credentials. The release binds the +remaining source files. + +The runner validates results and deducts reported cost from the attempt's shared +budget. Unsupported cost limits fail before launch. A paid adapter must also use +the existing appliance, credential and cost-receipt controls; declaring a native +cost limit is not proof that an external scaffold enforces it. A standalone agent +has a deadline. An authenticated campaign delegates that deadline to its +supervisor so time grants remain effective. Cancellation stops the owned process +tree and group. Captured output is limited to 64 MiB per stream; exceeding that +limit rejects the result. + +Claude and Codex use the same `commands/agent.ts` prompt and grading path. +Their CLI arguments, process handling, and result parsing live in +`container/coding-providers.ts`; their trusted API forwarding and usage parsing +live in `container/broker-protocols.ts`. Add a provider there when its protocol +differs. Keep authentication in `container/container-auth.ts`, outside the +coding container. A new provider must supply normalized token usage, preserve +its tool transcript for the shared audit, and enforce the plan's cost bound. +Test it with a local mock upstream before a paid run. Do not copy the container +runner or add provider conditions to prompts, grading, or campaign scheduling. + +`tests/agent-adapters.test.ts` sends a compiled visible task to an independent, +model-free entry point in all four modes. It checks exact delivery, shared budget +accounting and process cleanup. These tests qualify the protocol boundary, not a +new provider's billing integration or a complete install-to-run walkthrough. + +## Generated files + +Run `npm run graph` to rebuild `docs/dependency-graph.html` from the versioned +ecommerce graph. Do not edit generated output by hand. + +Build output, run artifacts, transcripts, local plans, and operational notes are +not product documentation and must remain untracked. diff --git a/tools/stack-bench/docs/execution-jobs.md b/tools/stack-bench/docs/execution-jobs.md new file mode 100644 index 00000000000..b8d24ad9639 --- /dev/null +++ b/tools/stack-bench/docs/execution-jobs.md @@ -0,0 +1,130 @@ +# Submit execution jobs + +A campaign describes a test. A job selects where and with which credentials to run it. +Each job runs one whole campaign on one host. Multiple workers can run different jobs +at the same time. The runner preserves the campaign's requested parallelism. + +## Submission + +Store the campaign manifest under the appliance results `plans/` directory. Use a frozen +plan for paid work. A draft is accepted only for a non-billable, model-free trial. +Configure [named credential profiles](credential-profiles.md) in trusted worker storage. + +Create a submission file: + +```json +{ + "key": "release-42-l2-repairs", + "planFile": "l2-repairs.json", + "credentials": { + "adapters": { "claude-code": "claude-work", "codex": "openai-api" } + }, + "hostId": "worker-east", + "capacityPolicy": "wait" +} +``` + +Only name adapters present in the plan. Credentials can also have a `default` and an +`attempts` map keyed by exact compiled attempt IDs. Attempt selections take precedence. +Omit `hostId` to let an eligible worker claim the job. This field restricts placement; +it is not host authentication. Omitting credentials retains the existing operator environment. + +Through the controller: + +```sh +job submit submission.json +job status +job list --limit 50 +job work --host worker-east +job cancel +``` + +For source development, use `node dist/commands/job-cli.js` before these arguments. +Set `STACK_BENCH_RESULTS_DIR` or pass `--results`. The normal appliance controller command +sets runtime image identity for `job work`. A worker must use the matching frozen controller +and coding images. Named secret paths must exist on that worker. + +Submission snapshots the plan. Repeating the same key and request returns the same job. +Reusing a key for different inputs fails. Submission does not start a model call. +`job work` claims and runs one job; an existing task queue can invoke that command on the +chosen worker. Credentials are resolved and pinned at attempt admission, not at submission. + +## API and service integration + +The existing local dashboard controls expose: + +- `POST /api/jobs`: submit the JSON above; returns 202 and the durable job status. +- `GET /api/jobs?limit=50&after=`: list one page. +- `GET /api/jobs/`: read status, assigned host, capacity wait, and campaign directory. +- `POST /api/jobs//cancel`: request cancellation. + +Writes require the same origin, browser token, and control-secret headers as existing +dashboard controls. This remains a local operator API. An authenticated product service +can instead call `submitExecutionJob` and `workExecutionJob` from +`src/campaigns/execution-jobs.ts`. Authenticate callers and authorize credential/profile +access before calling them. Scope idempotency keys by caller in that service. + +The job records contain references, not secret values or secret file paths. Per-execution +evidence records the admitted credential profile and version. Unexpected credential changes +fail before further provider calls. No automatic account rotation occurs. + +## Ownership, waiting, and recovery + +### Automatic local dispatch + +Run a worker to pick up queued jobs without invoking `job work` for each submission: + +```sh +job worker --host worker-east --concurrency 2 +``` + +Concurrency here counts **campaign jobs**, not attempts. Two jobs can each run nine +attempts. Each campaign retains its selected parallelism. There is no fixed job ceiling; +set concurrency to the work the host and selected provider accounts can support. +The worker checks host assignments and uses the same exclusive job claims as `job work`. +It polls the local job store once per second when idle. No second queue or dependency is used. + +The appliance provides an opt-in `worker` Compose profile. Set `STACK_BENCH_HOST_ID` +and `STACK_BENCH_JOB_CONCURRENCY`, then start the `worker` service with the normal setup +environment. Starting it authorizes execution of eligible queued jobs. Do not point an +experimental worker at a live queue. Use the controller image required by those plans. + +SIGTERM/SIGINT stops new claims and waits for active jobs. Use `job cancel` to stop a +specific campaign. Compose allows 24 hours for draining; override `stop_grace_period` +if admitted jobs can run longer. A forced kill retains claims and requires inspection. +A job failure stays recorded while the worker continues. A store or pre-claim error +stops admission and drains active work, so broken input does not enter a retry loop. + +This dispatcher is for the local appliance. At large backlog sizes, use the surrounding +product's durable queue to call `job work`; the local store scans directories. A production +multi-host deployment also needs shared credential quotas, a durable central job store, +and explicit evidence transfer. These are not supplied by the local dispatcher. + +Workers claim jobs with an atomic immutable record. A second worker cannot launch the +same job. Claims do not expire: a worker that loses contact may still have paid requests +in flight. A killed worker therefore leaves a retained claim for investigation rather than +an automatic duplicate. Use campaign status, stop, and authenticated reconciliation to +resolve owned resources. Failed jobs are not automatically retried by `job work`. +Reconciliation proves cleanup; it does not restore a live database or agent session. +See [interruption and recovery](../appliance/RECOVERY.md) before releasing retained work. + +The worker reserves only the actual stack resources for a dispatched attempt. It releases +them after verified cleanup. `capacityPolicy: "wait"` retains pending work and reports the +capacity wait; `"fail"` returns the resource error. Configuration and credential errors are +not retried as capacity waits. Cancellation reaches the runner and its cleanup path. + +Use the same local resource-lock root for all controllers targeting the same Docker host. +For separate hosts, those locks and runtime/work paths must be host-local. A shared job +store must support atomic hard links, rename, and durable writes, and all workers must see +the same job/result paths. Test these properties before using a remote filesystem. + +## Current boundaries + +- Placement is per campaign. One campaign's attempts are not distributed across hosts. +- This is not a replacement for the surrounding product's queue, authentication, or secret store. +- Shared provider quota and account-wide spend controls are not supplied by this job store. + Existing per-attempt money limits and staggered provider retries remain enforced. +- Full campaign state is still materialized. The compiler rejects work that cannot fit + numeric, array, serialization, or available heap limits before expansion. Removing the + old repetition ceiling does not make memory unlimited. +- No production multi-host throughput claim is made by the model-free local tests. diff --git a/tools/stack-bench/docs/grading-coverage.md b/tools/stack-bench/docs/grading-coverage.md new file mode 100644 index 00000000000..f41b8ffb640 --- /dev/null +++ b/tools/stack-bench/docs/grading-coverage.md @@ -0,0 +1,313 @@ +# Grading coverage and limits + +This guide explains what the checks observe, what they do not establish, and how +to review a failure. Current check counts and points come from the +[compiled recipe](../tracks/ecommerce/composition/README.md); qualification +status comes from the calibration and its evidence +(`qualification status --track --level --recipe `). This +document does not maintain a second count or status. + +Old runs keep their original definition; a new interface requirement is not counted against a saved application that +predates it. + +## Scored checks and diagnostics + +Scored checks belong to the selected recipe and count toward completion. +Diagnostics are reference-only observations outside scored campaigns. They add no +feature points and do not change historical results. + +- **Purchase contention** (optional purchase, scarce-stock, and restock + scenarios) uses the request recorder and verified reference database readers. + It compares accepted purchases with each buyer's new orders and payments, + preserves earlier records, and reconciles stock against order allocations and + restock requests. Scarce stock limits accepted sales; ample stock requires every + purchase to succeed. It establishes net per-warehouse conservation and + per-buyer counts. It does not identify each order by a durable request ID, + expose every compensating error, or prove intermediate state, server execution + overlap, crash safety, or sustained throughput. +- **Concurrent cancellation** (`diagnostic-cancellation-contention.json`) sends + overlapping cancellation calls from two sessions of one account. It verifies + the pending order first, records every request outcome, then checks stored + warehouse allocations, cancelled status, preserved order and payment history, + and fresh revenue. Repeated calls may refuse or succeed without additional + effects. It covers non-credit orders only. + +Both use verified reference schemas. Saved model apps need verified reader +mappings before these observations apply. The +[contention diagnostic](development.md#optional-contention-diagnostic) runs +repeated request bursts; it is not a capacity test. + +## Observation rules + +- **Navigation.** Scenarios work with separate pages and single-page views. + After reload, reopen a declared entry control when it exists, then require the + destination to be visible before inspecting it. An absence assertion must not + pass merely because the whole view is closed. Select account rows by account + identity, not text shared by their role options. +- **Readiness markers.** A destination marker identifies the opened view, + including while it loads. `aria-busy` is false only after the signed-in + account's list loads successfully, including an empty result, so loading or + failed reads cannot earn empty-list credit. Submission-state markers report + success on the acting control before the grader proceeds. Markers are + app-reported evidence; fresh business observations still establish the effect. +- **Stored stock.** Stock reads use the item, warehouse, and stock interface + already required for external corrections, through the authenticated backend + lease. Zero and negative quantities remain observations. Missing, invalid, or + ambiguous data cannot become a fabricated zero or a passing comparison. + PostgreSQL resolves the declared relational links; MongoDB and SpacetimeDB use + their declared stock interfaces. This is not an independent read of arbitrary + application tables such as payments or orders. +- **Business effects.** Price, transfer, cancellation, and return checks prove the + original business effect before its preservation or reversal. Authorized + operations establish a working route before refusal checks. Idempotent success + is accepted when fresh observations prove one business effect. +- **Live updates.** Dedicated live-update checks keep their observers on the open + page. Other checks use fresh views, and stored reads follow the live assertions + so they cannot give the UI extra time. +- **Concurrent calls.** Named concurrent calls retain request timing and + distinguish responses, transport errors, and timeouts. A timeout has no known + business outcome until state is reconciled. Client request overlap is not proof + of overlap inside the server. +- **Privacy.** Capture includes HTML, JSON, text, native EventSource, and the + WebSocket decoder. Positive owner observations establish that the data was + delivered. Dropped, unreadable, or unfinished evidence cannot establish absence. + Fetch-based SSE streams are not supported and fail closed. +- **Deferred work.** Checks anchor on the original time and use early and late + observations. Missing an observation window is unmeasured, not an app failure. + The probes do not change the host or client clock. +- **Source snapshots.** Application snapshots exclude `.log` and `.pid` files at + every depth. Required source and seed inputs must be in source files. Clean + reconstruction must work without excluded runtime files; use clean + reconstruction for reproducibility claims. + +## Notes on specific checks + +- **Shipping result** waits for the declared submission state, then verifies + fresh staff and customer views. It does not require the queue or customer view + to update live. The separate live fulfilment check covers new orders appearing + in an open queue, not live removal or live customer-status updates. +- **Shipping accounting** uses the named shipping action with staff credentials + and waits for an accepted server response before fresh order, stock, and + revenue reads. It does not test the shipping button; the UI check owns that. +- **Support refund accounting** checks a second, unrefunded order after replay and + fresh login, detecting refunds applied beyond the selected order. +- **Return and refund (L6)** exercises both operation orders. Accept the physical + return once, restore stock once, and refund only the amount still owed. + Cumulative refunds cannot exceed the amount paid. +- **Low-stock live** keeps its observer on the open list while a separate + administrator restocks. It does not assume the admin area resets its subtab. +- **Stock alerts.** The initial alert request must report successful submission + before the first restock; a rejected or unconfirmed submission stops setup. The + duplicate-alert check samples a fresh client ten seconds after the second + restock. It is not continuous observation and does not exclude later + duplicates. A read that triggers overdue work can still pass, so this does not + establish autonomous notification execution. +- **Restock race** requires an ordinary stored restock in setup, then verifies + stored stock, each buyer's order, and UI agreement. +- **Staff roles (621b)** requests a different role through HTTP and reducer + replay, then reloads the administrator view to verify no role changed. **621d** + checks administrator-role removal with the same signed-in staff session before + and after. `admin` grants administrator access; `staff` and `inventory` do not. + It does not establish subscription revocation or token logout. +- **Login input (101a)** replaces the password in one captured JSON login request + with query-like text. A protected purchase must be refused and stored stock + unchanged, with normal login working before and after. This does not establish + general SQL or NoSQL injection safety. Missing, ambiguous, repeated, redirected, + non-JSON, or incomplete captures are inconclusive. +- **Checkout crash integrity and acknowledged-order durability** are owned by the + checkout feature. PostgreSQL and MongoDB use separate application and database + process crashes. SpacetimeDB uses one combined process crash, with an explicit + shared observation for the application boundary. An uncertain outcome or a + missed fault window is unmeasured. + +## Expected production criteria + +Each specification family has a product reason. Its scope follows the selected +product features. + +| Specification family | Product reason and acceptance rule | +| --- | --- | +| `ecommerce.spec.state-durability` | Separate session continuity from saved data. Check cart, orders, profile, preferences, staff roles, and support history after runtime restart and fresh login. Retain reload checks for browser continuity. | +| `ecommerce.spec.access-control` | Customer data and staff operations have different owners. Test direct server calls as well as the visible interface. A refusal must have a defined result; transport failure is not proof. | +| `ecommerce.spec.live-state` | Shared catalog, inventory, cart, and operations views must reflect changes where the product calls for live information. Use distinct actors and bounded waits. Polling is acceptable when it meets the same observable rule. | +| `ecommerce.spec.concurrency-safety` | Several valid customers can act at once. Classify every request, allow stack-specific conflict results, and prove stock/cart/order invariants. Do not prescribe locks, reducers, or queue design. | +| `ecommerce.spec.external-data-sync` | A shared data view must not depend only on one client's local writes. Retain equivalent stack-specific mutation paths and fresh observations. | +| `ecommerce.spec.transactional-integrity` | Stock and money cannot be created or lost by partial operations. Prove the before/after quantities and totals, including rejected overdrafts. | +| `ecommerce.progression.cancellation-queue-specifications` | A cancelled order must leave the work queue. Keep queue visibility distinct from monetary accounting. | +| `ecommerce.progression.cancellation-accounting-specifications` | Cancellation must reverse only the appropriate stock and revenue effects. Repeat requests cannot reverse them twice. | +| `ecommerce.progression.price-accounting-specifications` | Current price edits must not rewrite earned revenue. Historical and future prices have different meanings. | +| `ecommerce.progression.price-history-specifications` | A buyer's receipt must keep the agreed purchase price. The scenario owns exact probe prices. | +| `ecommerce.progression.inventory-conservation-specifications` | A warehouse transfer changes location, not total inventory. Reject insufficient stock with no partial effect. | +| `ecommerce.progression.operations-access-specifications` | Administrative changes must follow product roles. A hidden button alone is insufficient. | +| `ecommerce.l3.deferred-access-specifications` | Scheduled work is still an authorized business operation. Schedule creation and execution cannot bypass access rules. | +| `ecommerce.l3.deferred-durability-specifications` | Reservations and scheduled restocks must survive the specified restart. Do not score a restart failure as an application assertion. | +| `ecommerce.l3.deferred-integrity-specifications` | Deferred work must produce one business effect. A replay can return success when stored state still proves one effect. | +| `ecommerce.l3.server-time-specifications` | Observe reservation expiry with its browser closed and scheduled work after restart. These probes do not establish clock-skew tolerance. | +| `ecommerce.progression.review-access-specifications` | Review ownership and visibility follow the product's role rules. Exercise the direct access path and an independent observer. | + +Keep first-build measurements separate from post-feedback repairs. Do not call a +score “production readiness”: these checks cover the declared product behaviors, +not all security, accessibility, operational, or performance requirements of a +deployed service. + +Concurrent requests need classified outcomes. Keep adversarial values in +scenarios, not product interfaces. For authenticated idempotent replay, verify +unchanged totals and one business record through a fresh authoritative read. +Unauthorized replay remains a separate refusal check. These rules follow the +distinction between interface and server authorization checks in the +[OWASP authorization testing guide](https://owasp.org/www-project-web-security-testing-guide/v42/4-Web_Application_Security_Testing/05-Authorization_Testing/02-Testing_for_Bypassing_Authorization_Schema), +and its advice to verify business data in +[integrity tests](https://owasp.org/www-project-web-security-testing-guide/v42/4-Web_Application_Security_Testing/10-Business_Logic_Testing/03-Test_Integrity_Checks). + +## Claim limits + +| Observation | Does not establish | +| --- | --- | +| Hosted app restart for PostgreSQL/MongoDB; SpacetimeDB runtime restart with retained data | Common database crash semantics, power-loss recovery, or corruption recovery | +| Checkout interrupted by an application or database process kill; recovered cart and orders reconciled with recorded requests | Power loss, disk corruption, every crash timing, or external payment durability | +| Private marker absent from supported captured responses | All endpoints, encodings, binary formats, or arbitrary object-reference attacks | +| Exact final stock, orders, and totals | Every intermediate state, general serializability, or an external payment ledger | +| Bounded concurrent requests | Sustained throughput, many independent users, or server execution overlap | +| Serial promotion redemption limit | Concurrent competition for the last redemption | +| Hidden return/activity controls and displayed activity fields | Server-side return authorization, audit-log confidentiality, or tamper evidence | + +Later-depth limits are listed with the [ecommerce levels](../tracks/ecommerce/LEVELS.md#later-depths-l4l6). +Chat has additional qualification blockers in [its level notes](../tracks/chat/LEVELS.md). + +## Controls and qualification + +Source coverage and executed controls are separate evidence. A declared mutation +target is not a successful control, and a failed setup is not a target kill. +Use the current graph, recipe, reference registry, and mutation manifests as the +source of truth. The [mutation coverage checks](../tests/progression.mutation.ts) +report missing exact depth 1–3 targets. Source checks do not replace live controls. + +For a race control, preserve ordinary serial behavior and challenge the +concurrent case. For restart survival, preserve execution before restart; a +disabled timer only proves detection of absent execution. The restart probe first +completes an identical ordinary timer. PostgreSQL and MongoDB controls remove +pending work at startup. The SpacetimeDB control keeps pending rows but loses its +process-local execution queue. + +The restock probe first verifies an ordinary purchase and restock. PostgreSQL and +MongoDB controls replace atomic reservation with an unlocked read and absolute +write. SpacetimeDB reducers remain atomic; its control sends stale absolute stock +from the client, then overwrites intervening purchases. These are distinct ways +to break the same stock invariant. Fixed delays widen overlap in defect controls +only; they do not measure a natural failure rate. + +Before a verified comparison, resolve material defects in the selected scope and +collect matching reference, null-control, and mutation evidence. Keep missing +definitions, unexecuted controls, failed or surviving controls, and stale evidence +separate. Record the exact source, engine, recipe, fixture, and result identities +beside each executed control. Do not copy old evidence into a changed calibration. +Static source inspection cannot prove that timing thresholds are attainable on the +appliance, that all reference stacks pass, or that a mutant fails only its target. + +An exploratory campaign can proceed with pending qualification, but its scores +are provisional. A gap outside its selected scope does not block it. A signed +distribution and its [release verification](../appliance/RELEASE.md) are separate +from grading qualification. + +For each content finding, record the check and owner, material delivered at the +relevant step, observation and timing assumptions, a valid alternative +implementation, control evidence, and disposition. Follow the +[authoring rules](authoring.md) when extending the workload. + +## Review a failed check + +A failed check records an observation that did not meet an assertion. It does not +identify an independent bug or prove its root cause. Several checks can fail from +one missing update path or one failed setup step. + +For each investigated failure, keep a short review beside the retained evidence: + +- Identify the campaign, execution, source hash, stable check IDs, and grade files. +- Record the observed result separately from the proposed cause. Read the action + evidence and setup result before the final assertion. Link relevant traces, + logs, and source lines. +- State whether evidence confirms an application defect or a harness defect, or + whether the cause remains unresolved. Keep provider and interrupted outcomes + separate. A valid failed assertion can have an unresolved application cause; + an uncertain measurement cannot establish an application failure. +- Group checks only when evidence supports a shared cause. Keep every check's + recorded outcome and score. Do not report the group count as a measured bug count. +- State what evidence is still needed and which focused check can supply it. + +Keep claims within the measured boundary. A button shown to a guest proves a UI +visibility failure only when the contract forbids it. It does not prove the server +accepts a guest purchase. A missing stock number does not prove overselling. Use +direct-call results and stored quantities to assess those claims. + +If review confirms a grader defect, preserve the original artifacts and explain +which comparisons are invalid. The automatic report reads artifact outcomes; a +review note does not change its classifications or scores. Fix the shared grader, +verify the affected behavior, and regrade the unchanged saved app into separate +evidence. Record the corrected grader identity and its relationship to the original +result. Do not present the original affected score as a valid comparison or count +the regrade as a new independent app build. + +For dependency runs, a corrected gate can change which work the agent receives +next. A regrade can measure the saved application, but cannot reconstruct that +different development path. Keep it separate from new attempts under the corrected +definition. Show raw checkpoint checks beside accepted target completion and +blocked descendants; blocked checks are not independent observed failures. + +### Replay a saved dependency candidate + +Use the `run` command with `--grade-from`, an explicit `--grade-level`, one or +more `--check` IDs, and a fresh `--out` directory outside the original execution. +The depth selects that depth's saved first-build candidate. It does not select +the final accepted app, which can be an earlier depth after rejection. + +```sh +node dist/commands/bench.js --grade-from /results/original/execution-1 \ + --grade-level 2 \ + --check ecommerce.spec.state-durability.session-reload.1e \ + --check ecommerce.spec.access-control.warehouse-write-boundary.103b \ + --out /results/diagnostics/session-and-authorization --no-media +``` + +Run this inside the configured Docker controller environment. The appliance +controller accepts the same arguments after `run`. No provider credentials or +model calls are required. The replay uses the original coding image, a fresh +owned backend and app directory, the saved credential aliases, and current checks. +It rebuilds the app from saved source; it does not restore old database contents. + +Independent diagnostic commands can run in parallel. Each claims its app ports +and backend resources through the same lease system as campaigns, and admission +reports resource conflicts. Use a separate output directory for each command. +Replays keep their saved run index and server endpoint, so candidates that need +the same ports must run at different times. + +Preserve the original generation dependencies. For example, regenerating STDB +bindings with a newer CLI can change their embedded version header and correctly +fail source verification. Set `STACK_BENCH_RELEASE_DEPS_VOLUME` to the original +release volume. Verify it with that release's immutable controller image and +`verify-deps`, mounted read-only. Then use Compose `run --no-deps` with the new +controller so its dependency initializer does not replace the old tooling. Keep +the new controller and backend identity in the diagnostic record. + +The source run must have finished and must not be contaminated. The selected +depth must identify one saved first-build candidate with matching source and +grading evidence. Checks must belong to that candidate's original scored scope. +Missing evidence, changed source, ambiguous depths, and overlapping output paths +are errors. An inconclusive original measurement can be investigated; the replay +does not make that original result valid. + +Read `regrade.json` and its separate `grading/bundle.json`. The receipt identifies +the original run, source candidate, original grading evidence, current definition, +and cleanup result. It is diagnostic evidence, not a campaign run. Do not add its +checks or zero additional model cost as another trial in a stack comparison. + +Without `--grade-level`, the single-level sequential regrade retains its original +product-contract and scoring-scope checks. Dependency replay deliberately permits +current check definitions and records the difference. + +Grading bundles include optional `phaseTimings` for application stop, database +reset, application start, readiness probes, and grader execution. Durations use a +monotonic clock and include failed operations. `suite: null` identifies +preparation before the scenario loop. `threw` records an exception, not whether a +check passed. The grader duration includes its child process and evidence +handling, so do not add it to the child grade duration. These timings are +diagnostics and do not change scores or timeout budgets. diff --git a/tools/stack-bench/docs/how-it-works.html b/tools/stack-bench/docs/how-it-works.html new file mode 100644 index 00000000000..2216ae58feb --- /dev/null +++ b/tools/stack-bench/docs/how-it-works.html @@ -0,0 +1,826 @@ + + + + + + +Stack Bench · the proving ground + + + +
+ + STACK BENCH +
+
+

+ +
+ + + + diff --git a/tools/stack-bench/docs/prompting.md b/tools/stack-bench/docs/prompting.md new file mode 100644 index 00000000000..398be3cca14 --- /dev/null +++ b/tools/stack-bench/docs/prompting.md @@ -0,0 +1,356 @@ +# Prompting method + +Stack Bench gives the coding agent a normal software request. It does not tell +the agent that it is in a benchmark. What Stack Bench asks for and what Stack +Bench measures are separate choices. + +## Prompt inputs + +Each request is assembled from these owners: + +| Input | Purpose | Owner | +|---|---|---| +| Product framing | Says whether to build a new app or add work to an existing app | Recipe | +| Current features | Describes the product work to implement now | Feature packs | +| Requested production behavior | States production requirements that the campaign chose to disclose | Specification packs | +| Stack material | Gives required access details and the selected level of technical guidance | Guidance profile and backend document | +| API reference | Supplies selected SDK material, including SpacetimeDB skills | Guidance profile | +| Starting data | Gives the new app's fixed catalog; later requests retain original entity names and relationships without resetting live data | Fixture | +| Application interface | Names the controls or operations needed for reliable use | Feature contracts | +| Repair report | Describes conclusive application failures from the last grade | Condition repair policy | + +The recipe and selected packs own the text. The prompt builder orders that text +and adds the small controller contract, such as the application directory, +listening address, start script, and completion response. + +## What the coding agent receives + +A new-build request has this shape: + +```text +Build the application described below and leave it running. + +Build the app in /app. +The web application must listen on 0.0.0.0. + +## Stack + + +## Selected API reference + + +## New application + + +## + + +## Starting catalog + + +## Application interface + +``` + +This is an abridged example. The exact request is composed from versioned files +and bound to the campaign by hashes. + +The new-build request does not include: + +- grader source or scenario files; +- check names, point values, expected scores, or comparison results; +- exact adversarial inputs chosen by a scenario; +- future dependency nodes that are not ready; +- production expectations assigned to the `expected` or `observed` treatments. + +## Features and production expectations + +A feature is product work. A production expectation describes how selected +features should behave under conditions such as reload, reconnect, concurrent +writes, authorization boundaries, or direct data changes. + +Each selected production expectation has one treatment: + +| Treatment | Included in request | Main score | Repair feedback | +|---|---:|---:|---:| +| `requested` | Yes | Yes | Yes | +| `expected` | No | Yes | Yes, after a conclusive failure | +| `observed` | No | No | No | + +`Expected` lets a no-repair study measure production behavior that was not +explicitly requested as a specification. The supplied features, interfaces, and +skills can still disclose related expectations. Audit their exact text before +claiming a behavior was supplied without being asked. + +`Observed` is a separate first-build diagnostic. It cannot change the main +score or steer repairs. + +### Example: expected durability + +The campaign selects account creation as current work. It also selects session +durability as expected production behavior. + +The coding agent sees product text such as: + +```text +## Accounts + +Visitors can create an account with a username and password. Returning users +can sign in, see which account is active, and sign out. +``` + +The request does not mention reload behavior. Stack Bench can still verify that +the signed-in session survives a reload. A conclusive failure affects the main +score and can produce repair feedback. + +### Example: requested durability + +The campaign selects the same account feature and changes durability to +`requested`. The request now also includes text such as: + +```text +## State durability: accounts + +A signed-in session survives a page reload as the same account. +``` + +The scored behavior is the same. Only disclosure changed. This makes the two +conditions comparable without changing the feature itself. + +### Example: observed durability + +The campaign changes durability to `observed`. The request again omits reload +behavior. Stack Bench measures it after the first build, records the result as +a diagnostic, and does not include it in the score or repair report. + +### Request boundaries + +A product request states what the product does, clearly enough that its +semantics are not ambiguous: buyer reviews, cancellation before shipping, +reservations, stock scheduling, and account features. "A shipped order becomes +delivered after 60 seconds" defines a feature. Restart survival, duplicate +execution, clock authority, and cross-account access do not need implementation +instructions in that request. Product policy stays explicit where a reasonable +implementation could differ, such as refund and restock rules or verified-buyer +reviews. Measure the enforcement of that policy separately. + +Neutral requests do not state atomicity, conservation, exactly-once, no-reload, +or cross-account isolation instructions. Those belong to specification packs and +enter the request only when a study selects them as `requested`. + +The **Production-quality app** option adds one line to the request: “Build a +production-quality application suitable for real users, not a prototype or +demo.” It is on by default and recorded with the run. It changes the request, +not the checks. Compare results with the same setting, or label the difference. +Plans set `productionQuality: false` to opt out. + +## Stack guidance + +Stack selection and guidance selection are separate. + +- Neutral guidance gives the required stack, connection details, startup + contract, and selected stack material. The coding agent chooses libraries, + architecture, and project structure within those requirements. +- Prescribed guidance can add design advice selected by the campaign. + +Neutral does not mean that the supplied skills contain no design advice. The +neutral profiles record `designAdvice: true`, and the selected SpacetimeDB skills +are intentional study inputs. Keep their text intact and retain it in the +evidence. The experiment compares delivered stack packages, not databases with +identical guidance. These skills are not grader source or scenario scripts. + +### Guidance profiles + +A condition's `guidanceProfile` selects one profile from +[`conditions/catalog.json`](../conditions/catalog.json). Only the SpacetimeDB +material differs between the neutral profiles; other stacks are unchanged. + +| Profile | SpacetimeDB skills | +|---|---| +| `neutral` (default) | TypeScript server, TypeScript client, CLI | +| `neutral-dev` | The same, plus the `spacetime dev` watch workflow | +| `neutral-managed-dev` | The same, plus the managed `/deps/spacetime-dev start\|status\|stop` helper | +| `neutral-no-sdk` | None | +| `neutral-dev-no-sdk` | Dev workflow only | +| `prescribed` | SDK skills plus prescribed stack documents with design advice | + +The dashboard's **SDK skills** and **Dev workflow** choices select among these. +Each profile has its own guidance identity. None changes grading or repair +policy, and none starts a watcher by itself. Label a skill ablation separately +from standard guidance, and start a separate run to compare guidance choices. + +The managed helper serializes starts, reports initial readiness, and keeps a log +in the agent home. It runs as the agent user, so normal container cleanup stops +it. It supports one assigned database and TypeScript binding targets inside +`/app`, and needs a controller and coding image built with this support. + +An abridged neutral PostgreSQL section is: + +```text +# PostgreSQL + +Use PostgreSQL for the application data. Choose the libraries, architecture, +and project structure. + +Use the supplied DATABASE_URL. Serve the application on the supplied port. +``` + +An abridged neutral SpacetimeDB section is: + +```text +# SpacetimeDB + +Use SpacetimeDB for the application data. Put the TypeScript module in the +required module directory. Choose the schema, libraries, architecture, and the +rest of the project structure. + +Use the supplied server URI, module name, CLI, SDK package, and web port. +``` + +## Application interfaces + +Feature contracts name stable controls or operations when deterministic use +requires them. They do not prescribe layout, data models, frameworks, or visual +design. The one data-shaped item a contract may name is an interoperability +surface that other systems write to directly, such as the stock tables; the +behavior expected around that surface stays in the specification. + +For example, the account contract names fields such as `signup-username` and +`signin-submit`. An HTTP stack also exposes the account operations through HTTP. +A reducer-based stack exposes the equivalent reducer operations. The product +behavior stays the same while the usable interface matches the selected stack. + +A readiness marker must distinguish an empty result from a failed read. A +contracted action must be the actual UI action, not a separate endpoint that can +pass while the product is broken. A hook can reveal that an operation exists, +but must not state the expected security or synchronization policy. + +Scenario files own exact test data and edge-case values. Those values do not +belong in the product request or interface contract. + +## Dependency progression + +Dependency mode composes the request from features that are ready now. + +- A new app receives the framing, current root features, applicable requested + production expectations, starting data, and their interfaces. +- An upgrade receives the newly ready feature work. By default, it also retains + the interface contracts disclosed in earlier requests for the same app. +- Upgrades and repairs retain the original catalog names and relationships. + They explicitly do not reset current stock, prices, or user data. +- Earlier feature requirements are not repeated as new work. Retained contracts + do not claim that the earlier implementation passed its checks. +- Blocked descendants are not included until their dependencies pass. + +New dependency plans default `mode.retainPriorContracts` to `true`. The compiler +records this choice in the frozen plan. To compare incremental interfaces alone, +set the option explicitly: + +```json +"mode": { + "id": "dependency", + "workSelection": "progressive", + "retainPriorContracts": false +} +``` + +Campaign execution uses the frozen plan's setting. Set the option in the +campaign definition before compilation; it cannot be changed during execution. + +Retained contracts come from previously issued requests, not from grading +results. Each authored contract appears once. Future interfaces and undisclosed +production expectations are not added. The coding agent can refactor the app +while preserving its declared interface. This context is included in token +accounting and does not enable repair feedback. + +An upgrade therefore has three parts: the existing app context, the current +feature changes, and the applicable interface contracts. Existing compiled plans +and stored results are not rewritten. A changed prompt treatment requires a new +plan and separate comparison data. + +For example, if accounts and catalog items are ready, the request can include +those two features. Customer profile stays out until its account dependency +passes. A failure in the catalog path does not add or remove work from the +account path. + +## Repair requests + +A repair starts only after Stack Bench completes grading and records a +conclusive application failure. A repair report can name an expected +production behavior not stated in the initial request; that disclosure +happens only through the repair policy, after a conclusive failure, and under +the same rule for every stack. The coding agent receives a plain bug report: + +```text +Fix the reported application bugs. + +Expected: The signed-in account remains active after a reload. +Actual: The page returned to the signed-out state after reload. + +Change only what is needed. Do not alter behavior that is already correct. +``` + +The repair request also supplies the affected product area, current feature text, +application interface, and original catalog baseline. Provider failures, harness +failures, and interrupted work do not become application bug reports. + +Behavior feedback uses the authored expectation for that behavior, a finding from the +grader's closed catalog rendered as one sentence (a control that did not +appear, a number below its required value, a request that was accepted +when it had to be refused), and the application's own console errors. +Reports include measured quantities and expected results when they explain the +failure, such as two orders where one was expected. They retain the affected +control, missing or duplicate entries, and HTTP error statuses. They do not copy +scenario scripts, unrelated fixture data, or instructions for a particular +algorithm or data structure. + +Reports also name the recent completed controls and lifecycle actions when these +explain where execution stopped. A missing control before a reload, restart, or +server request is not evidence that the later durability or access check failed. +Keep that limit explicit instead of presenting the full requirement as an observed failure. + +The current repair policy records this disclosure as +`scenarioValues: "failed-observations"`. This changes the condition identity +from the earlier `"withheld"` policy. New plans must use the current identity; +old frozen plans and reports keep their original metadata. The value permits +exact expected and actual values only for the failed observations above. + +When setup fails, the report uses that setup's observation once, even if it +prevented several checks. It does not repeat the expectation of a later check +that never ran. Distinct failures remain separate. Console errors are supporting +observations from the same product area, not proof of a cause; duplicate lines +are removed. Initial feature requests still exclude scenario inputs. + +## Authoring rules + +No-repair runs measure behavior supplied without repair feedback. Normal repair +runs report observed failures and expected behavior, including production +guarantees, but do not prescribe algorithms or implementation changes. Their +results measure remediation, not unprompted guarantees. Earlier repairs also +carry into later levels and source-seeded campaigns. A later depth's first +build can therefore contain earlier repair guidance. Label it a pre-repair +checkpoint at that depth, not a fresh measure of unsolicited guarantees. + +Put positive controls in scenario setup. If setup fails, preserve that actionable +application failure and its setup phase. The check earns no credit, but the +failure does not prove that the later guarantee is broken. Repair feedback must +not claim that an unperformed authorization or integrity assertion failed. + +- Put product asks in feature prompts. +- Put optional production requirements in specification prompts. +- Put stable controls and operations in contracts. +- Put connection facts and API material in stack guidance. +- Put exact values and probes in scenarios. +- Never solve a check by adding its private input or expected implementation to + the request. +- Keep equivalent stacks equally informed about the product. +- Give every replay, forgery, or direct call a named application action that + declares both the HTTP route and the reducer. A campaign does not compile + while a selected check cannot be measured on a selected stack. +- Invalidate qualification tied to changed prompt inputs. Qualification comes + from matching evidence; do not add status fields to recipes or references. + +After a prompt change, run `npm run check:composition`, `npm run check:prompts`, +and the exact scenario check for the affected recipe. Inspect the rendered +request for every affected stack and depth. Do not run unrelated qualification +or paid work. diff --git a/tools/stack-bench/docs/stack-bench.html b/tools/stack-bench/docs/stack-bench.html new file mode 100644 index 00000000000..1b12d4df209 --- /dev/null +++ b/tools/stack-bench/docs/stack-bench.html @@ -0,0 +1,1806 @@ + + + + + +Stack Bench + + + + +
+ +
← → or scroll
+ + +
+
+ + SpacetimeDB +
+

Stack Bench

+

Build the same product on different app stacks. Test what works and compare the evidence.

+
+ + +
+
what it includes
+

Stack Bench manages the full workflow.

+

One system runs the build, tests the finished product, supports optional repairs, and preserves comparison evidence.

+
+
+ +

AI build runner

+

Runs the selected model with the product brief, stack, and budget.

+
+
+ +

Modular product spec

+

Versions features, dependencies, prompts, checks, and scoring rules.

+
+
+ +

Isolated stack runtime

+

Starts the generated app and selected stack services in controlled containers.

+
+
+ +

Automated grader

+

Exercises real user flows, direct data changes, failures, and concurrency.

+
+
+ +

Repair controller

+

Returns observed failures when the plan enables repairs.

+
+
+ +

Evidence and comparison

+

Packages source, prompts, scores, cost, screenshots, video, and traces.

+
+
+
+ + +
+
one controlled run
+

Stack Bench controls every step.

+

Each run keeps the request and environment fixed, isolates the build and services, and saves the results and evidence.

+ + + Stack Benchcontrols the run + preflightchecks setup first + agentbuilds or fixes + app stackselected for the run + buildisolated workspace + servicesisolated and reset + apprunning on the selected stack + graderruns the product tests + resultspass · fail · could not test + evidenceprompt · source · cost · visuals + + + + failed testsreturn to the agent + +
+ + +
+
the run plan
+

Each run starts with a fixed plan.

+

Choose the model, stack, features, tests, and repair limit. Stack Bench records the plan and builds the request the agent receives.

+ + + + + + + + FIXED FOR THE RUN + AGENT REQUEST + + + Model + provider + exact model + + + + Stack + selected stack + tools + + + + Features + unlocked for this level + + + + Tests + checks + scoring + + + + Repairs + limit for each feature + + + + + + + + + + + BUILD, ADD, OR FIX + Work on the current features + + + Product brief + app + current features + Stack access + connection details + API reference + selected SDK material + Testing interface + hooks + lint command + + + On repair: one feature + its failed tests + +
+ + +
+
how testing works
+

Stack Bench tests the finished product.

+

The grader uses real browser sessions, direct data changes, service interruptions, and concurrent actions, then checks both visible and persisted state.

+ + + REAL USER FLOWSmultiple browser actors + ACCESS + OWNERSHIPprotected and cross-account actions + DURABILITY + RECOVERYreload · reconnect · restart + APP UNDER TEST + LIVE STATEdirect data changes reach open pages + CONCURRENCYoverlapping actions · exact totals + OPERATIONS + ACCOUNTINGshipping · pricing · revenue + +
+ + +
+
optional repair and retest
+

Failed tests return to the agent.

+ + + + + + + + + start level + dependencies passed + + + build + current features + + + test + unlocked features + + + all pass? + + + continue + next level opens + + + repairs left? + + + repair + one failed feature + + + that path stops + other paths continue + + + + + + yes + + + no + + yes + + + + none + + + continue with passed paths + +
+
levelL1L2
+
feature repairs0 of 31 of 3
+
features passed2 of 124 of 12
+
cost$0.00$3.20$6.40
+
+

This example uses a repair limit per feature and one feature per repair. The plan can disable repairs. A completed coding repair uses one repair, even if grading later fails. Provider errors and interrupted coding use none.

+
+ + +
+
dependency mode
+

Working features open the next work.

+

A feature can move forward when its product behavior works. Production checks still affect its score. A failed feature blocks only the paths that need it.

+ + DEPTH 1 + DEPTH 2 + DEPTH 3 + + + + + + + + + + + accountsOPENWORKINGPASS + catalogOPENWORKINGFAIL + cartOPENWORKINGPASS + warehouseOPENWORKINGPASS + + operator accessOPENWORKINGPASS + searchBLOCKED + checkoutOPENWORKINGFAIL + stock transfersOPENWORKINGPASS + + account recoveryOPEN + recommendationsBLOCKED + returnsBLOCKED + scheduled restocksOPEN + +
+ + + + + +
+
testing the benchmark
+

Stack Bench tests itself.

+

The same selected checks run against controlled apps. A correct app must pass, a planted defect must fail its target, and an empty app must score zero. Full qualification remains pending; these diagrams show the required outcomes.

+ +
+ + +
+
results
+

Scores show where each stack works.

+

Illustrative scores, not measured results. Each questline has its own score. Blocked and unfinished work stays in the denominator.

+ +
+ + +
+
comparison
+

Compare score, cost, duration, and repairs.

+

Illustrative comparison, not measured results. Each stack uses the same plan. The report separates first-build results, optional repairs, and cost.

+ +
+ + +
+
evidence
+

Every run preserves its evidence.

+

Open the exact result, source, visuals, and run economics behind the score.

+ +
+ + + + diff --git a/tools/stack-bench/docs/study-method.md b/tools/stack-bench/docs/study-method.md new file mode 100644 index 00000000000..5aff13cef53 --- /dev/null +++ b/tools/stack-bench/docs/study-method.md @@ -0,0 +1,286 @@ +# Study method + +This guide describes how to collect and report a defensible Stack Bench +comparison. The campaign manifest and its evidence record what a study actually +ran; this guide covers the decisions around them. + +## What a study measures + +Measure how much model usage each defined stack needs to implement the same +product, and how much of the selected behavior it completes. Compare delivered +stack packages, including the selected SpacetimeDB skills. This is not a +database-only experiment. + +The no-repair question is which expected production behaviors appear without +failure feedback. The repair question is how much completion and cost follow +actionable failure reports. Neither question sets a preferred stack's outcome. + +The baseline design uses dependency mode with progressive work selection and no +repairs or execution retries. Keep the graph, target depth, model, guidance, +budgets, repetition count, and concurrency in the frozen campaign manifest. +The model-free Docker demo is setup evidence, not agent performance evidence. + +Dependency depth comes from the feature graph. Progressive selection groups +available new work at each depth. Failed prerequisites block dependent features; +other branches can remain available. Previously completed behavior is checked +again as the app grows. Targeting a depth does not guarantee reaching every +selected node. Graph depths are not sequential L1/L2/L3 product releases; keep +those experiment names and denominators separate. + +## Study stages + +Set sample counts from the study's purpose; a pilot count is not a statistical +power calculation. One block means one fresh attempt on each selected stack under +the same protocol. Repetitions start with clean apps and independent agent sessions. + +| Stage | Collection | Purpose and exit condition | +| --- | --- | --- | +| Measurement pilot | A balanced block on the selected stacks | Confirm that the harness produces valid measurements. Diagnose failures before scaling. | +| Initial comparison | A fixed number of new balanced blocks under one frozen protocol | Show every result and its variation. Choose the count and concurrency before launch. | +| Focused confirmation | Separate frozen batch, sized from pilot variance and a decision threshold | Test a stated claim with uncertainty. Freeze count, budget, exclusions, and analysis before launch. | +| Wider scope | Deeper dependency work, another product, or another model | Test whether findings extend beyond the initial condition. Keep each condition separate. | + +Keep pilots and revised cohorts separate. Do not replace original inputs or +increase an attempt's allowance after seeing its result. The time and money +limits cover the complete attempt, not each depth. They are ceilings, not price +or duration forecasts. Early blocking and regression checks change actual usage. +Do not extrapolate from sequential L1 by multiplying by three. + +Before each comparison batch, freeze the exact graph, selected checks, guidance, +images, model, limits, and analysis. If any protocol input changes, start a +separate dataset and disclose the change. + +### Other designs + +A repair-enabled dependency study is a separate experiment, with its own frozen +allowance and all repair cost included. A claim that repairs improve results needs separate comparable repair and no-repair +cohorts. Comparing a repaired app with its own earlier checkpoint alone does not +isolate feedback from additional work and model usage. + +Sequential mode requires a whole level to pass before advancing and answers a +different question from the dependency study. A reference-seeded upgrade is a +further distinct experiment: record baseline provenance and excluded +construction cost, and do not call it a fresh build. Do not weaken gates or give +successful source to selected stacks to produce higher-level scores. +Reference-seeded and fresh-build outcomes need separate tables and claims. + +## Parallel execution and collection cost + +Each campaign explicitly sets its parallelism. Shared host capacity determines +when the campaign can start; it does not change the requested parallelism. +Resource or credential admission can delay dispatch; report that delay rather +than silently reducing the experiment. + +A balanced wave contains equal numbers of all stacks. Do not assign each stack a +different host or load level. Use the existing balanced-rotation order and retain +its seed. A seed controls ordering; it does not make model generation deterministic. + +For each capacity step, retain Docker allocation, host/architecture, actual +concurrency over time, peak memory, CPU pressure, OOM events, disk availability, +provider throttling, phase wall time, and cleanup outcome. Separate configured +limits from measured usage. If a measurement is unavailable, say so. Stop +increasing load on OOM, admission or ownership failure, incomplete evidence, or +saturation that prevents a fair comparison. + +Load-test timing and steady-load comparison timing are not interchangeable. +Report infrastructure contention separately from application defects. If +capacity changes between study batches, retain batch identity and report results +by batch. + +Before collection, choose a common per-attempt cap from observed usage plus a +stated headroom allowance. The maximum campaign authorization is attempts +multiplied by that cap, plus any explicitly authorized retries. Report the cap +and actual spend. An attempt that reaches the cap stops, is recorded as "Cost cap +reached", and is excluded from comparison; it is not permission to increase the +cap mid-study. Frequent cap stops mean the headroom was too small. + +## Freeze the method before the main batch + +Use the compiled campaign manifest and its retained artifacts for +machine-recorded fields. Keep only the research question and analysis decisions +not represented there in a small method note beside it. Link the manifest; do not +copy its fields into a second configuration. Record: + +- Research question, primary outcomes, sample count, stopping rule, and budget. +- Repository commit, image digests, platform, compiled plan and definition hashes. +- Model and adapter versions, provider route, context policy, and pricing snapshot. +- Exact agent-visible product request, contracts, stack material, and skills. + Retain the text as well as its hash. A hash cannot reconstruct missing content. +- Features and checks, requested/expected/observed specification roles, weights, + progression rules, repair disclosure, repair allowance, and retry policy. +- Host, resource limits, concurrency, ordering seed, dates, cache treatment, + and any other work sharing the host. +- Failure classes, exclusion rule, replacement rule, and report calculations. + +Keep the normal product request and expected production checks separate. Do not +expose scoring material to agents. Disclose the selected guidance profile and +skills. Give each stack the same opportunity to use its declared tools and +supported setup. Measure guidance as a separate ablation. + +Never add runs until a preferred stack wins. Do not choose “representative” apps +after seeing scores. Preserve failures and costs from every execution. A harness +or provider failure is excluded from app comparison under the frozen rule, but +remains in the operational and spending tables. Report attempted, eligible, +excluded, stopped, and reached counts for every stack. Missing cost stays unknown. + +## Measures and analysis + +Keep cost and completion as two primary outcomes. Do not hide their tradeoff in +one composite score. + +Evaluate eligibility separately for each measure. A valid completed outcome can +retain its completion metric when exact cost is unavailable. Exact cost, an upper +bound, and unknown cost are distinct; never replace an unknown amount with zero. +This does not waive run validation: if missing receipts also prevent +verification of the declared spending cap, the attempt has an unresolved protocol +issue and is not automatically eligible for comparison. + +Use the report's selected, passed, failed, blocked, and unmeasured counts. Its +unmeasured count does not distinguish all unattempted, deferred, and inconclusive +checks. Use linked grade evidence for those distinctions when available, and +state when a breakdown cannot be recovered. Do not infer attempted counts from +selected minus blocked, or sum overlapping property groups. Timing failures need +evidence-based attribution; timing alone is not a harness failure class. + +| Measure | Required interpretation | +| --- | --- | +| Check completion | Passed / selected checks, with both counts. Weighted points remain separate. | +| Feature completion | Fully passed dependency nodes / selected nodes. A node with an unfinished guarantee is not fully complete. | +| Build checkpoints | Show each measured progressive build. For repair cohorts, separate pre-repair and repaired checkpoints and include all repair cost. | +| Feature and depth reach | Show nodes started, passed, failed, and blocked at each graph depth out of all assigned attempts. A reached depth need not mean all its nodes passed. | +| Full target delivery | Fraction of assigned attempts that passed the complete target; show exclusions separately. | +| API-equivalent cost | Use receipt status and frozen rates; distinguish exact, upper-bound, and unknown. It is not a subscription invoice. | +| Token usage | Separate ordinary input, output, cache reads, and cache writes; retain receipt-level cache-write durations. | +| Time | Show end-to-end wall time, planned pause time, and execution duration separately. Campaign timeout excludes verified planned depth pauses; provider waits still consume the allowance. Retain the raw timestamps. | +| Reliability | Harness/provider failures, evidence failures, OOMs, cleanup failures, and cap stops. | +| Regression | Previously passed checks lost after new work or repair, with source/checkpoint identity. | + +For dependency results, show node/depth completion and a separate whole-target +view using the frozen selected checks. Do not sum repeated checks across depths +as independent accomplishments. Work not reached gets no completion credit; label +it blocked, not measured app failure. Do not report completion only among apps +that reached the target depth. Validate the denominator against the frozen +selection. + +Separate UI, feature, and production-quality checks. Report the latest depth's +results alongside cumulative results; a high cumulative percentage must not +obscure an authorization or concurrency failure. + +A pre-repair checkpoint at a later depth can inherit guidance from earlier +repairs. It is not an unsolicited-guarantee baseline. Preserve feedback history +when extending or seeding from an existing attempt. A +[planned depth pause](../appliance/README.md#planned-depth-pause) and an +uninterrupted attempt are separate conditions until evidence supports a narrower +equivalence claim. A source-seeded extension does not restore the original +database or become a fresh run from zero. + +Audit failures against the saved source, exact issued request, and check +evidence. Record the measurement stage (setup, assertion, blocked, or +inconclusive), the application cause, where the requirement was disclosed +(current request, earlier request, or not disclosed), and any unsupported harness +assumption. These are separate facts, not mutually exclusive blame labels. A +missing interface can be an app regression and expose a prompt limitation. +Repeating its contract is a testable treatment, not proof that the omission +caused the failure. + +Total tokens count repeated processing, including cached input. They do not +measure unique prompt size or generated code. Show cost/completion scatter plots +and measured checkpoint curves. Do not interpolate unmeasured success between +checkpoints. Cost per passed check can be an appendix diagnostic, but is a poor +headline: checks differ in difficulty and a zero-score app has no finite ratio. + +For each comparison dataset, show every attempt, median, IQR, and mean cost. +State the small sample size beside each comparison. Use matched batch differences +to describe stack contrasts, while recognizing that model outputs are independent +draws, not identical seeded tasks. A check is not an independent sample; levels, +repairs, and regrades from one app are not new app builds. + +For confirmation, first choose the smallest decision-relevant cost difference +and completion difference. Use observed between-build variation to plan sample +size and precision. Analyze whole attempts or blocks, preserving their +dependence; do not bootstrap individual check rows. Predeclare primary contrasts +and treat other cuts as exploratory. Use an appropriate binomial interval for +full-target success rates, especially with small samples or zero failures. Three +runs per stack support a useful initial comparison, not a general claim of +superiority. + +Blocking is a standard way to account for nuisance factors such as batch or +host conditions ([NIST](https://www.itl.nist.gov/div898/handbook/pri/section3/pri332.htm)). +Small-sample success-rate intervals need care; normal approximations can be +inaccurate ([NIST](https://itl.nist.gov/div898/handbook/prc/section2/prc241.htm)). +Correlated checks from one app are not independent experimental replicates +([NIST](https://www.nist.gov/publications/expanding-ai-evaluation-toolbox-statistical-models)), +and measurement validity needs documentation and independent review +([NIST AI RMF](https://airc.nist.gov/airmf-resources/playbook/measure/)). + +## Failure review and defensibility + +Review failures with the +[grading coverage procedure](grading-coverage.md#review-a-failed-check). Where +feasible, use the same reviewer rubric without stack labels, then disclose the +source needed to verify the diagnosis. Do not repair generated apps manually in +the primary dataset. New prompt or interface requirements require a new cohort +when old source is not compatible. + +Before a public comparative claim, obtain independent external review of the +frozen protocol, exclusions, scoring, and analysis. Record unresolved objections +and disclose reviewer affiliations. Parallel agent review is an internal check; +it is not independent external review or replication. Do not claim replication +until another team reproduces the method and reports its results. + +Verified publication requires [qualification](../reference-apps/README.md) of +the exact reported selection. Exploratory collection can proceed before that, +labelled provisional. + +## Research pack and archive + +Aim for a 4–6 page decision report, a one-page run guide, and linked evidence. +Page count is a reading target, not a limit on the data retained. + +The decision report should contain: + +1. Scope and method, including guidance, skills, and qualification status. +2. Cost versus completion for every attempt, colored by stack. +3. Feature and selected-depth reach, completion, blocked work, and build checkpoints. +4. Cost/token breakdown and observed variation, with sample counts. +5. A short failure table with confirmed causes and linked evidence. +6. Limits, exclusions, and the next experiment that would change the decision. + +Ship the existing HTML report and validated JSON, frozen plan, method file, +attempt-level CSV, and manifest-listed artifacts with their relative paths +intact. Use one row per attempt-level checkpoint where needed; do not count those +rows as independent attempts in analysis. + +The `export-manifest.json` is an index, not a portable archive. The +`campaign export --out ` command copies its +listed public artifacts and adds attempt and execution CSVs. It omits source +trees, raw transcripts, media, and external evidence; links to omitted files +cannot work offline. Check included links and hashes. Add a separately reviewed +source and prompt archive if claiming full reconstruction. The complete campaign +copy described in the [appliance guide](../appliance/README.md#results-and-cleanup) +is an internal backup; it is not automatically safe for public distribution. + +Keep full original campaign evidence privately: all receipts, prompts, source +checkpoints, grades, logs, media, admissions, resource records, exclusions, and +qualification evidence. For external sharing, review free text and generated +source for credentials and private data. Exclude private authority, provider +credentials, and environment secret files. Record omissions. Retain immutable +originals and hash the shared pack. + +The run guide must distinguish the free reference demo from a paid model +campaign. Include the tested Docker command, platform requirements, credentials +needed for paid work, budget controls, output location, and how to inspect and +copy results. Rebuilding the same plan must be possible; identical stochastic +outputs are not promised. + +## Agent and model conditions + +The registry includes Claude Code, Codex, and OpenRouter adapters. Registration +is not qualification of every model, credential route, or execution mode. Before +collection with a new adapter condition, verify its declared launch, repair, +continuation, usage, budget, and failure paths with matching evidence. + +Keep provider, model, agent runtime and version, tools, reasoning settings, and +context policy distinct in the frozen condition. Compare stacks within each +condition; do not pool different agent conditions into an unexplained stack +average. Evidence from one model does not establish results for another. diff --git a/tools/stack-bench/docs/system-design.md b/tools/stack-bench/docs/system-design.md new file mode 100644 index 00000000000..67ce42c0494 --- /dev/null +++ b/tools/stack-bench/docs/system-design.md @@ -0,0 +1,177 @@ +# Stack Bench system design + +Stack Bench turns one versioned test plan into traceable comparison evidence. The +system must make every decision, action, result, and cost traceable without +using chat history or operator memory. + +## One owner for each fact + +| Layer | Owns | Durable output | +|---|---|---| +| Definitions | Product work, prompt modules, checks, stacks, models, and budgets | Versioned source files | +| Compiler | The exact work matrix and all bound identities | `plan.json` | +| Job store and worker | Immutable submission, host placement, credential references, and exclusive execution claim | Job and claim records | +| Admission | Whether the exact plan can run on this appliance | Admission artifact | +| Scheduler | Attempt order, concurrency, continuations, and terminal state | `state.json` | +| Run engine | Build, grade, repair, resource ownership, and cleanup | Attempt directory | +| Grader | Typed check results and evidence | Grade artifacts | +| Progression engine | Open, passed, failed, and blocked features | `progression-state.json` | +| Report | A reproducible view of retained evidence | `report.json` and `report.html` | + +No layer can silently replace a decision from a layer above it. A view can +summarize durable data, but it cannot create new run state. + +## Terms + +One word per thing. The CLI, dashboard, and artifacts use these. + +- **campaign**: one comparison job. A plan file fixes the product, the stacks, + the model, the checks, the budgets, and the repetitions; a result directory + holds everything it produced. +- **attempt**: one stack building the product once inside a campaign. A + campaign with three stacks and one repetition has three attempts. +- **execution**: one process run of an attempt. A retried attempt has two. +- **session**: one conversation with the coding agent. A build session writes + the app; a repair session reacts to a failure report. +- **stack**: the technology under test, such as SpacetimeDB, PostgreSQL, or + MongoDB. Flags still spell it `--backend`; the word is stack. +- **level**: one rung of a sequential campaign (L1, L2). **depth**: how far down + the feature graph a dependency campaign has reached. They share a field but + never a meaning. +- **feature**: one node of the dependency graph, the unit the agent builds and + the grader scores. A feature opens when its parents pass. +- **questline**: a named path of features through the graph, such as identity + or fulfilment. One questline can stop while the others continue. +- **check**: one scored criterion with a stable id such as `601b`. A **gate** + check must pass before the feature's descendants open; a **guarantee** check + costs points but never blocks. +- **disclosure**: whether a specification is requested (in the prompt), + expected (not in the prompt, scored), or observed (not in the prompt, not + scored). +- **first build**: the score before any repair. **repair**: one paid session + that reacts to a failure report, plus the regrade after it. A repair candidate + is accepted only under the mode's regression rules; rejected source remains + available as evidence. +- **passed** and **failed** are measured outcomes; failed is the application's + fault. **inconclusive** means the harness could not measure the check: no + credit, no blame, and the reason is recorded. **blocked** means a prerequisite + failed, so the check was not attempted. +- **harness failure** and **provider failure** mean the benchmark or the model + provider broke; the attempt is **excluded** from comparison data, as is a + **contaminated** attempt whose agent read grading material. +- **qualification**: matching reference, null-control, and defect-control + evidence for an exact check selection. Without it, results are provisional. +- **needs attention**: a campaign that stopped and needs a person. +- **preflight**: the verifications before an attempt, each a **probe** such as + `registry.cache`. A **smoke** preflight starts a real coding container without + a model. **admission** is the record that preflight and policy allowed the + campaign to start. +- **coding container**: the container the agent works in, created from the + **build image**. It sees the app, its stack material, and nothing else. +- **clean source**: the accepted application source with nothing the agent's + process left behind. Every grade starts the app from clean source. +- **credential broker**: the local proxy that holds the provider key so the + coding container never sees it. Its **cost receipt** is the proof of what a + session spent. +- **lease**: the record of which containers, ports, database, and locks an + attempt owns, so cleanup and recovery act only on those. + +## Data flow + +```text +versioned definitions + | + v +compiled plan -> admission -> scheduler -> run engine -> grader + | | | + v v v + state.json run.json evidence + \ | / + \ v / + -> inspection -> report +``` + +The coding agent receives only the app request, current work, selected stack +material, and repair evidence allowed by the plan. It does not receive the +benchmark, grader, future work, expected implementation, or comparison data. + +## Operator loop + +An operator, human or agent, uses one loop: + +1. **Define.** Select one versioned campaign file. Do not rebuild the plan from + command flags. +2. **Validate.** Compile it and inspect the exact attempts, stacks, model, + prompt policy, checks, points, budgets, images, and parallelism. +3. **Admit.** Prove credentials, images, ports, resource capacity, and stack + access before model work starts. +4. **Run.** Start the exact stored plan or use an eligible continuation. A paid + action is always explicit. Resume is not general process or database recovery. +5. **Observe.** Read durable campaign state first. Open logs only to diagnose a + live phase or failure. +6. **Decide.** Continue only through a legal state transition. Never hide an + invalid attempt or retry it outside the frozen policy. +7. **Report.** Generate the result from retained run evidence. Publish it as + verified comparison data only when grading qualification is complete. +8. **Clean.** Remove temporary owned resources. Keep the campaign package. + +The CLI and dashboard use the same compiler, scheduler, state reader, and run +commands. The dashboard is a view and input surface. It is not another control +plane. + +## Agent interface + +The operator interface must answer these questions without source inspection: + +- What exact plan am I controlling? +- Can it start without spending model usage? +- What is running now, and in which phase? +- What has it cost and how long has it run? +- Which results are valid application results? +- Which failures belong to Stack Bench, the provider, the stack tools, the + host, or the operator? +- What evidence proves each answer? +- Which actions are legal now? + +Machine-facing commands return stable JSON. A compact response gives the plan +identity, campaign state, active work, cost, failures, and legal next actions. +Detailed responses add attempts and artifact paths. Logs and raw artifacts stay +available, but an operator does not need to parse them for normal control. + +Errors must name the failed subsystem, failure owner, retryability, retained +evidence, and next safe action. `inconclusive` is an intermediate measurement +state, not an accepted final explanation. + +## Resource rules + +- Compile and inspect before any model call. +- Run focused source checks after a change. Run the integrated source gate once + for the final source identity. +- Reuse qualification evidence only when its bound inputs match, or a validated + evidence slice proves unchanged scope and a reviewed executable equivalence + decision covers any runtime hash change. Preserve the original artifacts. +- Do not repeat reference, mutation, or null work for unchanged scope. +- Stop new paid attempts after a harness, provider, host, or operator failure. +- Retry only when the frozen attempt policy permits it. Extra repair grants + require a separate operator action. +- Run independent attempts in parallel only within the plan and admitted host + capacity. +- Preserve a failed package before a source or plan change. + +## Accumulated knowledge + +Operational knowledge belongs in typed artifacts, not chat transcripts or a +growing journal. Each completed action records its inputs, identity, outcome, +cost, duration, evidence paths, and owner. A later operator can reconstruct the +campaign from the retained package. Continuation still requires the engine's +eligibility checks; evidence alone cannot restore a lost live database or session. + +Local notes can explain an active investigation. They cannot authorize a run, +change a score, or replace a missing artifact. + +## Design test + +Every major structure must have one purpose, one owner, and one current +consumer. If its reason cannot be stated in one sentence, simplify or remove it. +Complexity is allowed only when it protects result validity, isolation, +security, recovery, or a current operator need. diff --git a/tools/stack-bench/docs/technical-guide.html b/tools/stack-bench/docs/technical-guide.html new file mode 100644 index 00000000000..8051ab36436 --- /dev/null +++ b/tools/stack-bench/docs/technical-guide.html @@ -0,0 +1,222 @@ + + + + + + Stack Bench — technical guide + + + +
+

Stack Bench / Technical guide

+

From product request to measured result

+

How Stack Bench compares coding agents across technology stacks, controls the experiment, and retains the evidence behind each result.

+ Documentation index +
+ +
+
+

One run path across stacks

+

Each attempt builds the same selected product work on one stack. The controller uses shared campaign, grading, and repair logic. Stack adapters supply the database and runtime operations.

+
    +
  1. DefineSelect work, guidance, checks, model, and budgets.
    2. +
  2. CompileFreeze the work matrix and its input identities.
    3. +
  3. PreflightCheck the runner and activate isolated resources.
    4. +
  4. BuildGive the agent the current product request.
    5. +
  5. Grade / repairMeasure behavior and apply the chosen repair policy.
    6. +
  6. RecordKeep source, outcomes, cost, time, and cleanup evidence.
    7. +
+

Application failures, provider failures, and harness failures remain separate. The report reads saved evidence; it does not infer success from an agent's final message.

+
+ +
+

What the experiment fixes

+
+

Product and checks

The track defines the product. Feature packs supply requested work and required interfaces. Specification packs supply expected production behavior. A recipe selects the modules and checks.

+

Delivered guidance

A condition selects stack material, SDK skills, disclosed specifications, and repair feedback. These are recorded inputs to the comparison.

+

Execution policy

The campaign fixes models, stacks, repetitions, parallelism, time and cost limits, repair budgets, images, and pricing. Compilation binds their identities.

+
+

Compare compatible stack–agent–condition groups. SpacetimeDB's TypeScript server, client, and CLI skills are intentional parts of its delivered package. This measures the complete package used by the agent, not the database in isolation.

+

Research method and comparison rules · Definition ownership

+
+ +
+

What the coding agent receives

+
+

The product request

  • The brief and current feature work.
  • The original catalog names and relationships.
  • Required application controls or action interfaces.
  • Selected stack access details, SDK references, and skills.

Later requests retain disclosed contracts and catalog facts without resetting live application data.

+

A repair request

When enabled, feedback gives the affected behavior, expected result, and observed failure. It can report a failed production expectation even when the initial request did not state it.

Feedback does not prescribe an implementation. Grader source, test scripts, scores, and comparison results stay with the controller.

+
+

No repairs measures behavior before failure feedback. With repairs measures completion and cost after that feedback. A later depth's first build can inherit earlier repairs, so it is not a fresh no-repair sample.

+

Claude Code, Codex, and OpenRouter use registered adapters and shared execution controls. Adapter registration alone is not live model qualification.

+

Prompt and repair policy · Disclosure rules · Provider credentials

+
+ +
+

How work advances

+

Sequential mode completes each selected level before the next. Dependency mode opens a feature when its required parents pass. A blocked branch does not prevent unrelated branches from advancing. Earlier work is checked again for regressions.

+
+

Work selection

  • feature: one ready feature.
  • progressive: all currently ready features.
  • all-at-once: the full selected graph.

The graph owns prerequisites. Work selection does not change the scored target.

+

Repair selection

Repair one failed feature or a batch of current failures. Limits can apply to the attempt, feature, or depth. When combined, the tightest remaining limit applies.

The unchanged-failure limit is separate. The initial failure counts as one observation; pure regrading does not spend a repair.

+
+
  1. Save the source checkpoint and grade the selected work.
  2. If repairable failures and budget remain, send the allowed failure report.
  3. Grade the changed source and check prior behavior for regressions.
  4. Accept the candidate under the mode's rules, or restore accepted source. Keep the rejected candidate as evidence.
+

A dependency gate determines whether child work can open. Full feature completion is stricter: all its selected checks, including production guarantees, must pass.

+

Explore the dependency graph · Run and repair options

+
+ +
+

How execution stays separate

+
+

Attempt isolation

Each attempt owns its containers, database, ports, workspace, and resource lease. Coding agents do not receive the grader, result store, provider secret files, or Docker socket.

+

Job dispatch

A job runs one campaign on one host. Local workers claim queued jobs. Campaign parallelism controls attempts; worker concurrency controls campaigns. Resource leases are allocated on dispatch.

+

Credential selection

Named profiles select credentials per attempt or adapter. The trusted broker holds the secret and records usage. Selection is explicit; the system does not rotate accounts automatically.

+
+

There is no manually sized runner-slot pool. Host resources and provider limits still constrain execution. The local worker does not provide a distributed attempt scheduler or account-wide quota service.

+
Optional SpacetimeDB development workflow

neutral-dev supplies guidance for the agent to run spacetime dev. neutral-managed-dev supplies the /deps/spacetime-dev start|status|stop helper. The agent creates project configuration and starts the watcher. Both keep the TypeScript server, client, and CLI skills.

These are different recorded guidance treatments. Neither changes grading or repair policy.

+

Pause, stop, and continuation

+
+ + + + +
MethodWhat it preservesBoundary
Planned depth pauseThe live app, database, execution, and cumulative budgets.Declare the full target and pause depth before launch. Keep the controller running. Database timers still advance.
Stop and reconcileSaved evidence and private cleanup authority.Interrupts work. It does not restore the lost agent session or database runtime.
Source-seeded extensionVerified source and parent lineage.A separate campaign with fresh runtime and budgets. Earlier work is regraded. Include parent cost when reporting the full path.
+

A planned pause excludes its verified hold time from the working allowance. It does not establish equivalence to uninterrupted execution. A controller shutdown cannot be recovered as the same live pause.

+

Workers and jobs · Credential profiles · Depth pause commands · Cleanup and recovery

+
+ +
+

How to read a result

+

Each selected check records an outcome and its supporting observations. Only conclusive application failures can enter a repair report.

+
+ + + + + +
OutcomeMeaningTreatment
PassedThe measured assertion was met.Earns its selected points and completion credit.
FailedThe application did not meet the assertion.No credit; eligible for feedback under the repair policy.
InconclusiveThe evidence cannot establish pass or fail.No credit or application blame; retain the reason.
Harness failureThe test system could not perform a valid measurement.No credit or application repair request; diagnose the harness.
+
+

Check completion

Accepted passed checks divided by all selected positive-point checks.

+

Feature completion

Fully passed dependency nodes divided by all selected nodes.

+

Weighted score

Accepted passed points divided by all selected points.

+
+

Blocked and unmeasured work stays in the denominator. Feature, production, and interface categories describe checks; they are separate from the Features/Checks counting unit.

+

Cost includes build and repair work. Keep exact, upper-bound, and unknown receipts distinct. Subscription usage uses frozen API-equivalent rates, not a subscription invoice. Report wall time, planned pause time, and execution duration separately.

+

The dashboard switches between completion, cost, and distribution, with stack and repetition toggles. Features are selected by default. Lines connect saved observations, not continuous measurements. Open an attempt for checks, source, screenshots, logs, and the agent transcript.

+
Evidence retained with the run
  • plan.json: the compiled experiment and input identities.
  • state.json: attempt scheduling and execution history.
  • run.json: build, grade, repair, cost, and outcome records.
  • progression-state.json: feature state and event history.
  • Grade artifacts: actions, expected and observed values, and media.
  • recovery.json: cleanup outcome and retained resource evidence.

Keep the complete campaign archive. The public research export is a smaller package and can omit source, transcripts, and media.

+

Dashboard guide · Categories and counting units · Analysis and reporting

+
+ +
+

What makes a comparison defensible

+

The selected checks need matching live evidence. A successful build, a static mutation inventory, or an older report does not qualify a changed definition.

+
+

Correct reference

The known-good application must pass the selected scope on each stack. Use the calibration's repetition count. Extra repeats can check stability.

+

Known defects

Each mutation must change observable behavior and fail its declared assertions. Setup errors and unrelated failures do not count as a clean catch.

+

Empty application

An empty app must fail the selected scored checks conclusively. Zero points caused by a broken harness are not a valid control.

+
+

Scope of the claim. These are finite behavioral tests. Runtime restart does not establish power-loss or database crash recovery. A bounded contention burst does not establish sustained capacity. Review the exact probe and delivered request before attributing a result to an unrequested production guarantee.

+

Qualification is determined by the frozen definition and matching artifacts, not a status table in this guide. Pending qualification permits provisional runs but blocks verified comparison claims. Report exclusions and missing measurements separately.

+

Qualification commands · Probe coverage and limits · Research protocol · Release verification

+
+ +
+

Where to make a change

+

Use the existing owner for each concern. The source is TypeScript; builds emit ESM JavaScript into dist/.

+
+
  • src/campaigns/Plan compilation, scheduling, jobs, budgets, and reporting.
  • src/progression/Feature dependencies, work selection, repair state, and event history.
  • src/composition/Pack and recipe compilation, prompt composition, and bound identities.
  • tracks/Product work, interfaces, scenarios, and feature graphs.
+ +
+

Start with a focused test for the changed boundary. Run live Docker or qualification checks when that boundary needs them; do not repeat unchanged evidence for reassurance.

+

Development and test commands · Add or change a feature and its checks · Ownership and system design

+
+
+ + + diff --git a/tools/stack-bench/grader/README.md b/tools/stack-bench/grader/README.md new file mode 100644 index 00000000000..05702a455a6 --- /dev/null +++ b/tools/stack-bench/grader/README.md @@ -0,0 +1,251 @@ +# Stack Bench grader + +The grader runs versioned scenarios against a generated app. It collects +browser, transport, lifecycle, and database evidence for each check. + +Each scenario actor receives a separate browser context. A live-update check +passes only when the page that was already open changes. The grader does not +reload a failed assertion and try again. + +## Outcomes and scoring + +Every check produces one outcome: + +- `passed`; +- `failed`; +- `inconclusive` when required evidence is unavailable; +- `harness_failure` when Stack Bench could not perform the measurement. + +Only a passed check adds its declared points. Other outcomes add zero and never +change the declared denominator. Console errors remain diagnostics and do not +change unrelated scores. + +Authorization and replay checks pass only when the requested call ran and +produced verifiable evidence. Visible UI behavior cannot replace missing server +evidence. + +### Outcome rules + +- If an app prerequisite fails, the dependent checks are reported as **blocked**. + They receive no credit, but this is not evidence that their target assertions + failed. The prerequisite observation remains available for repair. +- Page navigation timeouts and other navigation transport failures are + unmeasured: external resources can delay page readiness. Connection refusal is + a measured reachability failure. +- Invalid selectors, grader scripts, and browser protocol errors are harness + failures. Observation helpers must not convert these errors into missing controls. +- Harness and provider failures remain unmeasured and cannot become app failures. +- If the app itself stops answering mid-grading, checks measured before that keep + their outcomes, the rest of the current work fails, and earlier features' + unreached checks are recorded as not run. A readiness probe that times out is + unmeasured, not an app failure. +- An application refusal is the stack's defined error result: HTTP 400, 401, + 403, 404, 409 or 422, a SpacetimeDB reducer failure (530), or a thrown + `ConvexError`. SpacetimeDB's HTTP reply does not separate a deliberate + reducer error from a panic, so a panicking reducer also counts as a refusal. + An HTTP 500 does not. +- A request-tampering sign-in or sign-up step modifies the credential request the + app actually sends. Positional arguments, such as a SpacetimeDB function call, + take an added field only where the module schema names that parameter; a + field with no parameter is recorded as absent and the request goes as sent. If + the request cannot be captured or its parameters cannot be read, the step is + unmeasured. An ordinary sign-in never stands in for the probe. +- Concurrent actions drain every branch before returning; measurement failures + take priority over app failures. Check verdicts cannot contradict failed or + unmeasured action evidence. +- Concurrent named calls retain every request outcome when cancelled, including + responses received before cancellation. A lost response or request timeout is + an unknown result, not proof that the app rejected or failed to commit the + operation. Missing or unknown outcomes make the response assertion + inconclusive, even if another request returned an app error. HTTP success + alone does not prove the stored business effects. +- Bundle and dependency grading both inspect partial observations and cleanup + evidence before accepting an app abort. +- Account checks require a real application session and an independently + observed application-database write. Credential storage and log audits are + source-specific diagnostics; they do not certify password storage or logging + in arbitrary generated apps. + +Campaign grading retries only the affected isolated suite, once, when its +evidence is explicitly retryable and inconclusive. It preserves completed suites +and both executions in the grade bundle and raw artifacts. Product failures, +mixed failure/inconclusive suites, cleanup failures and harness failures do not +retry. Qualification runs do not enable this policy. If grading remains +incomplete, it stops the attempt without treating the timeout as a failed feature +or selecting later work. + +### Database reset between scenarios + +Between isolated scenarios, every stack runs its normal application startup +after the database reset. This includes SpacetimeDB apps that perform +initialization outside module `init`. The harness does not guess migration names. +The shared agent request states that startup must initialize the supplied data +and accounts in an empty database, and preserve current quantities, prices, and +user data when a database already exists. + +PostgreSQL resets recreate only the leased database, including its schema and +migration history. Build preparation, scenario isolation, and repair rollback +use the same reset. Durability probes restart services without resetting data. + +Convex runs a pinned, self-hosted backend in each attempt's private network; no +cloud account is required. The grader uses the declared native operations and +independent database reads. Login probes preserve native WebSocket calls and +require matching replies. Authenticated replays use the observed native bearer +identity or session argument; missing or ambiguous credential transport remains +unmeasured. A Convex backend crash also stops its application functions, so the +grader measures that boundary once. + +## Fault probes + +The current campaign checks do not yet include controlled checkout write rejection +or forced scheduled-worker overlap. The rules below govern adding those probes; +they are not a claim of current coverage. + +The purchasing and cart contracts do not fix order storage or ID generation. +An ID-collision probe verified on one saved app therefore cannot be applied to +all generated apps. Do not require sequential IDs just to make that probe work. +A general write-rejection probe needs an external fault method that supports the +app's actual storage, with proof that the intended write was rejected. + +A database stall tests recovery from a stall. It does not by itself prove a late +write failed or that two workers selected the same job. Keep those claims distinct. + +Scored fault probes leave the generated source and dependencies unchanged. Inject +faults through the isolated runtime or database, then check persisted application +state. Record the fault target, activation, release, and observed result. A setup +timeout or an unobserved fault is not an application failure or a pass. + +Use instrumented copies only as grader controls, with their changes recorded. +Before promoting a probe, require normal-operation success, a known defect caught +at the intended check, a correct implementation passing under the same fault, +and successful recovery after release. An unsupported stack is not a passing +control; do not include the probe in a shared comparison until each stack has a +verified method for testing the same behavior. + +A duplicate-checkout test does not establish rollback after a failed order write. +A restart test does not establish safety when scheduled workers overlap. Keep +those cases separate in check definitions and reported coverage. + +Confirm the delay on at least one worker. Do not require a second worker to reach +the same write: correct job claiming can prevent it. Verify the final effect after +release, and check that a later poll does not repeat it. + +## Failure reports + +An action never fails with a sentence. It fails with a finding from the closed +catalog in `src/actions/action-findings.ts`: a kind and its fields, where a +field is a contract control name, an action id, an actor label, a number, a +count, or an HTTP status. Every reader renders the finding from its one +template. Raw diagnostics travel in a `detail` field that is never rendered. + +## Scenario ownership + +Scenario JSON contains actors, setup steps, actions, and scored checks. The +action contracts are compiled and registered in `src/actions/`. Scenario prose +is not executable behavior. + +Actions run through capability-scoped executors. Browser, transport, +concurrency, lifecycle, and database actions use the same typed result contract. +Each stack adapter declares the capabilities it provides and whether named +application actions travel as HTTP routes or reducer calls. The campaign +compiler resolves every selected check against every selected stack and +refuses a campaign that a stack could not measure. + +When authoring assertions: + +- scope repeated elements to their owning row, room, message, or user; +- assert visible values, not the presence of an empty container; +- require the original open page for live-update behavior; +- use separate actors for identity boundaries; +- say in the criterion's `note` why it carries its points when they differ + from the feature's other criteria. + +Example: + +```json +{ + "do": "expect", + "actor": "bob", + "testid": "unread-badge", + "in": { "testid": "room-item", "contains": "{room:unread-main}" }, + "within": 5000 +} +``` + +## Run the grader + +Use `dist/commands/run-suite.js` for normal grading. It owns database reset, +provenance checks, contract linting, scenario execution, logs, and bundle +creation. + +Direct `dist/grader/grade.js` execution is for focused scenario authoring only: + +```bash +node dist/grader/grade.js --url http://localhost:6173 \ + --spec tracks/ecommerce/scenarios/01-account-create.json \ + --label spacetime-l1 --out report.json +``` + +If the grader exits before writing JSON, inspect the retained +`grader-.stdout.log` and `grader-.stderr.log` files. + +## Validate checks + +Live reference runs test that intended behavior passes. Null controls test that +an empty app fails each selected scored check conclusively. Live mutations test +that each selected check detects its assigned defect. These are finite controls +for an exact definition, not proof of general production readiness. + +This command checks mutation definitions and source anchors only. It does not +start an app or show that the grader detects a defect: + +```bash +npm run check:mutations -- --app --mutations +``` + +For live controls, use the scoped commands in the +[reference guide](../reference-apps/README.md#live-qualification). Declare the +recipe and depth explicitly. A bare default command can measure a different scope. +During development, run only affected mutations. The full selected mutation set +is a release qualification gate and requires separate authorization. + +The mutation runner requires: + +- a fully passing clean baseline; +- one exact source anchor for every edit; +- a conclusive failure at the intended check; +- no unrelated failures; +- successful source restoration and app reset. + +Setup, infrastructure, and inconclusive failures do not count as defect +detection. A surviving mutation can be equivalent, so confirm that its source +edit changes observable behavior before changing the check. + +For concurrent checks, a defect control must preserve ordinary serial behavior. +For restart checks, ordinary execution must work before the restart. A disabled +operation does not isolate a race or a restart defect. Keep the baseline, +mutation source, action evidence, and cleanup outcome together. A control for one +defect does not validate all alternative implementations or failure modes. + +## Media evidence + +`--media ` records videos and failure screenshots. `--trace` adds a +Playwright trace with DOM and network snapshots. + +```bash +npx playwright show-trace +``` + +Inspect the failing actor's evidence before attributing a failure. Media belongs +with run output and is not tracked in the repository. + +## Execution target + +Preflight binds the stack adapter, database or module name, ports, container +identity, and run lease. The suite runner verifies that exact target before +grading. A mismatch is a harness failure and cannot produce an application +score. + +When several stacks fail the same check, inspect the structured evidence. A +shared failure is useful diagnostic information, but it does not prove whether +the apps or the check are wrong. diff --git a/tools/stack-bench/grader/grade.ts b/tools/stack-bench/grader/grade.ts new file mode 100644 index 00000000000..4ee9acfc340 --- /dev/null +++ b/tools/stack-bench/grader/grade.ts @@ -0,0 +1,1148 @@ +#!/usr/bin/env node +/// +// Score declared criteria from one observed run in isolated actor contexts. +// +import { chromium } from 'playwright'; +import { attemptBrowserLaunchOptions } from '../container/browser-pipe.js'; +import type { Browser, BrowserContext, Page } from 'playwright'; +import { sanitiseConsoleError } from '../src/evidence/diagnostic-sanitizer.js'; +import { inspectSavedDiagnostic } from '../src/runtime/saved-diagnostic.js'; +import { randomUUID } from 'node:crypto'; +import { readFileSync, mkdirSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { harnessBrowserFailure, harnessProcessFailure, + runBrowserInfrastructureOperation } from '../src/evidence/harness-errors.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { materializeScenarioCredentials } from '../src/composition/credential-aliases.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { isFinding } from '../src/actions/action-findings.js'; +import type { Finding } from '../src/actions/action-findings.js'; +import { recipeArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { resolveGradeRecipeArtifactBinding } from '../src/composition/recipe-release.js'; +import { resolveBoundRecipeTaskRequest, selectScenarioChecks } from '../src/composition/recipe-selection.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { ActionApplicationFailure, ActionInconclusive, executeAction } from '../src/actions/action-contract.js'; +import { runApplicationNavigation } from '../src/actions/browser-navigation.js'; +import { createCheckEvidence, evidenceIsMeasured, evidencePassed } from '../src/evidence/check-evidence.js'; +import { evidenceNowMs } from '../src/evidence/evidence-timing.js'; +import { renderEvidenceConsoleLine } from '../src/evidence/evidence-presentation.js'; +import { measureGradePackRuntime } from '../src/composition/pack-runtime.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { + createNamedActionsCapability, +} from '../src/actions/actor-transport-action-executors.js'; +import type { ConcurrentCallResult } + from '../src/actions/actor-transport-action-executors.js'; +import { + createDatabaseWriteCapability, + createDatabaseReadCapability, + createLifecycleCapability, +} from '../src/actions/runtime-action-executors.js'; +import { requireLeasedDatabase } from '../src/stacks/backend-reset-guard.js'; +import type { LeasedDatabase } from '../src/stacks/backend-reset-guard.js'; +import { controlAppServer, controlBackendRuntime, parseRuntimeControlSpec, prepareRuntimeCrash } + from '../src/runtime/backend-control.js'; +import type { RuntimeControlSpec } from '../src/runtime/backend-control.js'; +import { leaseFromEnv } from '../src/runtime/backend-lease.js'; +import type { LeasedSpacetimeTarget } from '../src/runtime/spacetime-target.js'; + +import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js'; +import { captureResponses, ReceivedTransport } from './transport-frames.js'; +import { installResponseLoss } from './response-loss.js'; +import { installAuthWebSocketCapture } from '../src/actions/auth-request-patch.js'; +import { recordConvexSession } from '../src/stacks/backends/convex-browser-session.js'; +import type { ActionEvidence } from '../src/actions/action-contract.js'; +import type { CheckEvidence, CheckEvidenceAttachment, CheckEvidencePhase, + CheckEvidenceStatus } from '../src/evidence/check-evidence.js'; +import type { CompletedGradeFeatureResult, CompletedGradeReport, GradeCleanupFailure } + from '../src/evidence/grade-report.js'; +import type { CompiledFeature, CompiledScenarioDefinition, + CompiledStep } from '../src/composition/definition-compiler.js'; +import type { RecipeCheck, RecipeGradeRelease, RecipeRelease } from '../src/composition/recipe-release.js'; +import type { TrackAction } from '../src/composition/tracks.js'; + +type JsonRecord = Record; +type ActorWrite = { + url: string; + method: string; + headers: Record; + body: JsonRecord | null; +}; +type ActorWebSocketWrite = { event: unknown; body: JsonRecord }; +type ActorContextEntry = { context: BrowserContext; name: string; page: Page | null; traceStarted?: boolean }; +type CleanupBrowserContext = { + tracing: { stop(options: { path: string }): Promise }; + close(): Promise; +}; +type CleanupVideo = { saveAs(path: string): Promise; delete(): Promise }; +type CleanupPage = { video(): CleanupVideo | null }; +type CleanupActorContextEntry = { + traceStarted?: boolean; + context: CleanupBrowserContext; + name: string; + page: CleanupPage | null; +}; +type FeatureResult = Omit & { + setupEvidence?: CheckEvidence; +}; +type GradeArgs = { + url?: string; + level: number; + headed: boolean; + selectedCheckKeys: string[]; + out?: string; + label?: string; + feature?: number; + spec?: string; + restartSpec?: RuntimeControlSpec; + backend?: string; + track?: string; + recipe?: string; + recipeTask?: Parameters[1]; + expectedRecipeSha256?: string; + credentialAliases?: unknown; + selectionSha256?: string; + parentAttemptId?: string; + dbName?: string; + app?: string; + media?: string; + failureMedia?: string; + trace?: boolean; + nullControl: boolean; + diagnostic?: boolean; + savedDiagnostic?: ReturnType; + browserWsEndpoint?: string; +}; +type GradeRunContext = { + contractIds?: readonly string[]; + savedReader?: { path: string; sha256: string }; + checkoutActivity?: { unsettled: boolean }; + checkoutSnapshots?: ReturnType['checkoutSnapshots']; + actionCancellation?: { reason: string | null }; + runId: string; + roomName: (base: string) => string; + restartSpec?: RuntimeControlSpec; + url: string; + backend?: string; + actions: TrackAction[]; + spacetime: LeasedSpacetimeTarget | null; + dbName?: string; + databaseLease?: LeasedDatabase | null; + appDir?: string; + scope?: string; + extraContexts?: ActorContextEntry[]; + recorded?: Record; + unverified?: string[]; + verified?: string[]; + actionEvidence?: Array<{ actor: string | null; evidence: ActionEvidence }>; + serverCheck?: string | null; + lastCalls?: ConcurrentCallResult | null; + defaultWithin?: number; + nullControl: boolean; + // True while a scenario step has stopped the application server and no + // later step or restore has started it again. + applicationStopped?: boolean; +}; +type ActionFailure = Error & { actionEvidence?: ActionEvidence; actionActor?: string | null }; +const APPLICATION_RESTORE_SETTLE_MS = 8000; +const APPLICATION_RESTORE_TIMEOUT_MS = 60_000; +class ApplicationNotRestored extends Error { + constructor(reason: string) { + super(`the application server stopped by the harness was not restored: ${reason}`); + } +} +type CheckFailure = { + status: CheckEvidenceStatus; + code: string; + actor: string | null; + summary: string | null; + finding: Finding | null; + observation: unknown; + expected: unknown; + retryable: boolean; +}; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function actionFailure(error: unknown): ActionFailure | null { + return error instanceof Error ? error as ActionFailure : null; +} +// The sentence the coding agent was given for this behaviour travels with the +// grade, so a repair report can repeat it instead of describing the check. +const authored = (criterion: { statedBy?: string }): { statedBy?: string } => + criterion.statedBy ? { statedBy: criterion.statedBy } : {}; +const DEFAULT_WITHIN = 5000; +const SETUP_WITHIN = 20000; +// Keep the cause when Playwright prefixes it with locator retry details. +function keepReason(detail: unknown, limit = 600): string { + const s = String(detail ?? ''); + if (s.length <= limit) return s; + const [head, ...rest] = s.split('\n'); + const reasons = rest + .map(l => l.trim()) + .filter(l => /^-\s/.test(l)) + .map(l => l.replace(/^-\s*/, '')) + .filter(l => !/^(waiting for|retrying|attempting|scrolling|done scrolling|locator resolved to|\d+ ×)/i.test(l)); + const kept = [...new Set(reasons)].slice(0, 4); + const out = kept.length ? `${head}\n - ${kept.join('\n - ')}` : s.slice(0, limit); + return out.length > limit ? out.slice(0, limit) : out; +} + +export function parseGradeArgs(argv: readonly string[]): GradeArgs { + const { values } = parseNodeArgs({ args: [...argv.slice(2)], options: { + url: { type: 'string' }, level: { type: 'string' }, out: { type: 'string' }, + label: { type: 'string' }, feature: { type: 'string' }, spec: { type: 'string' }, + 'restart-spec': { type: 'string' }, backend: { type: 'string' }, track: { type: 'string' }, + recipe: { type: 'string' }, 'expected-recipe-sha256': { type: 'string' }, + 'recipe-task-json': { type: 'string' }, + 'selected-check': { type: 'string', multiple: true }, + 'credential-aliases-json': { type: 'string' }, 'selection-sha256': { type: 'string' }, + 'parent-attempt-id': { type: 'string' }, 'db-name': { type: 'string' }, + app: { type: 'string' }, media: { type: 'string' }, 'failure-media': { type: 'string' }, + trace: { type: 'boolean' }, headed: { type: 'boolean' }, + 'null-control': { type: 'boolean' }, + diagnostic: { type: 'boolean' }, + 'saved-diagnostic': { type: 'string' }, + 'browser-ws-endpoint': { type: 'string' }, + } }); + const args: GradeArgs = { url: values.url, level: values.level === undefined ? 1 : Number(values.level), + out: values.out, label: values.label, + feature: values.feature === undefined ? undefined : Number(values.feature), spec: values.spec, + restartSpec: values['restart-spec'] === undefined ? undefined + : parseRuntimeControlSpec(JSON.parse(values['restart-spec'])), + backend: values.backend, track: values.track, recipe: values.recipe, + recipeTask: values['recipe-task-json'] === undefined ? undefined : JSON.parse(values['recipe-task-json']), + expectedRecipeSha256: values['expected-recipe-sha256'], + selectedCheckKeys: values['selected-check'] ?? [], + credentialAliases: values['credential-aliases-json'] === undefined + ? undefined : JSON.parse(values['credential-aliases-json']), + selectionSha256: values['selection-sha256'], parentAttemptId: values['parent-attempt-id'], + dbName: values['db-name'], app: values.app, media: values.media, + failureMedia: values['failure-media'], trace: values.trace, headed: values.headed ?? false, + nullControl: values['null-control'] ?? false, + diagnostic: values.diagnostic ?? false, + browserWsEndpoint: values['browser-ws-endpoint'] }; + if (!args.url || !args.spec) { + throw new Error('Usage: node dist/grader/grade.js --url --spec ' + + '--level [--out ] [--label ] [--feature ]'); + } + if (args.diagnostic && (args.recipe || args.recipeTask !== undefined || args.expectedRecipeSha256 || args.selectedCheckKeys.length)) { + throw new Error('diagnostic grades cannot select a scored recipe or check catalog'); + } + if (values['saved-diagnostic']) { + if (!args.diagnostic) throw new Error('saved readers require zero-point diagnostics'); + args.savedDiagnostic = inspectSavedDiagnostic(JSON.parse(values['saved-diagnostic']), process.cwd()); + if (args.backend !== args.savedDiagnostic.backend) throw new Error('saved diagnostic backend mismatch'); + } + let url: URL; + try { url = new URL(args.url); } + catch { throw new Error('--url must be a valid HTTP or HTTPS URL'); } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('--url must use HTTP or HTTPS'); + } + if (!Number.isInteger(args.level) || args.level < 1) { + throw new Error('--level must be a positive integer'); + } + if (args.feature !== undefined && (!Number.isInteger(args.feature) || args.feature < 1)) { + throw new Error('--feature must be a positive integer'); + } + if (args.selectionSha256 && !/^[a-f0-9]{64}$/.test(args.selectionSha256)) { + throw new Error('--selection-sha256 must be 64 lowercase hexadecimal characters'); + } + if (args.browserWsEndpoint) { + let endpoint: URL; + try { endpoint = new URL(args.browserWsEndpoint); } + catch { throw new Error('--browser-ws-endpoint must be a valid WebSocket URL'); } + if (!['ws:', 'wss:'].includes(endpoint.protocol)) { + throw new Error('--browser-ws-endpoint must use ws or wss'); + } + } + return args; +} + +const tid = stableElementSelector; +const uniq = () => randomUUID().slice(0, 16); +const MAX_CONSOLE_ERRORS = 200; + +// Isolated browser actor + +// Which requests count as writes worth capturing for replay and forgery. The +// default covers chat's routes; a scenario spec can widen it for an application +// whose endpoints are named differently (`writeUrlPattern`). +const DEFAULT_WRITE_URL = '\\/api\\/|\\/rooms|\\/messages'; +let WRITE_URL_RE = new RegExp(DEFAULT_WRITE_URL); + + +export class Actor { + readonly name: string; + readonly context: BrowserContext; + page!: Page; + readonly consoleErrors: string[]; + private readonly transport = new ReceivedTransport(); + readonly ready: Promise; + get received(): readonly string[] { return this.transport.chunks; } + lastWrite: ActorWrite | null = null; + lastWrites: Record = {}; + writes: ActorWrite[] = []; + lastWsWrite: ActorWebSocketWrite | null = null; + annotate = false; + responseLoss?: Awaited>; + + constructor(name: string, page: Page, context: BrowserContext) { + this.name = name; + this.context = context; + this.consoleErrors = []; + // Test privacy against delivered payloads, not rendered content. + this.ready = this.attach(page); + } + async attach(page: Page): Promise { + this.page = page; + await installAuthWebSocketCapture(page); + // Capture writes so checks can replay them with changed fields or actors. + this.lastWrite = null; + this.lastWrites = {}; + this.writes = []; + this.lastWsWrite = null; + page.on('dialog', dialog => { + // Dismissing beforeunload cancels the navigation that the scenario requested. + void (dialog.type() === 'beforeunload' ? dialog.accept() : dialog.dismiss()).catch(error => { + if (page.isClosed()) return; + this.consoleErrors.push(`dialog handling failed: ${errorMessage(error)}`); + if (this.consoleErrors.length > MAX_CONSOLE_ERRORS) this.consoleErrors.shift(); + }); + }); + // Capture wire data separately from what the application renders. + page.on('websocket', ws => { + ws.on('framesent', f => { + recordConvexSession(page, ws.url(), f.payload); + const p = typeof f.payload === 'string' ? f.payload : ''; + const m = p.match(/^\d+(\[.*\])$/s); + if (!m) return; + try { + const [event, arg] = JSON.parse(m[1] as string) as unknown[]; + if (arg && typeof arg === 'object' && !Array.isArray(arg)) { + this.lastWsWrite = { event, body: arg as JsonRecord }; + } + } catch { /* not a socket.io event frame */ } + }); + // Binary frames are decoded as UTF-8 too, after any SpacetimeDB frame + // compression: a binary wire format still carries message text as + // inline UTF-8 bytes, so a substring search finds it without the + // harness knowing the encoding. + ws.on('framereceived', f => this.record(f.payload)); + }); + page.on('request', req => { + if (req.method() === 'GET' || req.method() === 'OPTIONS') return; + const url = req.url(); + if (!WRITE_URL_RE.test(url)) return; + let body: JsonRecord | null = null; + try { + const candidate: unknown = JSON.parse(req.postData() ?? ''); + if (candidate && typeof candidate === 'object' && !Array.isArray(candidate)) { + body = candidate as JsonRecord; + } + } catch { /* bodyless, e.g. a DELETE */ } + // Forging needs a body to tamper with; replaying does not — a privileged + // action is often a bare DELETE whose meaning is entirely in the URL. + const write = { url, method: req.method(), headers: req.headers(), body }; + this.writes.push(write); + if (this.writes.length > 200) this.writes.shift(); + if (body && typeof body === 'object') { + this.lastWrite = write; + this.lastWrites[req.method()] = write; + } + }); + page.on('console', m => { + if (m.type() !== 'error') return; + const text = m.text(); + // Expected 4xx responses are not application console failures. + if (/Failed to load resource.*status of 4\d\d/.test(text)) return; + this.consoleErrors.push(text.slice(0, 200)); + if (this.consoleErrors.length > MAX_CONSOLE_ERRORS) this.consoleErrors.shift(); + }); + page.on('pageerror', e => { + this.consoleErrors.push(`pageerror: ${e.message.slice(0, 200)}`); + if (this.consoleErrors.length > MAX_CONSOLE_ERRORS) this.consoleErrors.shift(); + }); + await captureResponses(page, this.transport); + } + record(payload: string | Buffer): void { + this.transport.record(payload); + } + wasSent(needle: string, requireComplete = true): boolean { + return this.transport.contains(needle, requireComplete); + } + loc(testid: string, { contains, scope }: + { contains?: string; scope?: { testid: string; contains?: string } } = {}) { + // `scope` narrows the search to inside a specific container (e.g. the badge + // belonging to ONE room), so a stale element elsewhere can't satisfy it. + const root = scope + ? this.page.locator(tid(scope.testid), { hasText: scope.contains }).filter({ visible: true }).first() + : this.page; + const selector = tid(testid); + return (contains + ? root.locator(selector, { hasText: contains }) + : root.locator(selector)).filter({ visible: true }).first(); + } +} + +// Expand scenario aliases to the run-scoped values used by the app. +const expand = (s: unknown, ctx: GradeRunContext): unknown => + typeof s === 'string' + ? s.replace(/\{room:([^}]+)\}/g, (_, b) => ctx.roomName(b)) + // Keep generated usernames alphanumeric so ordinary validators accept them. + .replace(/\{user:([^}]+)\}/g, (_, n) => `${n}${ctx.scope}`) + : s; + + +// Put test context in recordings without exposing it to scoped app selectors. + +const OVERLAY_ID = '__stackbench_overlay'; + +async function annotate(actor: Actor | undefined, { feature, criterion, step, status }: + { feature?: string; criterion?: string; step?: string; status?: 'fail' | 'pass' } = {}): Promise { + if (!actor?.annotate) return; + await actor.page.evaluate(({ id, feature, criterion, step, status, who }) => { + let el = document.getElementById(id); + if (!el) { + el = document.createElement('div'); + el.id = id; + el.style.cssText = [ + 'position:fixed', 'inset:0 0 auto 0', 'z-index:2147483647', + 'font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace', + 'padding:6px 10px', 'pointer-events:none', 'white-space:pre', + 'background:rgba(12,12,16,.92)', 'color:#e8e8ef', + 'border-bottom:2px solid #4c8dff', + ].join(';'); + document.documentElement.appendChild(el); + } + const colour = status === 'fail' ? '#ff5c5c' : status === 'pass' ? '#3ddc84' : '#4c8dff'; + el.style.borderBottomColor = colour; + el.textContent = [ + `${who} ${feature ?? ''}`, + criterion ? ` ${status === 'fail' ? 'FAILED' : 'checking'}: ${criterion}` : '', + step ? ` > ${step}` : '', + ].filter(Boolean).join(String.fromCharCode(10)); + }, { id: OVERLAY_ID, feature, criterion, step, status, who: actor.name }).catch(() => {}); +} + +// Step execution + +function abortableSleep(ms: number, signal: AbortSignal | null = null): Promise { + if (signal?.aborted) return Promise.reject(signal.reason ?? new Error('action cancelled')); + return new Promise((resolve, reject) => { + const timer = setTimeout(done, ms); + function done() { + signal?.removeEventListener('abort', cancelled); + resolve(); + } + function cancelled() { + clearTimeout(timer); + signal?.removeEventListener('abort', cancelled); + reject(signal?.reason ?? new Error('action cancelled')); + } + signal?.addEventListener('abort', cancelled, { once: true }); + }); +} + +function browserActionCapabilities(actors: Map, ctx: GradeRunContext): Readonly> { + const defaultWithin = ctx.defaultWithin ?? DEFAULT_WITHIN; + const actorAccess = Object.freeze({ get: (name: string) => actors.get(name) }); + const runtimeValues = Object.freeze({ + applicationUrl: ctx.url, + defaultWithin, + expand: (value: unknown) => expand(value, ctx), + hyphenatedScopedUser: (name: string) => `${name}-${ctx.scope}`, + roomName: (base: string) => ctx.roomName(base), + scopedUser: (name: string) => `${name}${ctx.scope}`, + recorded: Object.freeze({ + get: (key: string) => ctx.recorded?.[key], + set: (key: string, value: unknown) => { (ctx.recorded ??= {})[key] = value; }, + }), + sleep: abortableSleep, + testId: tid, + clients: Object.freeze({ + async open(actor: Actor, settleMs: number, signal: AbortSignal) { + const fresh = await actor.context.newPage(); + fresh.setDefaultTimeout(defaultWithin); + await actor.attach(fresh); + await runApplicationNavigation(() => fresh.goto(ctx.url, { waitUntil: 'domcontentloaded', timeout: 20000 }), fresh); + await abortableSleep(settleMs, signal); + }, + async fresh(actor: Actor, sourceName: string, preserveStorage: boolean) { + const browser = actor.page.context().browser(); + if (!browser) throw new Error('actor browser is unavailable'); + const context = await browser.newContext(preserveStorage + ? { storageState: await actor.context.storageState({ indexedDB: true }) } + : undefined); + const name = `${sourceName}-fresh`; + // Own cleanup before page creation or navigation can fail. + const entry: ActorContextEntry = { context, name, page: null }; + ctx.extraContexts?.push(entry); + const fresh = await context.newPage(); + entry.page = fresh; + fresh.setDefaultTimeout(defaultWithin); + const observer = new Actor(`${actor.name}-fresh`, fresh, context); + await observer.ready; + // storageState omits sessionStorage. Seed the first document only; + // later reloads must retain the application's own storage changes. + const session = preserveStorage ? await context.newCDPSession(fresh) : null; + let seed: string | undefined; + try { + if (session) { + await session.send('Page.enable'); + const state = await actor.page.evaluate(() => ({ + origin: location.origin, entries: Object.entries(sessionStorage), + })); + const script = await session.send('Page.addScriptToEvaluateOnNewDocument', { + source: `if (window === window.top && location.origin === ${JSON.stringify(state.origin)}) { + for (const [key, value] of ${JSON.stringify(state.entries)}) sessionStorage.setItem(key, value); + }`, + }); + seed = script.identifier; + } + await runApplicationNavigation(() => fresh.goto(ctx.url, { waitUntil: 'domcontentloaded', timeout: 20000 }), fresh); + } finally { + if (session) { + try { if (seed) await session.send('Page.removeScriptToEvaluateOnNewDocument', { identifier: seed }); } + finally { await session.detach(); } + } + } + observer.annotate = actor.annotate; + actors.set(name, observer); + return name; + }, + }), + }); + const transportObservation = Object.freeze({ + defaultWithin, + expand: (value: unknown) => expand(value, ctx), + sleep: abortableSleep, + verification: Object.freeze({ + structural(message: string) { + ctx.verified?.push(message); + ctx.serverCheck = ctx.serverCheck ?? 'structural'; + }, + unverified(message: string) { + (ctx.unverified ??= []).push(message); + ctx.serverCheck = 'unverified'; + }, + verified(message: string) { + ctx.verified?.push(message); + ctx.serverCheck = 'verified'; + }, + }), + }); + const namedActions = createNamedActionsCapability({ + actions: ctx.actions, + backend: ctx.backend!, + url: ctx.url, + spacetime: ctx.spacetime, + lastCalls: Object.freeze({ + get: () => ctx.lastCalls ?? null, + set: value => { ctx.lastCalls = value; }, + }), + sleep: abortableSleep, + }); + const concurrency = Object.freeze({ + defaultWithin, + dispatch: (step: CompiledStep, signal: AbortSignal) => runRegisteredAction(step, actors, ctx, signal), + expand: (value: unknown) => expand(value, ctx), + sleep: abortableSleep, + testId: tid, + }); + return Object.freeze({ + actors: actorAccess, + 'response-loss': Object.freeze({ + async prepare(name: string) { + const actor = actors.get(name); + if (!actor || actor.responseLoss) throw new Error('response-loss actor is missing or already prepared'); + actor.responseLoss = await installResponseLoss(actor.context); + // Existing sockets predate interception. Replace them before setup writes. + await runApplicationNavigation(() => actor.page.reload({ waitUntil: 'domcontentloaded', timeout: 20000 }), actor.page); + }, + get(name: string) { + const gate = actors.get(name)?.responseLoss; + if (!gate) throw new Error('response loss was not prepared'); + return gate; + }, + }), + 'application-files': Object.freeze({ root: ctx.appDir ?? null, expand: (value: unknown) => expand(value, ctx) }), + 'application-lifecycle': applicationLifecycle(ctx), + 'backend-lifecycle': createLifecycleCapability({ + restartSpec: ctx.restartSpec, + target: 'backend-runtime', + control: controlBackendRuntime, + sleep: abortableSleep, + }), + 'browser-interaction': runtimeValues, + 'browser-observation': runtimeValues, + clock: Object.freeze({ sleep: abortableSleep }), + concurrency, + 'database-read': createDatabaseReadCapability({ + contractIds: ctx.contractIds, + savedReader: ctx.savedReader, + checkoutSnapshots: ctx.checkoutSnapshots ??= new Map(), + checkoutActivity: ctx.checkoutActivity ??= { unsettled: false }, + app: ctx.appDir, + backend: ctx.backend, + spacetime: ctx.spacetime, + databaseLease: ctx.databaseLease, + skip: ctx.nullControl, + expand: (value: string) => String(expand(value, ctx)), + }), + 'database-write': createDatabaseWriteCapability({ + backend: ctx.backend, + spacetime: ctx.spacetime, + databaseLease: ctx.databaseLease, + skip: ctx.nullControl, + expand: (value: string) => String(expand(value, ctx)), + }), + 'named-actions': namedActions, + 'process-crash': Object.freeze({ combinedBoundary: !ctx.nullControl && ['spacetime', 'convex'].includes(ctx.restartSpec?.backend ?? ''), + prepare: (target: 'application' | 'database') => { + if (!ctx.restartSpec || ctx.nullControl) throw new Error('process crash requires an owned grading runtime'); + return prepareRuntimeCrash(ctx.restartSpec, target); + } }), + subprocess: Object.freeze({ sleep: abortableSleep }), + 'transport-observation': transportObservation, + }); +} + +function applicationLifecycle(ctx: GradeRunContext) { + return createLifecycleCapability({ + restartSpec: ctx.restartSpec, + target: 'app-server', + control: controlAppServer, + sleep: abortableSleep, + onOperated: mode => { ctx.applicationStopped = mode === 'stop'; }, + }); +} + +async function runRegisteredAction(step: CompiledStep, actors: Map, ctx: GradeRunContext, + signal: AbortSignal | null = null): Promise { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + const actionEvidence = await executeAction(ACTION_REGISTRY, step.do, step, + { + capabilities: browserActionCapabilities(actors, ctx), + signal, + // Closing this grader's connection cancels pending Playwright calls before + // the action executor drains them. It does not stop the app or its database. + onAbort: ACTION_REGISTRY.get(step.do).capabilities.includes('actors') + ? async () => { + if (ctx.actionCancellation) ctx.actionCancellation.reason = 'browser session cancelled; no further actions are permitted'; + const browsers = new Set([...actors.values()].map(actor => actor.page.context().browser())); + await Promise.all([...browsers].map(browser => browser?.close())); + } : undefined, + }); + ctx.actionEvidence?.push({ actor: step.actor ?? null, evidence: actionEvidence }); + if (actionEvidence.status === 'passed') return actionEvidence.observation; + const error = new Error(actionEvidence.summary ?? `${step.do} did not complete`); + Object.defineProperty(error, 'actionEvidence', { value: actionEvidence }); + Object.defineProperty(error, 'actionActor', { value: step.actor ?? null }); + throw error; +} + +function classifyCheckFailure(error: unknown, fallbackActor: string | null = null): CheckFailure { + const actionError = actionFailure(error); + const actionEvidence = actionError?.actionEvidence; + if (actionEvidence) { + return { + status: actionEvidence.status, + code: actionEvidence.code, + actor: actionError?.actionActor ?? fallbackActor, + summary: actionEvidence.summary ?? `${actionEvidence.action.id} did not complete`, + finding: actionEvidence.finding, + observation: actionEvidence.observation, + expected: actionEvidence.expected, + retryable: actionEvidence.retryable, + }; + } + if (error instanceof ApplicationNotRestored) { + return { status: 'harness_failure', code: 'application_not_restored', actor: fallbackActor, + summary: error.message, finding: null, observation: null, expected: null, retryable: false }; + } + const processFailure = harnessProcessFailure(error); + if (processFailure) return { status: 'harness_failure', code: 'process_failure', actor: fallbackActor, + summary: processFailure, finding: null, observation: null, expected: null, retryable: false }; + const browserFailure = harnessBrowserFailure(error); + if (browserFailure) return { status: 'harness_failure', code: 'browser_failure', actor: fallbackActor, + summary: browserFailure, finding: null, observation: null, expected: null, retryable: false }; + if (error instanceof ActionApplicationFailure || error instanceof ActionInconclusive) { + return { status: error instanceof ActionInconclusive ? 'inconclusive' : 'failed', + code: error.classification, actor: fallbackActor, + summary: error.message, finding: isFinding(error.details.finding) ? error.details.finding : null, + observation: error.details.observation ?? null, + expected: error.details.expected ?? null, retryable: error.details.retryable === true }; + } + return { status: 'harness_failure', code: 'unclassified_exception', actor: fallbackActor, + summary: errorMessage(error ?? 'unknown grader failure'), + finding: null, observation: null, expected: null, retryable: false }; +} + +function buildCheckEvidence({ ctx, phase, startedAtMs, failure = null, actor = null, summary = null, + attachments = [], actions = ctx.actionEvidence ?? [], sensitivity = null }: { + ctx: GradeRunContext; phase: CheckEvidencePhase; startedAtMs: number; failure?: unknown; + actor?: string | null; summary?: string | null; attachments?: Array; + actions?: Array<{ actor: string | null; evidence: ActionEvidence }>; + sensitivity?: readonly string[] | null; + }): CheckEvidence { + const classified: CheckFailure = failure ? classifyCheckFailure(failure, actor) : { + status: 'passed', code: 'completed', actor: null, summary: null, finding: null, + observation: null, expected: null, retryable: false, + }; + const completedAtMs = Math.max(startedAtMs, evidenceNowMs()); + const evidenceSummary = summary ?? classified.summary; + return createCheckEvidence({ + ...classified, + phase, + summary: evidenceSummary == null ? null : keepReason(evidenceSummary), + startedAtMs, + completedAtMs, + actions, + attachments: attachments.map(attachment => typeof attachment === 'string' + ? { kind: 'screenshot', ref: basename(attachment) } : attachment), + sensitivity: sensitivity ?? actions.flatMap(entry => entry.evidence?.sensitivity ?? []), + }); +} + +async function runStep(step: CompiledStep, actors: Map, ctx: GradeRunContext): Promise { + return runRegisteredAction(step, actors, ctx); +} + +export async function closeActorContexts(entries: readonly CleanupActorContextEntry[], { + trace = false, media = null, slug = 'grade', +}: { trace?: boolean; media?: string | null; slug?: string } = {}): Promise { + const failures: GradeCleanupFailure[] = []; + const record = (name: string, stage: string, error: unknown): void => { failures.push({ + actor: name, + stage, + reason: keepReason(errorMessage(error)), + }); }; + for (const { context, name, page, traceStarted } of entries) { + if (trace && traceStarted) { + try { + await context.tracing.stop({ path: join(media ?? '.', `${slug}-${name}.trace.zip`) }); + } catch (error) { record(name, 'trace', error); } + } + let video = null; + if (media && page) { + try { video = page.video(); } + catch (error) { record(name, 'video-handle', error); } + } + try { await context.close(); } + catch (error) { record(name, 'context-close', error); } + if (video) { + try { await video.saveAs(join(media!, `${slug}-${name}.webm`)); } + catch (error) { record(name, 'video-save', error); } + try { await video.delete(); } + catch (error) { record(name, 'video-delete', error); } + } + } + return failures; +} + +// Feature grading + +function completedFeatureResult(result: FeatureResult): CompletedGradeFeatureResult { + if (!result.setupEvidence) { + throw new Error(`feature ${result.id} completed without setup evidence`); + } + return { ...result, setupEvidence: result.setupEvidence }; +} + +export async function gradeFeature(browser: Browser, feature: CompiledFeature, args: GradeArgs, + runCtx: GradeRunContext): Promise { + // Features share the app's DATABASE even though each gets fresh browser + // contexts, so user and room names are scoped per feature — otherwise a + // defect in one feature (e.g. a hijacked account) corrupts later setups. + const scope = `${runCtx.runId}f${feature.id}`; + runCtx.checkoutActivity ??= { unsettled: false }; + const extraContexts: ActorContextEntry[] = []; + const ctx: GradeRunContext = { ...runCtx, scope, roomName: (base: string) => `${base}-${scope}`, extraContexts, recorded: {}, checkoutSnapshots: new Map(), + unverified: [], verified: [], actionEvidence: [] }; + const actors = new Map(); + const contexts: ActorContextEntry[] = []; + const slug = `${args.label ?? 'run'}-f${feature.id}`; + + // A feature is worth what its criteria are worth. An explicit `max` is only + // a consistency check enforced by check-scenarios, never a top-up. + const featureMax = feature.criteria.reduce((n, c) => n + (c.points ?? 1), 0); + const result: FeatureResult = { + id: feature.id, name: feature.name, score: 0, max: featureMax, + criteria: [], consoleErrors: [], + }; + const restoreFailures: GradeCleanupFailure[] = []; + const closeAll = async () => { + for (const actor of actors.values()) { + for (const message of actor.consoleErrors) { + result.consoleErrors.push(`[${actor.name}] ${sanitiseConsoleError(message)}`); + } + } + // The abort hook already closed this connection, or reported that closure + // could not be confirmed. Do not hang again while collecting browser media. + if (ctx.actionCancellation?.reason) { + const failures = [{ actor: null, stage: 'browser-cancel', reason: ctx.actionCancellation.reason }]; + result.cleanupEvidence = { status: 'harness_failure', failures }; + return failures; + } + const failures = [...restoreFailures, ...await closeActorContexts([...contexts, ...extraContexts], { + trace: args.trace, media: args.media, slug, + })]; + if (failures.length) result.cleanupEvidence = { status: 'harness_failure', failures }; + return failures; + }; + // A criterion that stops the application server owns it only for its own + // steps. Whatever the outcome, the server is running again before the next + // criterion; a restore the harness cannot complete is the harness's failure + // and every later criterion in the feature is unmeasured, not failed. + const restoreApplicationServer = async () => { + if (!ctx.applicationStopped || restoreFailures.length) return; + try { + await applicationLifecycle(ctx) + .operate('start', APPLICATION_RESTORE_SETTLE_MS, AbortSignal.timeout(APPLICATION_RESTORE_TIMEOUT_MS)); + } catch (error) { + restoreFailures.push({ actor: null, stage: 'application-restore', + reason: keepReason(errorMessage(error)) }); + } + }; + const initializationStartedAtMs = evidenceNowMs(); + try { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + for (const name of feature.actors!) { + // Isolated storage per actor. Video is per-context, so each actor gets its + // own recording — you can watch what every participant saw, side by side. + const context = await runBrowserInfrastructureOperation('context creation', () => + browser.newContext( + args.media ? { recordVideo: { dir: args.media, size: { width: 1280, height: 800 } } } : {} + )); + contexts.push({ context, name, page: null }); + if (args.trace) { + await runBrowserInfrastructureOperation('trace start', () => + context.tracing.start({ screenshots: true, snapshots: true })); + contexts[contexts.length - 1]!.traceStarted = true; + } + const page = await runBrowserInfrastructureOperation('page creation', () => context.newPage()); + contexts[contexts.length - 1]!.page = page; + page.setDefaultTimeout(SETUP_WITHIN); + const actor = new Actor(name, page, context); + await actor.ready; + actor.annotate = Boolean(args.media); + actors.set(name, actor); + await runApplicationNavigation(() => page.goto(args.url!, { waitUntil: 'domcontentloaded', timeout: 20000 }), page); + } + } catch (error) { + const classified = classifyCheckFailure(error); + const reason = keepReason((classified.summary ?? '').trim()); + result.setupEvidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: initializationStartedAtMs, + failure: error, summary: reason }); + for (const criterion of feature.criteria) { + const points = criterion.points ?? 1; + const evidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: initializationStartedAtMs, + failure: error, summary: `browser setup failed: ${reason}`, actions: [] }); + if (evidence.status === 'failed') evidence.status = 'blocked'; + result.criteria.push({ id: criterion.id, desc: criterion.desc, points, evidence, + ...authored(criterion) }); + if (!evidenceIsMeasured(evidence)) result.inconclusive = [...(result.inconclusive ?? []), + { id: criterion.id, points, status: evidence.status, code: evidence.code, + phase: evidence.phase, summary: evidence.summary }]; + } + await closeAll(); + return completedFeatureResult(result); + } + + const captureFailureScreenshots = async (label: string): Promise => { + if (ctx.actionCancellation?.reason || !args.failureMedia) return []; + mkdirSync(args.failureMedia, { recursive: true }); + const captured: string[] = []; + for (const { name, page } of [...contexts, ...extraContexts]) { + if (!page) continue; + const path = join(args.failureMedia, `${slug}-${label}-${name}.png`); + const ok = await page.screenshot({ path, fullPage: true, timeout: 5000 }) + .then(() => true, () => false); + if (ok) captured.push(path); + } + return captured; + }; + + const setupStartedAtMs = evidenceNowMs(); + ctx.defaultWithin = SETUP_WITHIN; + ctx.actionEvidence = []; + try { + // Setup is not scored, but a failure makes the feature untestable (0). + for (const step of feature.setup) { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + await annotate(actors.get(step.actor), { feature: feature.name, criterion: 'setup', step: step.do }); + await runStep(step, actors, ctx); + } + } catch (err) { + // Preserve the typed setup failure on every affected criterion. + const classified = classifyCheckFailure(err); + const why = keepReason((classified.summary ?? '').trim()); + const screenshots = await captureFailureScreenshots('setup'); + result.setupEvidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: setupStartedAtMs, + failure: err, summary: why, attachments: screenshots }); + for (const c of feature.criteria) { + const base = why ? `Blocked by a failed prerequisite: ${why}` : 'Blocked by a failed prerequisite'; + const points = c.points ?? 1; + const evidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: setupStartedAtMs, + failure: err, summary: base, actions: [], sensitivity: result.setupEvidence.sensitivity, + attachments: [{ kind: 'check-evidence', ref: 'feature.setupEvidence' }, ...screenshots] }); + if (evidence.status === 'failed') evidence.status = 'blocked'; + const recorded = { id: c.id, desc: c.desc, points, evidence, ...authored(c) }; + result.criteria.push(recorded); + if (!evidenceIsMeasured(evidence)) { + result.inconclusive = [...(result.inconclusive ?? []), + { id: c.id, points, status: evidence.status, code: evidence.code, + phase: evidence.phase, summary: evidence.summary }]; + } + } + if (screenshots.length) result.screenshots = screenshots; + await closeAll(); + return completedFeatureResult(result); + } + result.setupEvidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: setupStartedAtMs }); + ctx.defaultWithin = DEFAULT_WITHIN; + for (const actor of actors.values()) actor.page.setDefaultTimeout(DEFAULT_WITHIN); + + for (const criterion of feature.criteria) { + let failure: unknown = null, detail: string | null = null, activeActor: string | null = null; + let criterionScreenshots: string[] = []; + const criterionStartedAtMs = evidenceNowMs(); + ctx.actionEvidence = []; + ctx.serverCheck = null; + try { + if (restoreFailures.length) throw new ApplicationNotRestored(restoreFailures[0]!.reason); + for (const step of criterion.steps) { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + activeActor = step.actor ?? activeActor; + await annotate(actors.get(step.actor) ?? actors.values().next().value, + { feature: feature.name, criterion: criterion.id, step: step.do }); + await runStep(step, actors, ctx); + } + for (const a of actors.values()) { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + await annotate(a, { feature: feature.name, criterion: criterion.id, step: 'passed', status: 'pass' }); + } + } catch (err) { + failure = err; + const classified = classifyCheckFailure(err, activeActor); + detail = classified.summary; + // captureFailureScreenshots also refuses a cancelled session. + if (!ctx.actionCancellation?.reason && args.media) { + for (const a of actors.values()) { + await annotate(a, { feature: feature.name, criterion: criterion.id, + step: errorMessage(err).slice(0, 120), status: 'fail' }); + } + const shotActor = actors.get(criterion.steps[criterion.steps.length - 1]?.actor) ?? actors.values().next().value; + const shot = join(args.media, `${slug}-${criterion.id}.png`); + const captured = await shotActor.page.screenshot({ path: shot, fullPage: true }) + .then(() => true, () => false); + if (captured) criterionScreenshots.push(shot); + } else { + criterionScreenshots = await captureFailureScreenshots(criterion.id); + } + if (criterionScreenshots.length) { + result.screenshots = [...(result.screenshots ?? []), ...criterionScreenshots]; + } + } + await restoreApplicationServer(); + const evidence = buildCheckEvidence({ ctx, phase: 'assertion', startedAtMs: criterionStartedAtMs, + failure, actor: activeActor, summary: detail, attachments: criterionScreenshots }); + result.criteria.push({ id: criterion.id, desc: criterion.desc, points: criterion.points, + evidence, ...authored(criterion), + ...(ctx.serverCheck ? { serverCheck: ctx.serverCheck } : {}) }); + if (evidencePassed(evidence)) result.score += criterion.points; + else if (!evidenceIsMeasured(evidence)) { + result.inconclusive = [...(result.inconclusive ?? []), + { id: criterion.id, points: criterion.points, status: evidence.status, code: evidence.code, + phase: evidence.phase, summary: evidence.summary }]; + } + } + + // Retain diagnostics for server-side checks that could not execute. The + // action executor marks those criteria inconclusive, so they cannot score. + if (ctx.unverified?.length) result.unverified = ctx.unverified; + if (ctx.verified?.length) result.verified = ctx.verified; + + await closeAll(); + if (args.media) result.videos = contexts.map(c => join(args.media!, `${slug}-${c.name}.webm`)); + return completedFeatureResult(result); +} + +// Main + +export function gradeDatabaseLease(backend?: string, env: NodeJS.ProcessEnv = process.env): LeasedDatabase | null { + if ((backend !== 'mongodb' && backend !== 'postgres') + || !(env.STACK_BENCH_LEASE || env.STACK_BENCH_LEASE_TOKEN)) return null; + return requireLeasedDatabase(leaseFromEnv(env, { backend, active: true }).lease); +} + +async function main(): Promise { + const startedAt = new Date().toISOString(); + const args = parseGradeArgs(process.argv); + const specPath = args.spec!; + let spec: CompiledScenarioDefinition; + try { + const compiled = compileScenarioDefinition(JSON.parse(readFileSync(specPath, 'utf8')), + { source: specPath }); + spec = materializeScenarioCredentials(compiled, args.credentialAliases); + } catch (error) { + throw new Error(`cannot compile scenario ${specPath}: ${errorMessage(error)}`, { cause: error }); + } + + if (typeof spec.writeUrlPattern === 'string' && spec.writeUrlPattern) { + WRITE_URL_RE = new RegExp(spec.writeUrlPattern); + } + + const candidateFeatures = args.feature ? spec.features.filter(f => f.id === args.feature) : spec.features; + if (args.diagnostic && candidateFeatures.some(feature => feature.criteria.some(check => check.points !== 0))) { + throw new Error('diagnostic grades require zero-point checks'); + } + if (args.feature && candidateFeatures.length === 0) { + throw new Error(`scenario ${specPath} has no feature ${args.feature}`); + } + const runId = uniq(); + // Where the named actions live. The track declares their names; the + // authenticated backend lease—not generated application config—selects the + // SpacetimeDB host, module and exact build container used for direct SQL. + let actions: TrackAction[] = [], spacetime: LeasedSpacetimeTarget | null = null, + selectedTask: ReturnType | null = null, + recipeRelease: RecipeGradeRelease | null = null, + recipeIdentityRelease: RecipeRelease | null = null, + calibration: ReturnType | null = null; + if (args.track) { + const track = loadTrack(args.track); + actions = track.actions; + const binding = args.diagnostic ? null : resolveGradeRecipeArtifactBinding(track, args.level, specPath, + args.feature ?? null, args.recipe); + recipeRelease = binding?.release ?? null; + recipeIdentityRelease = binding?.sourceRelease ?? null; + if (args.recipeTask !== undefined) { + if (!binding) throw new Error('recipe task requires a bound grade recipe'); + selectedTask = resolveBoundRecipeTaskRequest(binding.binding, args.recipeTask); + } + } + if (args.recipeTask !== undefined && !selectedTask) throw new Error('recipe task requires a bound grade recipe'); + if (args.expectedRecipeSha256 + && recipeRelease?.contentSha256 !== args.expectedRecipeSha256) { + throw new Error(`recipe changed before grading: expected ${args.expectedRecipeSha256}, ` + + `resolved ${recipeRelease?.contentSha256 ?? 'no recipe'}`); + } + const selectedScenario = selectScenarioChecks( + { ...spec, features: candidateFeatures }, recipeRelease, args.selectedCheckKeys); + const features = selectedScenario.features; + const selectedChecks = selectedScenario.checks; + if (!features.length) throw new Error(`scenario ${specPath} has no selected checks`); + if (args.track) { + const track = loadTrack(args.track); + calibration = resolveCalibrationForRelease(recipeIdentityRelease, { + trackRoot: track.dir, + stackBenchRoot: ROOT, + alias: `L${args.level}`, + }); + } + spacetime = args.backend + ? STACK_ADAPTER_REGISTRY.get(args.backend).grading.context({ requireBuildContainer: true }) + : null; + const databaseLease = gradeDatabaseLease(args.backend); + + const ctx: GradeRunContext = { actionCancellation: { reason: null }, runId, roomName: (base: string) => `${base}-${runId}`, + restartSpec: args.restartSpec, url: args.url!, + backend: args.backend, actions, spacetime, dbName: args.dbName, + databaseLease, + nullControl: args.nullControl, + contractIds: selectedTask?.task.contractIds, + appDir: args.app, savedReader: args.savedDiagnostic?.reader }; + + const browser = args.browserWsEndpoint + ? await chromium.connect(args.browserWsEndpoint) + : await chromium.launch({ headless: !args.headed, ...attemptBrowserLaunchOptions() }); + const report: JsonRecord & CompletedGradeReport & { + inconclusive?: Array; + cleanupEvidence?: { status: 'harness_failure'; failures: GradeCleanupFailure[] }; + } = { + definitionSchemaVersion: spec.schemaVersion, + recipeRelease, + ...(selectedTask ? { recipeTask: selectedTask.request } : {}), + label: args.label ?? null, url: args.url, level: args.level, runId, + total: 0, max: features.reduce((n, f) => n + f.criteria.reduce((m, c) => m + (c.points ?? 1), 0), 0), features: [], + selection: recipeRelease ? { + ...(args.selectionSha256 ? { sha256: args.selectionSha256 } : {}), + checks: selectedChecks.map(({ stableKey, packId, checkGroupId, featureId, criterionId, + description, points }) => { + if (!packId) throw new Error(`selected check ${stableKey} has no pack id`); + return { stableKey, packId, checkGroupId, featureId, criterionId, description, points }; + }), + } : null, + }; + const checkByCriterion = new Map(selectedChecks.map(check => [ + `${String(check.featureId)}\0${String(check.criterionId)}`, check, + ])); + + try { + for (const feature of features) { + process.stdout.write(`Feature ${feature.id}: ${feature.name} ... `); + const r = await gradeFeature(browser, feature, args, ctx); + if (recipeRelease) { + for (const criterion of r.criteria) { + const check = checkByCriterion.get(`${String(feature.id)}\0${String(criterion.id)}`); + if (!check) throw new Error(`graded criterion ${feature.id}/${criterion.id} has no recipe check`); + criterion.stableKey = check.stableKey; + } + } + report.features.push(r); + report.total += r.score; + // The recipe owns the denominator. An unmeasured criterion earns zero and + // remains explicitly inconclusive; it must never change the contract. + if (r.inconclusive?.length) { + report.inconclusive = [...(report.inconclusive ?? []), + ...r.inconclusive.map(c => ({ feature: r.id, ...c }))]; + } + console.log(`${r.score}/${r.max}`); + for (const c of r.criteria.filter(c => !evidencePassed(c.evidence))) { + console.log(` ${renderEvidenceConsoleLine(c.evidence, c.id)}`); + } + } + } finally { + try { if (!ctx.actionCancellation?.reason) await browser.close(); } + catch (error) { + report.cleanupEvidence = { status: 'harness_failure', failures: [{ + actor: null, stage: 'browser-close', reason: keepReason(errorMessage(error)), + }] }; + } + } + + if (recipeRelease) report.packRuntime = measureGradePackRuntime(report); + + console.log(`\nTOTAL ${report.total}/${report.max}`); + if (args.out) { + const artifactId = `grade-${runId}`; + writeArtifact(args.out, { + kind: 'grade', + id: artifactId, + attempt: { id: artifactId, parentId: args.parentAttemptId ?? null }, + timestamps: { startedAt, completedAt: new Date().toISOString() }, + identities: recipeArtifactIdentities(recipeIdentityRelease, { + calibration: calibration ? { id: calibration.id, + sha256: calibration.contentSha256 } : null, + stackAdapter: args.backend ? { id: args.backend } : null, + }), + payload: report, + }); + console.log(`Report written to ${args.out}`); + } +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main().catch(error => { + console.error(error instanceof Error ? (error.stack ?? error.message) : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/grader/mutation-test.ts b/tools/stack-bench/grader/mutation-test.ts new file mode 100644 index 00000000000..d4db19cdbc3 --- /dev/null +++ b/tools/stack-bench/grader/mutation-test.ts @@ -0,0 +1,865 @@ +#!/usr/bin/env node +// A valid mutation fails only its declared criterion against a passing baseline. +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { dirname, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; +import { execFileSync } from "node:child_process"; +import { parseArgs as parseNodeArgs } from "node:util"; +import { currentEngineIdentity, emptyArtifactIdentities, readArtifactPayload, + writeRunJson } from "../src/evidence/artifacts.js"; +import { controlAppServer, parseRuntimeControlSpec } from "../src/runtime/backend-control.js"; +import type { RuntimeControlSpec } from "../src/runtime/backend-control.js"; +import { leaseFromEnv } from '../src/runtime/backend-lease.js'; +import { inspectBuildContainer } from '../src/stacks/hosted-lifecycle.js'; +import { CODING_CONTAINER_APP_ROOT, codingContainerAgentCommand, codingContainerAgentExecOptions } + from '../src/runtime/coding-container-policy.js'; +import { + classifyMutationResult, + groupMutationsByScenario, + isRetryableMutationBaseline, + isRetryableMutationResult, + mutationFileEdits, + mutationTargetKeys, + readMutationManifest, + releaseScenarioCheckKeys, + resolveMutationScenarioPath, + reusableMutationBaseline, + resolveMutationFile, + validateMutationBaseline, + validateMutationDefinitions, +} from "../src/evidence/mutation-analysis.js"; +import { dbName, loadTrack, TRACK_MANIFEST_FILE } from "../src/composition/tracks.js"; +import { resolveRecipeRelease } from "../src/composition/recipe-release.js"; +import { createBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { resetBackend } from "../src/stacks/backend-reset.js"; +import { STACK_ADAPTER_REGISTRY } from "../src/stacks/stack-adapters.js"; +import { mutationShard } from "../src/evidence/mutation-shards.js"; +import { reusableMutationEvidence } from "../src/evidence/mutation-checkpoint.js"; +import { mutationGradeTimeoutMs } + from "../src/evidence/mutation-control.js"; +import { GRADER_SOURCE_TIMEOUT_MS, gradingSourceTimeoutMs } from '../src/runtime/grading-timeout.js'; +import { assertAppSourceIdentity } from "../src/runtime/source-snapshot.js"; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import type { LoadedMutationManifest, MutationDefinition } from '../src/evidence/mutation-analysis.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import type { MutationCheckpointBaseline, MutationCheckpointIdentity, + MutationCheckpointResult } from '../src/evidence/mutation-checkpoint.js'; + +type JsonRecord = Record; +type MutationSpec = LoadedMutationManifest; +type MutationArgs = { + app?: string; url?: string; mutations?: string; level?: string; spec?: string; backend?: string; + track?: string; recipe?: string; selectedCheckKeys?: string[]; dbName?: string; runIndex?: string; + restartSpec?: RuntimeControlSpec; out?: string; parentAttemptId?: string; + mutationShardIndex?: number; mutationShardCount?: number; resumeFrom?: string; checkpointOut?: string; + baselineBundle?: string; expectedCalibrationIdentity?: JsonRecord; maxRuntimeMinutes?: number; + imageId?: string; mutationAttemptId?: string; expectedRecipeSha256?: string; + reseedOnReset?: boolean; + gradeTimeoutMs?: number; + recipeTask?: ReturnType['request']; +}; +type ParsedMutationArgs = MutationArgs & { + app: string; + url: string; + mutations: string; + level: string; + recipe: string; + maxRuntimeMinutes: number; + mutationAttemptId: string; +}; +type GradeReport = { total?: unknown; max?: unknown; + features?: Array<{ id?: unknown; score?: unknown; + criteria?: Array<{ id?: string; stableKey?: unknown; evidence?: unknown }> }>; + [key: string]: unknown }; +type MutationResult = ReturnType & { id: string; scenario: string; targets: string[] }; +type BaselineEntry = MutationCheckpointBaseline & { total: unknown; max: unknown }; +type MutationFile = { target: string; backup: string; original: string; edits: ReturnType }; + +export function mutationFailureMessage(error: unknown): string { + return redactCredentials(error instanceof AggregateError + ? `${error.message}: ${error.errors.map(mutationFailureMessage).join('; ')}` + : errorMessage(error)); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function restoreMutationSource(file: { target: string; backup: string; original: string }): void { + // Restart hands files to the app UID. copyFileSync can fail chmod and unlink + // its destination even when the controller can write through group access. + writeFileSync(file.target, file.original); + if (readFileSync(file.target, 'utf8') !== file.original) { + throw new Error(`restore verification failed for ${file.target}`); + } + unlinkSync(file.backup); +} + +function jsonObject(value: unknown, label: string): JsonRecord { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as JsonRecord; +} + +const HERE = dirname(fileURLToPath(import.meta.url)); +const GRADER = join(HERE, "grade.js"); + +export function parseMutationArgs(argv: readonly string[]): ParsedMutationArgs { + const { values } = parseNodeArgs({ args: [...argv.slice(2)], options: { + app: { type: 'string' }, url: { type: 'string' }, mutations: { type: 'string' }, + level: { type: 'string' }, spec: { type: 'string' }, backend: { type: 'string' }, + track: { type: 'string' }, recipe: { type: 'string' }, + 'selected-check': { type: 'string', multiple: true }, 'db-name': { type: 'string' }, + 'run-index': { type: 'string' }, + 'restart-spec': { type: 'string' }, out: { type: 'string' }, 'parent-attempt-id': { type: 'string' }, + 'mutation-shard-index': { type: 'string' }, 'mutation-shard-count': { type: 'string' }, + 'resume-from': { type: 'string' }, 'checkpoint-out': { type: 'string' }, + 'baseline-bundle': { type: 'string' }, 'expected-calibration-json': { type: 'string' }, + 'max-runtime-minutes': { type: 'string' }, 'image-id': { type: 'string' }, + } }); + const a: MutationArgs = { app: values.app, url: values.url, mutations: values.mutations, + level: values.level, spec: values.spec, backend: values.backend, track: values.track, + recipe: values.recipe, selectedCheckKeys: values['selected-check'], dbName: values['db-name'], + runIndex: values['run-index'] ?? '0', + restartSpec: values['restart-spec'] === undefined ? undefined + : parseRuntimeControlSpec(JSON.parse(values['restart-spec'])), + out: values.out, parentAttemptId: values['parent-attempt-id'], + mutationShardIndex: values['mutation-shard-index'] === undefined + ? undefined : Number(values['mutation-shard-index']), + mutationShardCount: values['mutation-shard-count'] === undefined + ? undefined : Number(values['mutation-shard-count']), + resumeFrom: values['resume-from'] && resolve(values['resume-from']), + checkpointOut: values['checkpoint-out'] && resolve(values['checkpoint-out']), + baselineBundle: values['baseline-bundle'] && resolve(values['baseline-bundle']), + expectedCalibrationIdentity: values['expected-calibration-json'] === undefined + ? undefined : JSON.parse(values['expected-calibration-json']) as JsonRecord, + maxRuntimeMinutes: values['max-runtime-minutes'] === undefined + ? 60 : Number(values['max-runtime-minutes']), + imageId: values['image-id'] }; + if (!a.app || !a.url || !a.mutations || !a.level || !a.recipe) { + throw new Error( + "Usage: node dist/grader/mutation-test.js --app --url --mutations " + + "--level --recipe ", + ); + } + let url: URL; + try { url = new URL(a.url); } + catch { throw new Error('--url must be a valid HTTP or HTTPS URL'); } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('--url must use HTTP or HTTPS'); + } + if (!Number.isInteger(Number(a.level)) || Number(a.level) < 1) { + throw new Error('--level must be a positive integer'); + } + const shardFields = [a.mutationShardIndex, a.mutationShardCount] + .filter(value => value !== undefined); + if (shardFields.length === 1) { + throw new Error('--mutation-shard-index and --mutation-shard-count must be supplied together'); + } + if (a.mutationShardCount !== undefined + && (!Number.isInteger(a.mutationShardIndex) || !Number.isInteger(a.mutationShardCount) + || a.mutationShardIndex! < 0 || a.mutationShardCount < 1 + || a.mutationShardIndex! >= a.mutationShardCount)) { + throw new Error('--mutation-shard-index must be within the positive shard count'); + } + a.maxRuntimeMinutes ??= 60; + if (!Number.isFinite(a.maxRuntimeMinutes) || a.maxRuntimeMinutes < 1 + || a.maxRuntimeMinutes > 120) { + throw new Error('--max-runtime-minutes must be from 1 through 120'); + } + if (a.resumeFrom && !a.checkpointOut) a.checkpointOut = a.resumeFrom; + return { ...a, app: a.app, url: a.url, mutations: a.mutations, level: a.level, recipe: a.recipe, + maxRuntimeMinutes: a.maxRuntimeMinutes, + mutationAttemptId: `mutation-${new Date().toISOString().replace(/[:.]/g, "-")}` }; +} + +class MutationBatchDeadlineError extends Error {} + +export function remainingMutationBatchMs(deadlineMs: number, nowMs: number = Date.now()): number { + const remaining = Math.floor(deadlineMs - nowMs); + if (remaining <= 0) throw new MutationBatchDeadlineError('mutation batch deadline reached'); + return remaining; +} + +// Startup owns application initialization, including migrations outside module init. +export async function resetMutationDatabase(a: MutationArgs, deadlineMs: number | null, + control = controlAppServer): Promise { + const exec: TextCommandExecutor = deadlineMs === null ? execFileSync : ((file, commandArgs, options) => + execFileSync(file, commandArgs, { ...options, + timeout: Math.min(options.timeout, remainingMutationBatchMs(deadlineMs)) })); + try { + const requiresReseed = STACK_ADAPTER_REGISTRY.get(a.backend!).reset.requiresReseed; + const restartSpec = a.reseedOnReset && requiresReseed ? a.restartSpec : undefined; + if (a.reseedOnReset && requiresReseed && !restartSpec) { + throw new Error(`track ${a.track} requires a lease-authenticated --restart-spec to reseed after reset`); + } + const signal = deadlineMs === null ? null + : AbortSignal.timeout(remainingMutationBatchMs(deadlineMs)); + if (restartSpec) await control(restartSpec, "stop", { signal, exec }); + await resetBackend({ backend: a.backend!, app: a.app!, exec }); + if (restartSpec) await control(restartSpec, "start", { signal, exec }); + } catch (error) { + if (deadlineMs !== null && Date.now() >= deadlineMs) { + throw new MutationBatchDeadlineError('mutation batch deadline reached', { cause: error }); + } + throw error; + } +} + +export function mutationClientCommand(backend: string, command: string, args: readonly string[], + timeout: number, exec: TextCommandExecutor = execFileSync): string { + const { lease } = leaseFromEnv(process.env, { backend, active: true }); + const container = inspectBuildContainer(lease, exec); + return exec('docker', ['exec', ...codingContainerAgentExecOptions(), + '-w', `${CODING_CONTAINER_APP_ROOT}/client`, container.id, + ...codingContainerAgentCommand(command, args)], + { encoding: 'utf8', stdio: 'pipe', timeout }); +} + +function rebuildClientAfterSourceChange(a: MutationArgs, deadlineMs: number): void { + const timeout = deadlineMs - Date.now(); + if (timeout <= 0) throw new MutationBatchDeadlineError(); + try { + mutationClientCommand(a.backend!, 'npm', ['run', 'build'], timeout); + } catch (cause) { + if (Date.now() >= deadlineMs) throw new MutationBatchDeadlineError(); + throw new Error(`client build failed after source change: ${mutationFailureMessage(cause)}`, { cause }); + } +} + +export function mutationGradeArguments(a: MutationArgs, reportPath: string): string[] { + const gradeArgs: string[] = [ + GRADER, + "--url", + a.url!, + "--level", + a.level!, + "--out", + reportPath, + "--spec", + a.spec!, + "--backend", + a.backend!, + "--track", + a.track!, + "--app", + a.app!, + ]; + if (a.dbName) gradeArgs.push("--db-name", a.dbName); + if (a.restartSpec) gradeArgs.push("--restart-spec", JSON.stringify(a.restartSpec)); + if (a.mutationAttemptId) gradeArgs.push("--parent-attempt-id", a.mutationAttemptId); + if (a.recipe) gradeArgs.push("--recipe", a.recipe); + if (a.recipeTask) gradeArgs.push('--recipe-task-json', JSON.stringify(a.recipeTask)); + if (a.expectedRecipeSha256) { + gradeArgs.push("--expected-recipe-sha256", a.expectedRecipeSha256); + } + for (const stableKey of a.selectedCheckKeys ?? []) { + gradeArgs.push("--selected-check", stableKey); + } + if (a.selectedCheckKeys?.length) { + gradeArgs.push("--selection-sha256", sha256(JSON.stringify([...a.selectedCheckKeys].sort()))); + } + return gradeArgs; +} + +async function grade(a: MutationArgs, reportPath: string, deadlineMs: number | null = null): Promise { + await resetMutationDatabase(a, deadlineMs); + if (existsSync(reportPath)) unlinkSync(reportPath); + const gradeArgs = mutationGradeArguments(a, reportPath); + const sourceTimeout = a.gradeTimeoutMs ?? GRADER_SOURCE_TIMEOUT_MS; + const timeout = deadlineMs === null + ? sourceTimeout + : mutationGradeTimeoutMs(deadlineMs, Date.now(), sourceTimeout); + if (timeout === 0) throw new MutationBatchDeadlineError('mutation batch deadline reached'); + try { + execFileSync(process.execPath, gradeArgs, { + stdio: "pipe", + encoding: "utf8", + timeout, + }); + } catch (error) { + if (jsonObject(error, 'grader process error').code === 'ETIMEDOUT' && timeout < sourceTimeout) { + throw new MutationBatchDeadlineError('mutation grade reached the remaining batch deadline'); + } + throw error; + } + if (!existsSync(reportPath)) { + throw new Error("grader completed without producing its report"); + } + return readArtifactPayload(reportPath, { expectedKind: "grade" }); +} + +interface MutationGradeReceipt { + scenario: string; + mutationId: string | null; + status: 'running' | 'returned' | 'threw'; + report: { path: string; sha256: string | null }; +} + +// Reports stay private beside their control artifact, including failed and retried invocations. +export async function retainMutationGrade(outputPath: string, + context: Pick, + invoke: (path: string) => Promise, record: (receipt: MutationGradeReceipt) => void): Promise { + const directory = `${outputPath}.grades`; + mkdirSync(directory, { recursive: true }); + const path = join(mkdtempSync(join(directory, 'invocation-')), 'grade.json'); + const receipt: MutationGradeReceipt = { ...context, status: 'running', report: { + path: relative(dirname(outputPath), path).replaceAll('\\', '/'), sha256: null, + } }; + record(receipt); + try { + const result = await invoke(path); + receipt.status = 'returned'; + return result; + } catch (error) { + receipt.status = 'threw'; + throw error; + } finally { + if (existsSync(path)) receipt.report.sha256 = sha256(readFileSync(path)); + record(receipt); + } +} + +let args: ParsedMutationArgs; +let startedAt: number; +let startedIso: string; +const artifactPath = (id: string) => + resolve(args.out ?? join(HERE, "..", "results", `${id}.json`)); +let spec!: MutationSpec; +const gradeReports: MutationGradeReceipt[] = []; +let priorMutationControl: { path: string; sha256: string } | null = null; +let baselineBundleEvidence: { path: string; sha256: string } | null = null; +let currentControlArtifact: (() => Record) | null = null; + +function sha256(value: string | Buffer): string { + return createHash('sha256').update(value).digest('hex'); +} + +function scenarioKey(path: string): string { + return relative(resolve(HERE, '..'), path).replaceAll('\\', '/'); +} + +function checkpointGroup(path: string, mutations: MutationDefinition[], selectedCheckKeys: readonly string[]): + MutationCheckpointIdentity['groups'][number] { + const scenario = scenarioKey(path); + const scenarioSha256 = sha256(readFileSync(path)); + const mutationSha256 = sha256(JSON.stringify(mutations)); + const selectionSha256 = sha256(JSON.stringify([...selectedCheckKeys].sort())); + return { scenario, scenarioSha256, mutationSha256, selectionSha256, + identitySha256: sha256(JSON.stringify({ scenarioSha256, mutationSha256, selectionSha256 })), + mutationIds: mutations.map(mutation => mutation.id as string) }; +} + +function checkpointIdentity(groups: MutationCheckpointIdentity['groups'], shard: { index: number; count: number; + mutationIds: string[] }, track: ReturnType): MutationCheckpointIdentity { + return { + schemaVersion: 1, + engineSha256: currentEngineIdentity().sha256, + recipeSha256: args.expectedRecipeSha256, + fixtureSha256: spec.fixtureSha256, + calibrationSha256: args.expectedCalibrationIdentity?.sha256 ?? null, + imageId: args.imageId ?? null, + backend: args.backend, + track: args.track, + level: Number(args.level), + trackSha256: sha256(readFileSync(join(track.dir, TRACK_MANIFEST_FILE))), + shard: { index: shard.index, count: shard.count, mutationIds: shard.mutationIds }, + groups, + }; +} + +function resumableEvidence(path: string | undefined, identity: MutationCheckpointIdentity): + { results: MutationCheckpointResult[]; baselines: MutationCheckpointBaseline[] } { + if (!path || !existsSync(path)) return { results: [], baselines: [] }; + const prior = readArtifactPayload(path, { expectedKind: 'mutation_control' }); + const { results, baselines } = reusableMutationEvidence(prior, identity); + const shard = identity.shard as { mutationIds: string[] }; + console.log(`Resuming ${results.length}/${shard.mutationIds.length} completed mutations from ${path}`); + return { results, baselines }; +} + +export function mutationHarnessFailureArtifact(current: Record, reason: string, + completedAt: string): Record { + return { ...current, completedAt, ok: false, + outcome: { kind: 'harness_failure', phase: 'mutation-control', reason } }; +} + +function recordHarnessFailure(error: unknown): void { + const generatedAt = new Date().toISOString(); + const id = args.mutationAttemptId; + const artifact = { + id, + kind: "mutation_control", + startedAt: startedIso, + completedAt: generatedAt, + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + fixture: spec?.fixtureSha256 ? { id: "source-under-mutation", sha256: spec.fixtureSha256 } : null, + stackAdapter: (args.backend ?? spec?.backend) ? { id: args.backend ?? spec.backend } : null, + }), + durationMs: Date.now() - startedAt, + app: resolve(args.app), + mutations: resolve(args.mutations), + fixtureSha256: spec?.fixtureSha256 ?? null, + spec: args.spec ? resolve(args.spec) : null, + backend: args.backend ?? spec?.backend ?? null, + track: args.track ?? spec?.track ?? null, + ok: false, + gradeReports, + priorMutationControl, + baseline: { sourceBundle: baselineBundleEvidence }, + outcome: { + kind: "harness_failure", + phase: "mutation-control", + reason: mutationFailureMessage(error), + }, + }; + try { + const outputPath = artifactPath(id); + writeRunJson(outputPath, mutationHarnessFailureArtifact( + currentControlArtifact?.() ?? artifact, mutationFailureMessage(error), generatedAt)); + console.error( + `mutation harness failure: ${mutationFailureMessage(error)}\nartifact: ${outputPath}`, + ); + } catch (artifactError) { + console.error( + `mutation harness failure: ${mutationFailureMessage(error)}\nfailed to write failure artifact: ${errorMessage(artifactError)}`, + ); + } + process.exitCode = 2; +} + +async function main(): Promise { + spec = readMutationManifest(args.mutations!); + const fullMutations = spec.mutations; + const shard = args.mutationShardCount === undefined + ? { index: 0, count: 1, mutationIds: fullMutations.map(mutation => mutation.id as string), + mutations: fullMutations } + : mutationShard(fullMutations, + { index: args.mutationShardIndex!, count: args.mutationShardCount, + defaultScenario: spec.scenario }); + if (shard.mutations.length === 0) throw new Error('mutation shard has no assigned mutations'); + spec.mutations = shard.mutations; + if (args.backend && args.backend !== spec.backend) { + throw new Error( + `--backend conflicts with manifest backend ${spec.backend}`, + ); + } + if (args.track && args.track !== spec.track) { + throw new Error(`--track conflicts with manifest track ${spec.track}`); + } + args.backend = spec.backend; + args.track = spec.track; + const track = loadTrack(args.track); + const binding = resolveRecipeRelease(track, Number(args.level), args.recipe); + if (!binding) throw new Error(`${args.track} L${args.level} has no recipe release`); + args.recipe = binding.release.id; + args.expectedRecipeSha256 = binding.release.contentSha256; + args.recipeTask = createBoundRecipeTaskRequest(binding, { + taskMode: binding.plan.recipe.task.mode === 'action' ? 'fresh' : undefined, + }).request; + const recipeRelease = binding.release; + args.dbName ??= dbName(track, Number(args.runIndex)); + args.reseedOnReset = track.reseedOnReset; + const definitions = validateMutationDefinitions(spec.mutations, + { defaultScenario: spec.scenario, requireScenario: true }); + if (!definitions.ok) { + throw new Error( + `invalid mutation manifest: ${ + definitions.issues.map((issue) => + `${issue.mutation ?? ""}:${issue.kind}` + ).join(", ") + }`, + ); + } + const groups = new Map(); + for (const [scenario, mutations] of groupMutationsByScenario(spec)) { + const declaredSpec = resolveMutationScenarioPath(scenario); + groups.set(declaredSpec, mutations); + } + if (args.spec) { + const requested = resolve(args.spec); + if (groups.size !== 1 || !groups.has(requested)) { + throw new Error('--spec conflicts with the mutation manifest scenario selection'); + } + } + // Hosted apps serve this build; development servers compile client source on demand. + const clientDist = `${CODING_CONTAINER_APP_ROOT}/client/dist`; + const cleanClientDist = spec.mutations.some(mutation => + mutationFileEdits(mutation).some(edit => edit.file.replaceAll('\\', '/').startsWith('client/'))) + && existsSync(join(args.app, 'client', 'dist')) ? `/tmp/stack-bench-mutation-${randomUUID()}-client-dist` : null; + if (cleanClientDist) mutationClientCommand(args.backend!, 'cp', + ['-R', '--', clientDist, cleanClientDist], 120_000); + + // Reject backups left by an interrupted run before grading the baseline. + for (const m of spec.mutations) { + for (const file of new Set(mutationFileEdits(m).map(edit => edit.file))) { + const stale = resolveMutationFile(args.app, file) + ".mutation-backup"; + if (existsSync(stale)) { + throw new Error( + `${stale} exists; restore the interrupted mutation backup before running again`, + ); + } + } + } + + // Catch dirty source even when no backup file remains. + assertAppSourceIdentity(args.app, spec.fixtureSha256, 'mutation fixture'); + + // Reject missing or ambiguous edit anchors before baseline grading. + for (const m of spec.mutations) { + for (const edit of mutationFileEdits(m)) { + const source = readFileSync(resolveMutationFile(args.app, edit.file), "utf8"); + const matches = source.split(edit.find).length - 1; + if (matches !== 1) { + throw new Error( + `${m.id} anchor matched ${matches} times in ${edit.file}; expected exactly once`, + ); + } + } + } + + const plans = [...groups].map(([scenarioPath, mutations]) => { + const selectedCheckKeys = releaseScenarioCheckKeys(recipeRelease, track.dir, scenarioPath, + args.selectedCheckKeys ?? null); + return { scenarioPath, scenario: scenarioKey(scenarioPath), mutations, selectedCheckKeys, + checkpoint: checkpointGroup(scenarioPath, mutations, selectedCheckKeys) }; + }); + const cleanBaselineBundle = args.baselineBundle + ? readArtifactPayload(args.baselineBundle, { expectedKind: 'grade_bundle' }) + : null; + if (cleanBaselineBundle && !args.expectedCalibrationIdentity) { + throw new Error('a reusable clean baseline requires its expected calibration identity'); + } + const checkpoint = checkpointIdentity(plans.map(plan => plan.checkpoint), shard, track); + const resumed = resumableEvidence(args.resumeFrom, checkpoint); + const results: MutationResult[] = [...resumed.results] as MutationResult[]; + const baselines: BaselineEntry[] = [...resumed.baselines] as BaselineEntry[]; + const completedIds = new Set(results.map(result => result.id)); + if (completedIds.size !== results.length) { + throw new Error('mutation checkpoint contains duplicate results'); + } + const outputPath = artifactPath(args.mutationAttemptId); + if (args.baselineBundle) baselineBundleEvidence = { + path: relative(dirname(outputPath), resolve(args.baselineBundle)).replaceAll('\\', '/'), + sha256: sha256(readFileSync(args.baselineBundle)), + }; + if (args.resumeFrom) { + let priorPath = resolve(args.resumeFrom); + if (priorPath === outputPath || (args.checkpointOut && priorPath === resolve(args.checkpointOut))) { + const retainedPath = `${priorPath}.prior-${randomUUID()}.json`; + copyFileSync(priorPath, retainedPath); + priorPath = retainedPath; + } + priorMutationControl = { path: relative(dirname(outputPath), priorPath).replaceAll('\\', '/'), + sha256: sha256(readFileSync(priorPath)) }; + } + const deadline = startedAt + args.maxRuntimeMinutes * 60_000; + + const createControlArtifact = (status: 'running' | 'incomplete' | 'complete', reason: string | null = null) => { + const ordered = [...results].sort((left, right) => + shard.mutationIds.indexOf(left.id) - shard.mutationIds.indexOf(right.id)); + const clean = ordered.filter(result => result.status === 'CAUGHT'); + const orderedBaselines = plans.map(plan => baselines.find(entry => + entry.scenario === plan.scenario)).filter((entry): entry is BaselineEntry => Boolean(entry)); + const remaining = shard.mutationIds.filter(id => !completedIds.has(id)); + return { + id: args.mutationAttemptId, + kind: 'mutation_control', + startedAt: startedIso, + completedAt: status === 'running' ? null : new Date().toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + fixture: { id: 'source-under-mutation', sha256: spec.fixtureSha256 }, + recipe: { id: recipeRelease.id, sha256: recipeRelease.contentSha256 }, + stackAdapter: { id: args.backend }, + }), + durationMs: Date.now() - startedAt, + app: resolve(args.app), + mutations: resolve(args.mutations), + fixtureSha256: spec.fixtureSha256, + spec: plans.map(plan => plan.scenario), + backend: args.backend, + track: args.track, + shard: { index: shard.index, count: shard.count, mutationIds: shard.mutationIds }, + baseline: { + sourceBundle: baselineBundleEvidence, + total: orderedBaselines.reduce((sum, entry) => sum + Number(entry.total), 0), + max: orderedBaselines.reduce((sum, entry) => sum + Number(entry.max), 0), + scenarios: orderedBaselines, + }, + ok: status === 'complete' && clean.length === ordered.length + && ordered.length === shard.mutationIds.length, + ...(status === 'complete' ? {} : { outcome: { kind: 'incomplete', + phase: 'mutation-control', reason: reason ?? 'mutation batch is in progress' } }), + summary: { caught: clean.length, completed: ordered.length, + total: shard.mutationIds.length, remaining: remaining.length }, + results: ordered, + gradeReports, + priorMutationControl, + checkpoint: { ...checkpoint, status, maxRuntimeMinutes: args.maxRuntimeMinutes, + updatedAt: new Date().toISOString() }, + }; + }; + currentControlArtifact = () => createControlArtifact('incomplete'); + + const persist = (status: 'running' | 'incomplete' | 'complete', reason: string | null = null) => { + assertAppSourceIdentity(args.app, spec.fixtureSha256, + 'mutation fixture before checkpoint'); + const artifact = createControlArtifact(status, reason); + writeRunJson(outputPath, artifact); + if (args.checkpointOut && resolve(args.checkpointOut) !== outputPath) { + const rebase = (ref: { path: string; sha256: string | null }) => ({ ...ref, + path: relative(dirname(resolve(args.checkpointOut!)), resolve(dirname(outputPath), ref.path)).replaceAll('\\', '/') }); + writeRunJson(args.checkpointOut, { ...artifact, + baseline: { ...artifact.baseline, sourceBundle: baselineBundleEvidence ? rebase(baselineBundleEvidence) : null }, + gradeReports: gradeReports.map(receipt => ({ ...receipt, report: rebase(receipt.report) })), + priorMutationControl: priorMutationControl ? rebase(priorMutationControl) : null }); + } + return artifact; + }; + const invokeGrade = (mutationId: string | null) => { + const index = gradeReports.length; + return retainMutationGrade(outputPath, { scenario: scenarioKey(args.spec!), mutationId }, + path => grade(args, path, deadline), receipt => { + gradeReports[index] = receipt; + // During a mutant invocation source differs by design; this is a running receipt, not acceptance. + writeRunJson(outputPath, createControlArtifact('running')); + }); + }; + const stopAtBudget = () => { + const artifact = persist('incomplete', + `mutation batch reached its ${args.maxRuntimeMinutes} minute limit`); + console.log(`\n${artifact.summary.completed}/${artifact.summary.total} mutations completed; ` + + `${artifact.summary.remaining} remain`); + console.log(`checkpoint: ${args.checkpointOut ?? outputPath}`); + process.exitCode = 3; + }; + + for (const plan of plans) { + const { scenarioPath, scenario, mutations, selectedCheckKeys } = plan; + const pending = mutations.filter((mutation: MutationDefinition) => !completedIds.has(mutation.id as string)); + if (pending.length === 0) continue; + if (Date.now() >= deadline) return stopAtBudget(); + args.spec = scenarioPath; + args.selectedCheckKeys = selectedCheckKeys; + args.gradeTimeoutMs = gradingSourceTimeoutMs(binding.plan.packs, + recipeRelease.checkCatalog.filter(check => selectedCheckKeys.includes(check.stableKey))); + let baseline; + if (cleanBaselineBundle) { + const reused = reusableMutationBaseline(cleanBaselineBundle, { + backend: args.backend, + track: args.track, + level: Number(args.level), + fixtureSha256: spec.fixtureSha256, + recipe: { id: recipeRelease.id, sha256: recipeRelease.contentSha256 }, + identities: { + engine: currentEngineIdentity(), + calibration: args.expectedCalibrationIdentity, + stackAdapter: { id: args.backend }, + }, + selectedCheckKeys, + }); + if (!reused.ok) { + throw new Error(`cannot reuse clean baseline for ${scenarioPath}: ${reused.reason}`); + } + baseline = reused.report; + console.log(`Baseline (verified clean evidence, ${scenarioPath})...`); + } else { + console.log(`Baseline (unmutated app, ${scenarioPath})...`); + try { + baseline = await invokeGrade(null); + } catch (error) { + if (error instanceof MutationBatchDeadlineError) return stopAtBudget(); + throw error; + } + const validation = validateMutationBaseline(baseline, mutations); + if (!validation.ok && isRetryableMutationBaseline(validation.issues)) { + console.log(' transient baseline failure; retrying once'); + try { + baseline = await invokeGrade(null); + } catch (error) { + if (error instanceof MutationBatchDeadlineError) return stopAtBudget(); + throw error; + } + } + } + const baselineValidation = validateMutationBaseline(baseline, mutations); + if (!baselineValidation.ok) { + throw new Error( + `reference baseline is not known-good for ${scenarioPath}: ${ + JSON.stringify(baselineValidation.issues) + }`, + ); + } + console.log( + ` baseline: ${baseline.total}/${baseline.max} ${ + (baseline.features ?? []).map((f) => `F${f.id}:${(f as NonNullable[number]).score}`).join(" ") + }\n`, + ); + const baselineEntry = { scenario, identitySha256: plan.checkpoint.identitySha256, + total: baseline.total, max: baseline.max }; + const priorBaseline = baselines.findIndex(entry => entry.scenario === scenario); + if (priorBaseline === -1) baselines.push(baselineEntry); + else baselines[priorBaseline] = baselineEntry; + + for (const m of pending) { + if (Date.now() >= deadline) return stopAtBudget(); + const byFile = new Map(); + for (const edit of mutationFileEdits(m)) { + const target = resolveMutationFile(args.app, edit.file); + if (!byFile.has(target)) { + byFile.set(target, { + target, + backup: `${target}.mutation-backup`, + original: readFileSync(target, "utf8"), + edits: [], + }); + } + byFile.get(target)!.edits.push(edit); + } + const files = [...byFile.values()]; + const clientChanged = files.some(file => relative(args.app!, file.target) + .split(sep)[0] === 'client'); + const backedUp: MutationFile[] = []; + let r: GradeReport | undefined; + let classified: ReturnType | undefined; + let deadlineReached = false; + let mutationError: unknown = null; + try { + for (const file of files) { + copyFileSync(file.target, file.backup); + backedUp.push(file); + } + for (const file of files) { + writeFileSync(file.target, + file.edits.reduce((src, edit) => src.replace(edit.find, edit.replace), + file.original)); + } + if (clientChanged) rebuildClientAfterSourceChange(args, deadline); + r = await invokeGrade(m.id as string); + classified = classifyMutationResult( + baseline as Parameters[0], + r as Parameters[1], + m, + ); + if (isRetryableMutationResult(classified.status)) { + console.log(` ${classified.status} result; retrying once`); + r = await invokeGrade(m.id as string); + classified = classifyMutationResult( + baseline as Parameters[0], + r as Parameters[1], + m, + ); + } + } catch (error) { + if (error instanceof MutationBatchDeadlineError) deadlineReached = true; + else mutationError = error; + } + + const cleanupErrors: Error[] = []; + for (const file of backedUp) { + try { + restoreMutationSource(file); + } catch (error) { + cleanupErrors.push(new Error(`cannot restore ${file.target}: ${errorMessage(error)}`, + { cause: error })); + } + } + for (const file of files) { + try { + if (existsSync(file.backup) || readFileSync(file.target, 'utf8') !== file.original) { + cleanupErrors.push(new Error(`restore verification failed for ${file.target}`)); + } + } catch (error) { + cleanupErrors.push(new Error(`cannot verify restored source ${file.target}: ${errorMessage(error)}`, + { cause: error })); + } + } + if (clientChanged) { + try { + mutationClientCommand(args.backend!, 'rm', ['-rf', '--', clientDist], 120_000); + if (cleanClientDist) mutationClientCommand(args.backend!, 'cp', + ['-R', '--', cleanClientDist, clientDist], 120_000); + } catch (error) { + cleanupErrors.push(new Error(`cannot restore the built client: ${errorMessage(error)}`, + { cause: error })); + } + } + // A normal error leaves enough budget to restore the clean runtime. A + // deadline stop leaves clean source and lets the lease owner stop it. + if (mutationError !== null && cleanupErrors.length === 0) { + try { + await resetMutationDatabase(args, deadline); + } catch (error) { + cleanupErrors.push(new Error(`cannot restore the clean runtime: ${errorMessage(error)}`, + { cause: error })); + } + } + if (cleanupErrors.length > 0) { + const errors = mutationError === null ? cleanupErrors : [mutationError, ...cleanupErrors]; + throw new AggregateError(errors, + 'mutation cleanup failed; do not reuse this app source'); + } + if (mutationError !== null) throw mutationError; + if (deadlineReached) return stopAtBudget(); + if (!r || !classified) throw new Error('mutation grade completed without a result'); + results.push({ id: m.id, scenario, + targets: mutationTargetKeys(m), ...classified }); + completedIds.add(m.id); + persist('running'); + console.log( + `${classified.status.padEnd(20)} ${m.id} — expected ${ + classified.targetKeys.join(", ") + }`, + ); + if (classified.regressions.length) { + console.log( + ` failed criteria: ${ + classified.regressions.map((item) => item.key).join(", ") + }`, + ); + } + if (classified.targetOffAssertion.length) { + console.log(` stopped before an observation: ${classified.targetOffAssertion + .map(item => `${item.key} at ${item.action ?? 'no recorded action'}`).join(', ')}`); + } + } + } + + // Detect any source change outside the files restored above. + assertAppSourceIdentity(args.app, spec.fixtureSha256, 'mutation fixture after worker completion'); + // Restore the clean runtime and database before releasing the worker lease. + await resetMutationDatabase(args, null); + + if (cleanClientDist) mutationClientCommand(args.backend!, 'rm', ['-rf', '--', cleanClientDist], 120_000); + const artifact = persist('complete'); + console.log(`\n${artifact.summary.caught}/${artifact.summary.total} mutations cleanly caught`); + console.log(`artifact: ${outputPath}`); + if (!artifact.ok) process.exitCode = 1; +} + +function run(): void { + try { + args = parseMutationArgs(process.argv); + } catch (error) { + console.error(errorMessage(error)); + process.exitCode = 2; + return; + } + startedAt = Date.now(); + startedIso = new Date(startedAt).toISOString(); + main().catch(recordHarnessFailure); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) run(); diff --git a/tools/stack-bench/grader/mutations/convex-ecommerce.json b/tools/stack-bench/grader/mutations/convex-ecommerce.json new file mode 100644 index 00000000000..052ac031011 --- /dev/null +++ b/tools/stack-bench/grader/mutations/convex-ecommerce.json @@ -0,0 +1,2344 @@ +{ + "schemaVersion": 3, + "fixtureSha256": "6f0c177b439989afa30b335bcc4894238c69e2e279b9c00097b5280391954dfe", + "backend": "convex", + "track": "ecommerce", + "note": "Native Convex selected-L3 defect controls. Authored coverage is not qualification; every baseline and mutant requires execution against this exact source hash. Recovery controls use real native reconnect callbacks and writes.", + "mutations": [ + { + "id": "signup-does-not-expose-created-account", + "scenario": "tracks/ecommerce/scenarios/01-account-create.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Authenticated account views omit the created username, including native subscription updates.", + "file": "convex/accounts.js", + "edits": [ + { + "find": "id: a._id, username: a.username, roles: a.roles", + "replace": "id: a._id, username: '', roles: a.roles" + } + ] + }, + { + "id": "session-token-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-account-reload.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "Fresh sign-in works, but each new document discards the stored access token before initializing native authentication, so reload cannot restore the session.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "export const TOKEN_KEY = 'convex_shop_token';", + "replace": "export const TOKEN_KEY = 'convex_shop_token';\nlocalStorage.removeItem(TOKEN_KEY);" + } + ] + }, + { + "id": "reload-hydrates-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Existing-token page loads discard cart hydration from the native subscription; the first signup page and its working cart are unchanged.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "const CATALOG_PAGE_SIZE = 10;", + "replace": "const CATALOG_PAGE_SIZE = 10;\nconst discardHydratedCart = Boolean(localStorage.getItem(TOKEN_KEY));" + }, + { + "find": " setCart(state.cart);", + "replace": " setCart(discardHydratedCart ? { items: [], total: 0 } : state.cart);" + } + ] + }, + { + "id": "warehouse-view-omits-one-location", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "The admin warehouse projection truncates the final item-location row.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {overview.locations.map((loc) => (", + "replace": " {overview.locations.slice(0, -1).map((loc) => (" + } + ] + }, + { + "id": "cart-hydration-loses-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "Existing-token page loads discard cart hydration from the native subscription; the first signup page and its working cart are unchanged.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "const CATALOG_PAGE_SIZE = 10;", + "replace": "const CATALOG_PAGE_SIZE = 10;\nconst discardHydratedCart = Boolean(localStorage.getItem(TOKEN_KEY));" + }, + { + "find": " setCart(state.cart);", + "replace": " setCart(discardHydratedCart ? { items: [], total: 0 } : state.cart);" + } + ] + }, + { + "id": "purchased-review-ui-does-not-submit", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b" + ], + "desc": "The review form discards valid customer submissions. Direct authorized and unauthorized review calls retain their normal behavior.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"\");", + "replace": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"Review submission unavailable\");\n return;" + } + ] + }, + { + "id": "progression-customer-sees-fulfilment-content", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-access.json", + "targets": [ + "ecommerce.spec.access-control.fulfilment-area-boundary.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {(currentUser?.isStaff || currentUser?.isAdmin) && (\n ", + "replace": " setPreference(value => ({ ...value, order: !value.order }))} />" + }, + { + "find": "function SupportTicket({ ticket, user, act }: any) {", + "replace": "function OrderNotificationToggle({ on, onToggle }: { on: boolean; onToggle: () => void }) {\n const [shown] = useState(on); // mutant: the toggle keeps the state loaded when the card opened\n return ;\n}\n\nfunction SupportTicket({ ticket, user, act }: any) {" + } + ] + }, + { + "id": "role-editor-snaps-back-to-stored-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.progression.staff-roles.staff-roles.621c" + ], + "desc": "Pressing Save persists the role but resets the dropdown to the role that was stored before the save, so the assignment is not visible until a reload.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " ", + "replace": " " + } + ] + }, + { + "id": "review-script-unsafe-render", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Render stored review text as HTML so it executes in another customer session.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "
{r.comment}
", + "replace": "
" + } + ] + }, + { + "id": "shipping-counts-sale-twice", + "scenario": "tracks/ecommerce/scenarios/progression-shipping-accounting.json", + "targets": [ + "ecommerce.inventory-operations.shipping-accounting.202e" + ], + "desc": "Shipping doubles the authoritative booked order total.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { status: 'shipped' });", + "replace": "await ctx.db.patch(orderId, { status: 'shipped', total: order.total * 2 });" + } + ] + }, + { + "id": "checkout-retains-purchased-cart", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout does not consume its cart; concurrent calls create duplicate effects.", + "file": "convex/shop.js", + "edits": [ + { + "find": "for (const line of lines) await ctx.db.delete(line._id);", + "replace": "// Defect: the completed cart remains available for another checkout." + } + ] + }, + { + "id": "checkout-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Reject every checkout, including the positive control; refusal alone is insufficient.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!lines.length) throw new ConvexError('Cart is empty');", + "replace": "throw new ConvexError('Checkout rejected');" + } + ] + }, + { + "id": "purchases-do-not-decrement-stock", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Record purchases without decrementing stock, causing unchanged inventory, excess orders and excess revenue in the same purchase burst.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(row._id, { quantity: row.quantity - take });", + "replace": "await ctx.db.patch(row._id, { quantity: row.quantity });" + } + ] + }, + { + "id": "purchase-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Reject every purchase, leaving unchanged stock, zero successful orders and zero revenue in the required purchase burst.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const orderId = await ctx.db.insert('order_header',", + "replace": "throw new ConvexError('Purchases rejected');\n const orderId = await ctx.db.insert('order_header'," + } + ] + }, + { + "id": "order-history-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Order history exposes every customer's orders instead of only the signed-in customer's orders.", + "file": "convex/shop.js", + "edits": [ + { + "find": "orders: orderViews.filter(o => ownOrderIds.has(o.id)),", + "replace": "orders: orderViews," + } + ] + }, + { + "id": "profile-read-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.access-control.customer-profile-privacy.620b" + ], + "desc": "Profile reads expose another account saved address.", + "file": "convex/progression.js", + "edits": [ + { + "find": "profile: user?.profile || null", + "replace": "profile: (await rows(ctx, 'order_account')).find(a => a.profile.address)?.profile || null" + } + ] + }, + { + "id": "support-read-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "All customers and signed-out clients receive private support history.", + "file": "convex/progression.js", + "edits": [ + { + "find": ".filter(t => staff || (user && t.accountId === user._id))", + "replace": ".filter(() => true)" + } + ] + }, + { + "id": "support-reply-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-privacy.json", + "targets": [ + "ecommerce.spec.access-control.managed-support-privacy.613b" + ], + "desc": "A customer can reply to another customer support case.", + "file": "convex/progression.js", + "edits": [ + { + "find": "if (!ticket || (!isStaff(user) && ticket.accountId !== user._id)) throw new ConvexError('Ticket not found');", + "replace": "if (!ticket) throw new ConvexError('Ticket not found');" + } + ] + }, + { + "id": "preference-read-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.access-control.notification-preferences-privacy.630b" + ], + "desc": "Notification settings expose another customer saved preference.", + "file": "convex/progression.js", + "edits": [ + { + "find": "preference: user?.preference || { order: false, stock: false }", + "replace": "preference: (await rows(ctx, 'order_account')).find(a => a.preference.order || a.preference.stock)?.preference || { order: false, stock: false }" + } + ] + }, + { + "id": "staff-admin-access-survives-role-removal", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Role reassignment retains the previous administrator privilege.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(accountId, { roles: [role] });", + "replace": "await ctx.db.patch(accountId, { roles: [role, ...(await ctx.db.get(accountId)).roles.filter(previous => previous === 'admin' && previous !== role)] });" + } + ] + }, + { + "id": "review-eligibility-not-enforced", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Native review operation permits an account that never purchased the item.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!purchased) throw new ConvexError('Purchase this item before reviewing');", + "replace": "// Defect: non-buyers may submit a review." + } + ] + }, + { + "id": "review-script-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Reject every review, including benign text; rejecting all input is not safe rendering.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!purchased) throw new ConvexError('Purchase this item before reviewing');", + "replace": "throw new ConvexError('Reviews rejected');" + } + ] + }, + { + "id": "duplicate-review-accepted", + "scenario": "tracks/ecommerce/scenarios/01-review-uniqueness.json", + "targets": [ + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "desc": "Same purchaser can record multiple reviews for one item.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (await ctx.db.query('review').withIndex('author', q => q.eq('itemId', args.itemId).eq('accountId', user._id)).unique()) throw new ConvexError('Already reviewed');", + "replace": "// Defect: duplicate author reviews are accepted." + } + ] + }, + { + "id": "external-stock-client-snapshot-open", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "Keep the first subscribed catalog values instead of accepting later native stock changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => current.length ? current : state.items);" + } + ] + }, + { + "id": "external-stock-client-snapshot-restart", + "scenario": "tracks/ecommerce/scenarios/01-external-server-restart-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901c" + ], + "desc": "Keep the first subscribed catalog values instead of accepting later native stock changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => current.length ? current : state.items);" + } + ] + }, + { + "id": "external-stock-client-snapshot-reconnect", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "Keep the first subscribed catalog values instead of accepting later native stock changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => current.length ? current : state.items);" + } + ] + }, + { + "id": "duplicate-signup-reports-success", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "Duplicate signup authenticates the existing account without its password.", + "file": "convex/auth.js", + "edits": [ + { + "find": "const { user } = await createAccount(ctx, {", + "replace": "const existing = await retrieveAccount(ctx, { provider: 'password', account: { id: params.username } }).catch(() => null);\n const { user } = existing ?? await createAccount(ctx, {" + }, + { + "find": "if (user.registrationNonce !== registrationNonce) throw new ConvexError('Username already exists');", + "replace": "// Defect: creating an existing account grants a session." + } + ] + }, + { + "id": "signin-skips-password-verification", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Signin accepts an existing account without requiring its password to match.", + "file": "convex/auth.js", + "edits": [ + { + "find": "verifySecret: (password, hash) => new Scrypt().verify(hash, password),", + "replace": "verifySecret: async () => true," + } + ] + }, + { + "id": "signout-keeps-current-account", + "scenario": "tracks/ecommerce/scenarios/01-account-signout.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Signout disconnects the token state but leaves the current account and persisted credential in place.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " await signOut();\n clearSession();", + "replace": " // Defect: sign-out leaves the authenticated account intact." + } + ] + }, + { + "id": "cart-repeat-does-not-increment", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4a" + ], + "desc": "An existing cart line remains at its old quantity.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": "const next = add ? amount : amount;" + } + ] + }, + { + "id": "checkout-leaves-cart-claimed", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Checkout creates an order without emptying the cart.", + "file": "convex/shop.js", + "edits": [ + { + "find": "for (const line of lines) await ctx.db.delete(line._id);", + "replace": "// Defect: checked-out cart lines survive." + } + ] + }, + { + "id": "catalog-price-is-offset", + "scenario": "tracks/ecommerce/scenarios/01-catalog-values.json", + "targets": [ + "ecommerce.feature.catalog.catalog-values.2a" + ], + "desc": "Public catalog prices are reported one unit too high.", + "file": "convex/seed.js", + "edits": [ + { + "find": "name: entry.name, price: entry.price,", + "replace": "name: entry.name, price: entry.price + 1," + } + ] + }, + { + "id": "purchase-order-uses-zero-price", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "A direct purchase records the item but stores a zero order total instead of the price paid.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { total: total / 100 });", + "replace": "await ctx.db.patch(orderId, { total: 0 });" + } + ] + }, + { + "id": "review-comment-is-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Review submission persists an empty comment rather than the customer's submitted text.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.insert('review', { ...args, accountId: user._id });", + "replace": "await ctx.db.insert('review', { ...args, comment: '', accountId: user._id });" + } + ] + }, + { + "id": "authorized-restock-does-not-change-stock", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Accept an administrator restock without changing warehouse stock.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + args.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "purchases-do-not-affect-best-sellers", + "scenario": "tracks/ecommerce/scenarios/02-operational-best-sellers.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5d" + ], + "desc": "Rank signed-out recommendations without purchase counts.", + "file": "convex/shop.js", + "edits": [ + { + "find": "recommended: recommendations", + "replace": "recommended: [...catalog].sort((a, b) => a.name.localeCompare(b.name)).slice(0, 10)" + } + ] + }, + { + "id": "initial-dashboard-load-omits-low-stock", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5e", + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "The dashboard's initial load reports no low-stock items, so an item already at ten units or fewer is missing until stock moves; 5a's opening precondition re-asserts the same listing, so it necessarily regresses with 5e.", + "file": "convex/shop.js", + "edits": [ + { + "find": "lowStock: catalog.filter(i => i.stock <= 10).sort((a, b) => a.stock - b.stock)", + "replace": "lowStock: []" + } + ] + }, + { + "id": "category-totals-skip-the-newest-order", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5f" + ], + "desc": "Category totals leave out the most recent order, so the reloaded dashboard shows a purchase only once a later order pushes it into the aggregate.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const sold = standing.filter(l => itemById.get(l.item_id)?.category === category);", + "replace": "const sold = standing.filter(l => itemById.get(l.item_id)?.category === category).slice(0, -1);" + } + ] + }, + { + "id": "transfer-debits-source-without-crediting-existing-destination", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202a", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "A transfer debits the source but credits zero to the destination. This breaks both movement/conservation assertions and the authorized-transfer positive observation inside the authorization check; that last failure is not evidence of an unauthorized transfer.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(target._id, { quantity: target.quantity + args.quantity });", + "replace": "await ctx.db.patch(target._id, { quantity: target.quantity });" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c" + ], + "desc": "Cancellation changes order state but skips restoration, so a fresh client reads the persisted shortfall.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + allocation.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "transfer-creates-stock-during-race", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "A transfer creates one extra unit while a purchase runs.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(target._id, { quantity: target.quantity + args.quantity });", + "replace": "await ctx.db.patch(target._id, { quantity: target.quantity + args.quantity + 1 });" + } + ] + }, + { + "id": "customer-can-cancel-scheduled-restock", + "scenario": "tracks/ecommerce/scenarios/03-deferred-access.json", + "targets": [ + "ecommerce.l3.deferred-access.scheduled-work-access.317a" + ], + "desc": "Scheduled-restock cancellation routes omit their staff checks.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await operator(ctx, true);\n const restock = await ctx.db.get(restockId)", + "replace": "await account(ctx);\n const restock = await ctx.db.get(restockId)" + } + ] + }, + { + "id": "completed-restock-is-replayed", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.exactly-once.311a" + ], + "desc": "Completing the deferred work applies its stock effect twice.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + restock.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity + 2 * restock.quantity });" + } + ] + }, + { + "id": "due-restock-does-not-change-stock", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-apply.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a" + ], + "desc": "A due restock is recorded without adding stock.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + restock.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "cancelled-restock-remains-pending", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a" + ], + "desc": "Cancelling a restock leaves it pending so it later applies.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.scheduler.cancel(restock.scheduledId);\n await ctx.db.patch(restock._id, { status: 'cancelled' });", + "replace": "// Defect: cancellation does not cancel scheduled work." + } + ] + }, + { + "id": "server-time-restock-never-completes", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.312a" + ], + "desc": "Server time never selects a due restock after restart.", + "file": "convex/progression.js", + "edits": [ + { + "find": "ctx.scheduler.runAfter(args.delaySeconds * 1000,", + "replace": "ctx.scheduler.runAfter(86400000," + } + ] + }, + { + "id": "ship-acknowledges-without-changing-status", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-ship.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1c" + ], + "desc": "Shipping returns success but writes pending back to the order, leaving both live views unchanged.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { status: 'shipped' });", + "replace": "// Defect: shipping acknowledges but leaves the order pending." + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping route keeps authentication but drops its staff role gate.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const user = await operator(ctx);", + "replace": "const user = await account(ctx);" + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation retains authentication and pending-state validation but drops order ownership.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!order || order.account_id !== user._id) throw new ConvexError('Order not found');", + "replace": "if (!order) throw new ConvexError('Order not found');" + } + ] + }, + { + "id": "promotion-save-drops-bounded-values", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.progression.promotion-rules.promotion-rule-values.620a" + ], + "desc": "Promotion creation drops its bounded discount and limit.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.insert('promotion', args);", + "replace": "await ctx.db.insert('promotion', { ...args, discountPercent: 0, usageLimit: 0 });" + } + ] + }, + { + "id": "stock-alert-delivery-is-suppressed", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alert-delivery.json", + "targets": [ + "ecommerce.progression.stock-alerts.stock-alert-delivery.631c" + ], + "desc": "A request is accepted but restored stock does not deliver its alert.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.insert('notification', { accountId: alert.accountId, type: 'stock', message: `${item.name} is back in stock` });", + "replace": "// Defect: no notification is delivered." + } + ] + }, + { + "id": "support-intake-returns-no-reference", + "scenario": "tracks/ecommerce/scenarios/progression-support-intake.json", + "targets": [ + "ecommerce.progression.support-intake.support-intake.610a" + ], + "desc": "Support intake creates a ticket but omits its reference from the response.", + "file": "convex/progression.js", + "edits": [ + { + "find": "const reference = `SUP-${id.slice(-10)}`;", + "replace": "const reference = '';" + } + ] + }, + { + "id": "support-triage-discards-updates", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage.json", + "targets": [ + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c" + ], + "desc": "Support triage accepts but discards assignment, priority, and status updates.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(ticketId, change);", + "replace": "// Defect: support edits are acknowledged but discarded." + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Cancellation changes order state but skips restoration of its recorded warehouse allocations.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + allocation.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "Cancellation restores allocations but writes pending back to order history.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { status: 'cancelled', refunded: order.total });", + "replace": "await ctx.db.patch(orderId, { status: 'pending', refunded: order.total });" + } + ] + }, + { + "id": "cancellation-accounting-loses-stock-restoration", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + allocation.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "negative-cart-quantity-is-accepted", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109b" + ], + "desc": "Cart quantity updates accept values below one.", + "file": "convex/shop.js", + "edits": [ + { + "find": " quantity(amount, !add);", + "replace": " if (amount >= 0) quantity(amount, !add);" + } + ] + }, + { + "id": "customer-can-create-promotion", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.spec.access-control.promotion-management-boundary.620b" + ], + "desc": "Promotion creation omits its staff authorization check.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await operator(ctx);\n if (!args.code.trim()", + "replace": "await account(ctx);\n if (!args.code.trim()" + } + ] + }, + { + "id": "unpurchased-review-is-accepted", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "The review endpoint bypasses its completed-purchase eligibility check.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!purchased) throw new ConvexError('Purchase this item before reviewing');", + "replace": "// Defect: review requires no purchase." + } + ] + }, + { + "id": "customer-signin-gains-staff-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-access.json", + "targets": [ + "ecommerce.spec.access-control.staff-area-boundary.601b" + ], + "desc": "Treat every authenticated account, including the seeded customer, as staff in authorization and public account state.", + "file": "convex/accounts.js", + "edits": [ + { + "find": "export const isStaff = a => Boolean(a?.roles.some(role => ['admin', 'staff', 'inventory'].includes(role)));", + "replace": "export const isStaff = a => Boolean(a);" + } + ] + }, + { + "id": "staff-can-assign-roles", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-boundary.621b", + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Staff can call the role assignment endpoint, including after administrator access is revoked.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await operator(ctx, true);\n if (!['staff', 'inventory', 'admin'].includes(role))", + "replace": "await operator(ctx);\n if (!['staff', 'inventory', 'admin'].includes(role))" + } + ] + }, + { + "id": "stock-alerts-are-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alerts.json", + "targets": [ + "ecommerce.spec.access-control.stock-alert-privacy.631b" + ], + "desc": "Notification reads expose alerts from other accounts.", + "file": "convex/progression.js", + "edits": [ + { + "find": "(await ctx.db.query('notification').withIndex('account', q => q.eq('accountId', user._id)).collect()).map(publicRow)", + "replace": "(await rows(ctx, 'notification')).map(publicRow)" + } + ] + }, + { + "id": "staff-can-use-direct-restock", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "The direct restock route accepts staff instead of administrators only.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await operator(ctx, true); quantity(args.quantity);\n const stock", + "replace": "await operator(ctx); quantity(args.quantity);\n const stock" + } + ] + }, + { + "id": "concurrent-cart-add-does-not-increment", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a" + ], + "desc": "Concurrent cart adds leave the line quantity and its reservation list unchanged, so the existing cart remains checkable at checkout.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": "const next = add ? amount : amount;" + } + ] + }, + { + "id": "restock-race-records-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { total: total / 100 });", + "replace": "await ctx.db.patch(orderId, { total: total / 100 + 1 });" + } + ] + }, + { + "id": "queue-ignores-live-fulfilment-updates", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.spec.live-state.fulfilment-queue.1a" + ], + "desc": "The open staff queue ignores live fulfilment events, so a new order appears only after a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setFulfilmentQueue(state.fulfilment || { orders: [], depth: 0 });", + "replace": " // Defect: the subscribed fulfilment view is ignored." + } + ] + }, + { + "id": "managed-support-live-refresh-keeps-stale-tickets", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared.json", + "targets": [ + "ecommerce.spec.live-state.managed-support.613a" + ], + "desc": "The native subscription keeps the first support ticket snapshot. Saved replies remain visible on initial load, but another open client does not receive updates.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " useEffect(() => subscribeProgression(next => {\n setState({ ...next, loadedToken: token });", + "replace": " useEffect(() => subscribeProgression(next => {\n setState((current: any) => ({ ...next, tickets: current.loadedToken === token ? current.tickets : next.tickets, loadedToken: token }));" + } + ] + }, + { + "id": "open-review-list-ignores-live-update", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "The native detail subscription retains its first review list while other item data stays current. New reviews appear on reload but not in the already-open view.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItemDetails(state.details);", + "replace": " setItemDetails(previous => state.details.map((item: ItemDetailT) => ({ ...item, reviews: previous.find(old => old.id === item.id)?.reviews || item.reviews })));" + } + ] + }, + { + "id": "espresso-stock-row-ignores-live-updates", + "scenario": "tracks/ecommerce/scenarios/01-buying.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "The live catalogue handler preserves a stale Espresso Machine stock projection while applying all other item updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => state.items.map((item: ItemT) => item.name === 'Espresso Machine' ? current.find(previous => previous.id === item.id) || item : item));" + } + ] + }, + { + "id": "purchase-counts-never-affect-ranking", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "The catalogue ranking ignores recorded purchases and therefore never promotes the bought item.", + "file": "convex/shop.js", + "edits": [ + { + "find": "purchaseCount: standing.filter(l => l.item_id === i._id).reduce((n, l) => n + l.quantity, 0)", + "replace": "purchaseCount: 0" + } + ] + }, + { + "id": "live-review-average-uses-an-extra-divisor", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "The live review event divides the rating sum by one more review than actually exists.", + "file": "convex/shop.js", + "edits": [ + { + "find": "/ itemReviews.length : 0", + "replace": "/ (itemReviews.length + 1) : 0" + } + ] + }, + { + "id": "live-admin-updates-keep-stale-category-totals", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.spec.live-state.sales-dashboard.5b" + ], + "desc": "Live admin updates keep the category totals loaded at page load, so a purchase does not move units or revenue until a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setAdminOverview(state.admin);", + "replace": " setAdminOverview(previous => state.admin && previous ? { ...state.admin, categories: previous.categories } : state.admin);" + } + ] + }, + { + "id": "shared-cart-live-events-ignored", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.live-state.shared-cart.4c" + ], + "desc": "The cart accepts initial snapshots and updates after a local write, but an idle second session ignores remote cart changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [cart, setCart] = useState({ items: [], total: 0 });", + "replace": " const [cart, setCart] = useState({ items: [], total: 0 });\n const cartSnapshotAccount = useRef(null);\n const cartWrittenHere = useRef(false);" + }, + { + "find": " setCart(state.cart);", + "replace": " const cartAccount = state.user?.id ?? \"anonymous\";\n if (cartSnapshotAccount.current !== cartAccount) cartWrittenHere.current = false;\n if (cartSnapshotAccount.current !== cartAccount || cartWrittenHere.current) setCart(state.cart);\n cartSnapshotAccount.current = cartAccount;" + }, + { + "find": " await mutate(\"api:add_to_cart\", { itemId, quantity: 1 });", + "replace": " cartWrittenHere.current = true;\n await mutate(\"api:add_to_cart\", { itemId, quantity: 1 });" + } + ] + }, + { + "id": "transfer-totals-omit-destination-credit-live", + "scenario": "tracks/ecommerce/scenarios/02-transfer-totals.json", + "targets": [ + "ecommerce.spec.live-state.stock-transfers.2b" + ], + "desc": "A transfer debits the source but adds zero to the destination, so the two live warehouse totals do not move in opposite directions.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(target._id, { quantity: target.quantity + args.quantity });", + "replace": "await ctx.db.patch(target._id, { quantity: target.quantity });" + } + ] + }, + { + "id": "restock-does-not-increase-stock", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json", + "targets": [ + "ecommerce.spec.live-state.warehouse-stock.7c" + ], + "desc": "The restock endpoint accepts a positive quantity but increments authoritative stock by zero.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + args.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "notification-preference-is-not-saved", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.state-durability.notification-preferences-reload.630a" + ], + "desc": "Saved notification preferences remain disabled after reload.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch((await account(ctx))._id, { preference });", + "replace": "await ctx.db.patch((await account(ctx))._id, { preference: { order: false, stock: false } });" + } + ] + }, + { + "id": "role-assignment-drops-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.state-durability.staff-role-reload.621a" + ], + "desc": "Inventory role assignments are not saved; staff and administrator role transitions still work.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(accountId, { roles: [role] });", + "replace": "await ctx.db.patch(accountId, { roles: [role === 'inventory' ? 'staff' : role] });" + } + ] + }, + { + "id": "revenue-aggregation-ignores-order-totals", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "The admin revenue aggregation counts every order as zero regardless of its stored total.", + "file": "convex/shop.js", + "edits": [ + { + "find": "revenue: money(headers.reduce((n, o) => n + o.total - o.refunded, 0))", + "replace": "revenue: 0" + } + ] + }, + { + "id": "purchase-does-not-reduce-warehouse-stock", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107b" + ], + "desc": "Purchases create orders without reducing warehouse stock.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(row._id, { quantity: row.quantity - take });", + "replace": "await ctx.db.patch(row._id, { quantity: row.quantity });" + } + ] + }, + { + "id": "direct-purchase-total-ignores-store-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "The direct purchase creates one order but records a zero total rather than the store's current price.", + "file": "convex/shop.js", + "edits": [ + { + "find": "total += Math.round(item.price * 100) * line.quantity;", + "replace": "total += 100 * line.quantity;" + } + ] + }, + { + "id": "overdraw-transfer-is-accepted", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c" + ], + "desc": "The atomic source debit no longer requires sufficient quantity, so an overdrawn transfer succeeds and moves both warehouse totals.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (source.quantity < args.quantity) throw new ConvexError('Not enough stock');", + "replace": "// Defect: transfer may debit below zero." + } + ] + }, + { + "id": "recommendations-ignore-pending-purchases", + "scenario": "tracks/ecommerce/scenarios/02-operational-recommendations.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5c" + ], + "desc": "Never use customer purchases to choose recommended categories.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const purchasedCategories = new Set(standing.filter(l => ownOrderIds.has(l.order_id)).map(l => itemById.get(l.item_id)?.category));", + "replace": "const purchasedCategories = new Set();" + } + ] + }, + { + "id": "cart-add-line-lookup-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109a" + ], + "desc": "Cart line lookup ignores account ownership, so a second customer changes the first customer’s item quantity.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const existing = await ctx.db.query('order_cart').withIndex('line', q => q.eq('account_id', user._id).eq('item_id', itemId)).unique();", + "replace": "const existing = (await ctx.db.query('order_cart').collect()).find(row => row.item_id === itemId);" + } + ] + }, + { + "id": "direct-purchase-is-attributed-to-another-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-attribution.json", + "targets": [ + "ecommerce.spec.access-control.purchase-attribution.102a" + ], + "desc": "Purchase order is attributed to another account despite the real caller session.", + "file": "convex/shop.js", + "edits": [ + { + "find": "account_id: user._id, total: 0, refunded: 0, status: 'pending'", + "replace": "account_id: (await rows(ctx, 'order_account')).find(a => a._id !== user._id)._id, total: 0, refunded: 0, status: 'pending'" + } + ] + }, + { + "id": "unauthenticated-purchase-defaults-to-admin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Anonymous native buy defaults to the administrator account and commits a purchase.", + "file": "convex/shop.js", + "edits": [ + { + "find": "purchase(ctx, await account(ctx), [{ item_id: itemId, quantity: 1 }])", + "replace": "purchase(ctx, (await account(ctx, false)) || (await rows(ctx, 'order_account')).find(isAdmin), [{ item_id: itemId, quantity: 1 }])" + } + ] + }, + { + "id": "signed-out-visitor-purchase-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "Expose the purchase control while signed out and accept its anonymous buy as an administrator-owned purchase.", + "file": "convex/shop.js", + "edits": [ + { + "find": "purchase(ctx, await account(ctx), [{ item_id: itemId, quantity: 1 }])", + "replace": "purchase(ctx, (await account(ctx, false)) || (await rows(ctx, 'order_account')).find(isAdmin), [{ item_id: itemId, quantity: 1 }])" + }, + { + "file": "client/src/App.tsx", + "find": "const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff;", + "replace": "const isCustomer = !currentUser || (!currentUser.isAdmin && !currentUser.isStaff);" + } + ] + }, + { + "id": "staff-can-see-admin-navigation", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-area-boundary.7a" + ], + "desc": "Staff are incorrectly treated as administrators in reads, navigation, and writes.", + "file": "convex/accounts.js", + "edits": [ + { + "find": "export const isAdmin = a => Boolean(a?.roles.includes('admin'));", + "replace": "export const isAdmin = a => Boolean(a?.roles.some(role => ['admin', 'staff'].includes(role)));" + } + ] + }, + { + "id": "reconnect-hydration-loses-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "Network loss permanently closes the live client, so the account misses peer cart changes after reconnect.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nwindow.addEventListener('offline', () => { live.close(); }, { once: true });\nexport async function signOut()" + } + ] + }, + { + "id": "profile-data-is-lost-on-server-restart", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.state-durability.customer-profile-reload.620a" + ], + "desc": "Signing in from the fresh recovery browser resets a stored profile; the initial save and token reload remain valid.", + "file": "convex/accounts.js", + "edits": [ + { + "find": "if (await ctx.db.query('order_account').withIndex('auth_user', q => q.eq('auth_user_id', userId)).unique()) return;", + "replace": "const existing = await ctx.db.query('order_account').withIndex('auth_user', q => q.eq('auth_user_id', userId)).unique();\n if (existing) { await ctx.db.patch(existing._id, { profile: { name: '', address: '' } }); return; }" + } + ] + }, + { + "id": "stock-alert-repeats-while-in-stock", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alerts.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a" + ], + "desc": "One restock delivers two independent notifications for the same pending stock alert.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.insert('notification', { accountId: alert.accountId, type: 'stock', message: `${item.name} is back in stock` });", + "replace": "await ctx.db.insert('notification', { accountId: alert.accountId, type: 'stock', message: `${item.name} is back in stock` });\n await ctx.db.insert('notification', { accountId: alert.accountId, type: 'stock', message: `${item.name} is back in stock` });" + } + ] + }, + { + "id": "last-unit-allows-negative-stock", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Allocation permits an oversell and debits the first warehouse below zero.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (stocks.reduce((sum, row) => sum + row.quantity, 0) < count) throw new ConvexError('Not enough stock');", + "replace": "// Defect: insufficient stock is accepted." + }, + { + "find": "const take = Math.min(row.quantity, count);", + "replace": "const take = count;" + } + ] + }, + { + "id": "scheduled-restock-never-becomes-due-after-restart", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.311a" + ], + "desc": "A reconnect handler mistakenly cancels pending deferred work; ordinary scheduling before a reconnect still completes.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nlet lastRecoveryConnection = 1;\nlive.subscribeToConnectionState(({ isWebSocketConnected, connectionCount }) => {\n if (!isWebSocketConnected || connectionCount <= lastRecoveryConnection) return;\n lastRecoveryConnection = connectionCount;\n void (async () => {\n const token = await accessToken();\n if (!token) return;\n http.setAuth(token);\n const current = await http.query(makeFunctionReference<'query'>('progression:state'), {});\n if (current.user?.isAdmin) for (const task of current.scheduledRestocks) await mutate('api:cancel_scheduled_restock', { restockId: task.id });\n })().catch(() => undefined);\n});\nexport async function signOut()" + } + ] + }, + { + "id": "checkout-crash-integrity", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A disconnect-triggered recovery worker restores a wrong cart quantity through the native write path, producing neither the prepared cart nor a completed checkout. The worker retries native calls while the backend is down; it does not wait for the browser WebSocket reconnect backoff.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nlet recoveryStarted = false;\nlive.subscribeToConnectionState(({ isWebSocketConnected, connectionCount }) => {\n if (isWebSocketConnected || !connectionCount || recoveryStarted) return;\n recoveryStarted = true;\n void (async () => {\n const token = await accessToken();\n if (!token) return;\n http.setAuth(token);\n const deadline = Date.now() + 10000;\n while (true) {\n try {\n const current = await state();\n const line = current.cart.items[0] || current.orders[0]?.items[0];\n if (!line) throw new Error('Crash control has no cart or prior order');\n await mutate('api:update_cart_quantity', { itemId: line.itemId, quantity: 2 });\n console.error('Crash control committed', Date.now());\n return;\n } catch (error) {\n if (Date.now() >= deadline) throw error;\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n }\n })().catch(error => console.error('Crash control failed', String(error)));\n});\nexport async function signOut()" + } + ] + }, + { + "id": "checkout-crash-sibling-stock", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A disconnect-triggered recovery worker removes stock for Coffee Grinder while leaving cart and order data intact. The second-product stock discrepancy must fail crash atomicity. The worker retries native calls while the backend is down; it does not wait for the browser WebSocket reconnect backoff.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nlet recoveryStarted = false;\nlive.subscribeToConnectionState(({ isWebSocketConnected, connectionCount }) => {\n if (isWebSocketConnected || !connectionCount || recoveryStarted) return;\n recoveryStarted = true;\n void (async () => {\n const token = await accessToken();\n if (!token) return;\n http.setAuth(token);\n const deadline = Date.now() + 10000;\n while (true) {\n try {\n const current = await state();\n const priorOrder = current.orders.find((order: any) => order.status === 'pending');\n if (!priorOrder) throw new Error('Crash control has no pending order');\n await mutate('api:cancel_order', { orderId: priorOrder.id });\n console.error('Crash control committed', Date.now());\n return;\n } catch (error) {\n if (Date.now() >= deadline) throw error;\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n }\n })().catch(error => console.error('Crash control failed', String(error)));\n});\nexport async function signOut()" + }, + { + "file": "convex/shop.js", + "find": " for (const line of await ctx.db.query('order_line').withIndex('order', q => q.eq('order_id', orderId)).collect()) await restoreLine(ctx, line);\n await ctx.db.patch(orderId, { status: 'cancelled', refunded: order.total });", + "replace": " for (const line of await ctx.db.query('order_line').withIndex('order', q => q.eq('order_id', orderId)).collect()) {\n if ((await ctx.db.get(line.item_id))?.name !== 'Coffee Grinder') continue;\n const stock = await ctx.db.query('stock').withIndex('item', q => q.eq('item_id', line.item_id)).first();\n if (stock) await ctx.db.patch(stock._id, { quantity: stock.quantity - 1 });\n }" + } + ] + }, + { + "id": "checkout-crash-durability", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-durability.910b" + ], + "desc": "A disconnect-triggered recovery worker corrupts the total of an earlier acknowledged order through its native write path, without changing stock, allocations or cart state. The worker retries native calls while the backend is down; it does not wait for the browser WebSocket reconnect backoff.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nlet recoveryStarted = false;\nlive.subscribeToConnectionState(({ isWebSocketConnected, connectionCount }) => {\n if (isWebSocketConnected || !connectionCount || recoveryStarted) return;\n recoveryStarted = true;\n void (async () => {\n const token = await accessToken();\n if (!token) return;\n http.setAuth(token);\n const deadline = Date.now() + 10000;\n while (true) {\n try {\n const current = await state();\n const priorOrder = current.orders.find((order: any) => order.status === 'pending');\n if (!priorOrder) throw new Error('Crash control has no pending order');\n await mutate('api:cancel_order', { orderId: priorOrder.id });\n console.error('Crash control committed', Date.now());\n return;\n } catch (error) {\n if (Date.now() >= deadline) throw error;\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n }\n })().catch(error => console.error('Crash control failed', String(error)));\n});\nexport async function signOut()" + }, + { + "file": "convex/shop.js", + "find": " for (const line of await ctx.db.query('order_line').withIndex('order', q => q.eq('order_id', orderId)).collect()) await restoreLine(ctx, line);\n await ctx.db.patch(orderId, { status: 'cancelled', refunded: order.total });", + "replace": " await ctx.db.patch(orderId, { total: 0 });" + } + ] + }, + { + "id": "restock-trusts-submitted-admin-role", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust an administrator role supplied by staff in the restock request. Normal staff requests are still refused.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const restock = mutationGeneric({ args: { itemId: v.id('item'), warehouseId: v.id('warehouse'), quantity: v.number() },\n handler: async (ctx, args) => {\n await operator(ctx, true); quantity(args.quantity);", + "replace": "export const restock = mutationGeneric({ args: { itemId: v.id('item'), warehouseId: v.id('warehouse'), quantity: v.number(), role: v.optional(v.string()) },\n handler: async (ctx, args) => {\n await operator(ctx, args.role !== 'admin'); quantity(args.quantity);" + } + ] + }, + { + "id": "direct-purchase-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Ordinary purchases use the catalog price; an extra price field overrides the booked order and line price.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const buy = mutationGeneric({ args: { itemId: v.id('item') }, handler: async (ctx, { itemId }) =>\n purchase(ctx, await account(ctx), [{ item_id: itemId, quantity: 1 }]) });", + "replace": "export const buy = mutationGeneric({ args: { itemId: v.id('item'), price: v.optional(v.number()) }, handler: async (ctx, { itemId, price }) =>\n purchase(ctx, await account(ctx), [{ item_id: itemId, quantity: 1, price }]) });" + }, + { + "find": " if (!item) throw new ConvexError('Unknown item');\n const lineId", + "replace": " if (!item) throw new ConvexError('Unknown item');\n if (line.price !== undefined) item.price = line.price;\n const lineId" + } + ] + }, + { + "id": "order-cache-survives-account-switch", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Merge order updates into a browser cache that is never cleared when the account changes. Separate-browser ownership still works.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [orders, setOrders] = useState([]);", + "replace": " const [orders, setStoredOrders] = useState([]);\n const setOrders: typeof setStoredOrders = update => setStoredOrders(previous => {\n const incoming = typeof update === 'function' ? update(previous) : update;\n return [...new Map([...previous, ...incoming].map(order => [order.id, order])).values()];\n });" + } + ] + }, + { + "id": "checkout-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Accept an optional price field and overwrite the checkout total with it.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const checkout = mutationGeneric({ args: {}, handler: async ctx => {", + "replace": "export const checkout = mutationGeneric({ args: { price: v.optional(v.number()) }, handler: async (ctx, args) => {" + }, + { + "find": " const result = await purchase(ctx, user, lines);", + "replace": " const result = await purchase(ctx, user, lines);\n if (args.price !== undefined) await ctx.db.patch(result.orderId, { total: args.price });" + } + ] + }, + { + "id": "checkout-missing-sibling-line", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit the cart checkout with one product line missing while retaining the other product and the full order total.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const result = await purchase(ctx, user, lines);", + "replace": " const result = await purchase(ctx, user, lines);\n for (const line of await rows(ctx, 'order_line')) {\n if (line.order_id === result.orderId && (await ctx.db.get(line.item_id)).name === 'Coffee Grinder') await ctx.db.delete(line._id);\n }" + } + ] + }, + { + "id": "shipping-cancellation-cancel-shipped", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation accepts a shipped order and restores its stock after shipping.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (order.status !== 'pending')", + "replace": "if (false)" + } + ] + }, + { + "id": "shipping-cancellation-resurrect-cancelled", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Shipping accepts a cancelled order after its stock was restored.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!order || order.status !== 'pending')", + "replace": "if (!order)" + } + ] + }, + { + "id": "shipping-cancellation-combined-race", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation restores stock but records a shipped status, combining mutually exclusive effects.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (order.status !== 'pending')", + "replace": "if (false)" + }, + { + "find": "await ctx.db.patch(orderId, { status: 'cancelled', refunded: order.total });", + "replace": "await ctx.db.patch(orderId, { status: 'shipped', refunded: order.total });" + } + ] + }, + { + "id": "shipping-cancellation-reject-both", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Both cancellation and shipping refuse pending orders; useful progress must remain required.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (order.status !== 'pending')", + "replace": "if (true)" + }, + { + "find": "if (!order || order.status !== 'pending')", + "replace": "if (true)" + } + ] + }, + { + "id": "shipping-cancellation-queue-retains-removed-order", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "The live staff queue accepts new orders but retains rows after cancellation; the database state remains correct.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setFulfilmentQueue(state.fulfilment || { orders: [], depth: 0 });", + "replace": " setFulfilmentQueue(previous => { const next = state.fulfilment || { orders: [], depth: 0 }; return next.orders.length < previous.orders.length ? previous : next; });" + } + ] + }, + { + "id": "opposing-transfer-ignored", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is acknowledged without moving stock.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (args.fromWarehouseId === args.toWarehouseId) throw new ConvexError('Choose different warehouses');", + "replace": " if (args.fromWarehouseId === args.toWarehouseId) throw new ConvexError('Choose different warehouses');\n if (args.quantity === 7) return;" + } + ] + }, + { + "id": "opposing-transfer-rejected", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is refused despite ample source stock.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (args.fromWarehouseId === args.toWarehouseId) throw new ConvexError('Choose different warehouses');", + "replace": " if (args.fromWarehouseId === args.toWarehouseId) throw new ConvexError('Choose different warehouses');\n if (args.quantity === 7) throw new ConvexError('Transfer disabled');" + } + ] + }, + { + "id": "overlapping-cart-add-ignored", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is ignored while ordinary cart setup still works.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": " if (add && user.username.includes('checkout-overlap') && (await ctx.db.get(itemId)).name === 'Coffee Grinder') return;\n const next = add ? (existing?.quantity || 0) + amount : amount;" + } + ] + }, + { + "id": "overlapping-cart-add-rejected", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is rejected while ordinary cart setup still works.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": " if (add && user.username.includes('checkout-overlap') && (await ctx.db.get(itemId)).name === 'Coffee Grinder') throw new ConvexError('Cart add disabled');\n const next = add ? (existing?.quantity || 0) + amount : amount;" + } + ] + }, + { + "id": "mixed-history-lost-transfer", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: lost-transfer. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(target._id, { quantity: target.quantity + args.quantity });", + "replace": "await ctx.db.patch(target._id, { quantity: target.quantity });" + } + ] + }, + { + "id": "mixed-history-wrong-price", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-price. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": "total += Math.round(item.price * 100) * line.quantity;", + "replace": "total += 100 * line.quantity;" + } + ] + }, + { + "id": "mixed-history-ignored-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: ignored-write. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": " if (add && user.username.includes('history-') && (await ctx.db.get(itemId)).name === 'Coffee Grinder') return;\n const next = add ? (existing?.quantity || 0) + amount : amount;" + } + ] + }, + { + "id": "mixed-history-reject-valid-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: reject-valid-write. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": " if (add && user.username.includes('history-') && (await ctx.db.get(itemId)).name === 'Coffee Grinder') throw new ConvexError('Cart add disabled');\n const next = add ? (existing?.quantity || 0) + amount : amount;" + } + ] + }, + { + "id": "mixed-history-repeat-cancel", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: repeat-cancel. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (order.status !== 'pending') throw new ConvexError('Only pending orders can be cancelled');", + "replace": " if (!['pending', 'cancelled'].includes(order.status)) throw new ConvexError('Only pending orders can be cancelled');" + } + ] + }, + { + "id": "mixed-history-wrong-owner", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-owner. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": "account_id: user._id, total: 0, refunded: 0, status: 'pending'", + "replace": "account_id: (await rows(ctx, 'order_account')).find(a => a._id !== user._id)._id, total: 0, refunded: 0, status: 'pending'" + } + ] + }, + { + "id": "checkout-empty-replay-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Allow retrying the consumed empty cart to create another order after a lost reply.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (!lines.length) throw new ConvexError('Cart is empty');", + "replace": "// Defect: an empty consumed cart may create another order." + } + ] + }, + { + "id": "checkout-replay-loses-prior-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Retrying the consumed cart deletes an earlier acknowledged order.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (!lines.length) throw new ConvexError('Cart is empty');", + "replace": " if (!lines.length) { const prior = (await rows(ctx, 'order_header')).find(order => order.account_id === user._id); if (prior) await ctx.db.delete(prior._id); return { orderId: null }; }" + } + ] + }, + { + "id": "checkout-clears-cart-without-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout returns success and empties the cart without recording an order.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const result = await purchase(ctx, user, lines);", + "replace": " const result = { orderId: null };" + } + ] + }, + { + "id": "revenue-aggregation-truncates-order-population", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Calculate revenue from only the first 256 stored orders while preserving all acknowledged purchases.", + "file": "convex/shop.js", + "edits": [ + { + "find": "revenue: money(headers.reduce((n, o) => n + o.total - o.refunded, 0))", + "replace": "revenue: money(headers.slice(0, 256).reduce((n, o) => n + o.total - o.refunded, 0))" + } + ] + }, + { + "id": "catalog-truncates-created-population", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Omit committed products after the first 256 search results.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".slice(searchPage * CATALOG_PAGE_SIZE,", + "replace": ".slice(0, 256).slice(searchPage * CATALOG_PAGE_SIZE," + } + ] + }, + { + "id": "catalog-discards-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Acknowledge catalog creation without committing a product.", + "file": "convex/progression.js", + "edits": [ + { + "find": " if (!args.name.trim() || !args.category.trim() || !Number.isFinite(args.price) || args.price < 0) throw new ConvexError('Invalid product');", + "replace": " if (args.name.length >= 0) return; // mutant: successful mutation without product\n if (!args.name.trim() || !args.category.trim() || !Number.isFinite(args.price) || args.price < 0) throw new ConvexError('Invalid product');" + } + ] + }, + { + "id": "catalog-rejects-all-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Reject every catalog creation while keeping sign-in and catalog reads available.", + "file": "convex/progression.js", + "edits": [ + { + "find": " if (!args.name.trim() || !args.category.trim() || !Number.isFinite(args.price) || args.price < 0) throw new ConvexError('Invalid product');", + "replace": " throw new ConvexError('Catalog writes refused');\n if (!args.name.trim() || !args.category.trim() || !Number.isFinite(args.price) || args.price < 0) throw new ConvexError('Invalid product');" + } + ] + }, + { + "id": "shipping-trusts-submitted-staff-role", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Trust a staff role supplied by a customer; ordinary customers remain refused.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const ship = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const user = await operator(ctx);", + "replace": "export const ship = mutationGeneric({ args: { orderId: v.id('order_header'), role: v.optional(v.string()) }, handler: async (ctx, { orderId, role }) => {\n const user = role === 'staff' ? await account(ctx) : await operator(ctx);" + } + ] + }, + { + "id": "shipping-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Reject legitimate staff shipping as well as customer requests.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const ship = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const user = await operator(ctx);", + "replace": "export const ship = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n throw new ConvexError('Shipping refused');\n const user = await operator(ctx);" + } + ] + }, + { + "id": "cancel-trusts-submitted-owner", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Trust a username submitted by another signed-in customer as the order owner.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const cancel = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const order = await ownedOrder(ctx, orderId);", + "replace": "export const cancel = mutationGeneric({ args: { orderId: v.id('order_header'), username: v.optional(v.string()) }, handler: async (ctx, { orderId, username }) => {\n const user = await account(ctx);\n const claimed = username ? await ctx.db.query('order_account').withIndex('username', q => q.eq('username', username)).unique() : null;\n const order = await ctx.db.get(orderId);\n if (!order || order.account_id !== (claimed?._id ?? user._id)) throw new ConvexError('Order not found');" + } + ] + }, + { + "id": "cancel-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Reject legitimate cancellation as well as foreign requests.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const cancel = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const order = await ownedOrder(ctx, orderId);", + "replace": "export const cancel = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const order = await ownedOrder(ctx, orderId);\n throw new ConvexError('Cancellation refused');" + } + ] + }, + { + "id": "account-password-in-public-data", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Expose the actual submitted account password through data available to other clients.", + "edits": [ + { + "file": "convex/auth.js", + "find": "profile: { name: params.username, registrationNonce },", + "replace": "profile: { name: params.username, registrationNonce, image: params.password }," + }, + { + "file": "convex/accounts.js", + "find": " const user = await ctx.db.get(userId);", + "replace": " const user = await ctx.db.get(userId);\n const publicItem = await ctx.db.query('item').first();\n if (publicItem && user.image) await ctx.db.patch(publicItem._id, { name: publicItem.name + ' ' + user.image });" + } + ] + }, + { + "edits": [ + { + "file": "convex/auth.js", + "find": "verifySecret: (password, hash) => new Scrypt().verify(hash, password),", + "replace": "verifySecret: (password, hash) => password === \"' OR '1'='1\" || new Scrypt().verify(hash, password)," + } + ], + "id": "query-like-password-bypasses-signin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Interpret the query-like password as a login bypass while retaining normal password verification." + }, + { + "edits": [ + { + "file": "convex/auth.js", + "find": "verifySecret: (password, hash) => new Scrypt().verify(hash, password),", + "replace": "verifySecret: (_password, _hash) => false," + } + ], + "id": "account-disclosure-rejects-login", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Reject every valid login; absent disclosures alone must not earn a pass." + }, + { + "id": "signout-keeps-session", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Show a signed-out interface while retaining the browser credential and its purchase authority.", + "edits": [ + { + "file": "client/src/App.tsx", + "find": " const clearSession = () => {\n localStorage.removeItem(TOKEN_KEY);", + "replace": " const clearSession = () => {\n // mutant: retain the current request token after signout" + }, + { + "file": "client/src/App.tsx", + "find": " await signOut();\n clearSession();", + "replace": " clearSession(); // mutant: hide the account without removing server authority" + }, + { + "file": "client/src/App.tsx", + "find": " {currentUser ? (", + "replace": " {currentUser && token ? (" + } + ] + }, + { + "edits": [ + { + "file": "convex/auth.js", + "find": "verifySecret: (password, hash) => new Scrypt().verify(hash, password),", + "replace": "verifySecret: (_password, _hash) => false," + } + ], + "id": "purchase-session-rejects-returning-login", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Refuse every returning login; blocking all access after logout must not pass the session check." + }, + { + "id": "account-password-in-script", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in the public JavaScript bundle while normal signup and login still work.", + "edits": [ + { + "file": "client/src/main.tsx", + "find": "import React from \"react\";", + "replace": "import React from \"react\";\nObject.assign(window, { debugPassword: 'sb-private-pw-76dc492ea153' });" + } + ] + }, + { + "id": "account-password-in-style", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in a public stylesheet while normal signup and login still work.", + "edits": [ + { + "file": "client/src/index.css", + "find": ":root {", + "replace": ":root {\n --debug-password: \"sb-private-pw-76dc492ea153\";" + } + ] + }, + { + "id": "signup-accepts-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust a submitted signup administrator claim and grant protected stock-write authority.", + "edits": [ + { + "file": "convex/schema.js", + "find": "registrationNonce: v.optional(v.string()),", + "replace": "registrationNonce: v.optional(v.string()), isAdmin: v.optional(v.boolean())," + }, + { + "file": "convex/auth.js", + "find": "profile: { name: params.username, registrationNonce },", + "replace": "profile: { name: params.username, registrationNonce, isAdmin: params.role === 'admin' }," + }, + { + "file": "convex/accounts.js", + "find": "username: user.name, roles: [],", + "replace": "username: user.name, roles: user.isAdmin ? ['admin'] : []," + } + ] + }, + { + "id": "admin-boundary-rejects-all-signups", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Keep fixture provisioning available, but refuse every new customer registration; blocking useful signup must not pass the authority check.", + "edits": [ + { + "file": "convex/auth.js", + "find": "if (params.flow === 'signUp') {", + "replace": "if (params.flow === 'signUp') {\n if (!['admin', 'staff', 'customer'].includes(params.username)) throw new ConvexError('Registration disabled');" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/mutations/mongodb-ecommerce.json b/tools/stack-bench/grader/mutations/mongodb-ecommerce.json new file mode 100644 index 00000000000..1b0477e275f --- /dev/null +++ b/tools/stack-bench/grader/mutations/mongodb-ecommerce.json @@ -0,0 +1,3367 @@ +{ + "schemaVersion": 3, + "fixtureSha256": "837e7e81b58f17afdaf29de8ddb7a17da2ac95d3716145c4f0ef996b4b4f5e7d", + "backend": "mongodb", + "track": "ecommerce", + "note": "Mutation definitions for the MongoDB ecommerce reference.", + "mutations": [ + { + "id": "recommendation-dismissal-lost-on-restart", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.spec.state-durability.recommendation-feedback-restart.504c" + ], + "desc": "Erase saved recommendation dismissals when the application starts again.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await mongoose.connect(DATABASE_URL);", + "replace": " await mongoose.connect(DATABASE_URL);\n await Dismissal.deleteMany({});" + } + ] + }, + { + "id": "pending-order-item-return-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-order-return-boundary.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3f" + ], + "desc": "Accept a pending order return and restore its stock before shipment.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (!['shipped', 'delivered'].includes(value.status)) throw new Error('No shipped order found');", + "replace": "if (!['pending', 'shipped', 'delivered'].includes(value.status)) throw new Error('No shipped order found');" + } + ] + }, + { + "id": "staff-admin-access-survives-role-removal", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Keep administrator access after changing the assigned role back to staff.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " target.isAdmin = role === \"admin\";", + "replace": " target.isAdmin = target.isAdmin || role === \"admin\";" + } + ] + }, + { + "id": "shipping-counts-sale-twice", + "scenario": "tracks/ecommerce/scenarios/progression-shipping-accounting.json", + "targets": [ + "ecommerce.inventory-operations.shipping-accounting.202e" + ], + "desc": "Shipping succeeds but doubles the completed sale value in authoritative revenue.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { $set: { status: \"shipped\" } },", + "replace": " { $set: { status: \"shipped\" }, $mul: { total: 2 } }," + } + ] + }, + { + "id": "signup-does-not-expose-created-account", + "scenario": "tracks/ecommerce/scenarios/01-account-create.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Signup succeeds but the client discards the created account identity from its current session view.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " saveSession(data.token, data.user);\n };\n\n const handleSignIn", + "replace": " saveSession(data.token, { ...data.user, username: \"\" });\n };\n\n const handleSignIn" + } + ] + }, + { + "id": "duplicate-signup-reports-success", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "A duplicate username is reported as a successful empty signup response instead of a refusal.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (existing) return res.status(409).json({ error: \"Username is already taken\" });", + "replace": " if (existing) return res.json({}); // mutant: duplicate signup is falsely accepted" + } + ] + }, + { + "id": "signin-skips-password-verification", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Signin accepts an existing account without requiring its password to match.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!valid) return res.status(401).json({ error: \"Invalid username or password\" });", + "replace": " if (false && !valid) return res.status(401).json({ error: \"Invalid username or password\" });" + } + ] + }, + { + "id": "signout-keeps-current-account", + "scenario": "tracks/ecommerce/scenarios/01-account-signout.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Signout disconnects the token state but leaves the current account and persisted credential in place.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const handleSignOut = () => {\n clearSession();\n };", + "replace": " const handleSignOut = () => {\n setToken(null); // mutant: visible and persisted account state is not cleared\n };" + } + ] + }, + { + "id": "session-token-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-account-reload.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "The active session is kept only in React state and is unavailable after a page reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " localStorage.setItem(TOKEN_KEY, tok);\n setToken(tok);", + "replace": " void tok; // mutant: the session token is never persisted\n setToken(tok);" + } + ] + }, + { + "id": "purchase-counts-never-affect-ranking", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "The catalogue ranking ignores recorded purchases and therefore never promotes the bought item.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " purchaseCount: purchaseMap.get(id) || 0,", + "replace": " purchaseCount: 0, // mutant: ranking ignores durable purchase counts" + } + ] + }, + { + "id": "signed-out-visitor-purchase-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "The UI exposes purchase controls to visitors and the buy route assigns unauthenticated requests an unverified identity, allowing an actual stock-debiting order.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff;", + "replace": " const isCustomer = !currentUser?.isAdmin && !currentUser?.isStaff;" + }, + { + "file": "server/src/index.ts", + "find": "app.post(\"/api/items/:id/buy\", requireAuth, async (req, res) => {", + "replace": "app.post(\"/api/items/:id/buy\", async (req, _res, next) => {\n (req as any).user = await userFromToken(extractToken(req)) || { _id: new Types.ObjectId() };\n next();\n}, async (req, res) => {" + } + ] + }, + { + "id": "espresso-stock-row-ignores-live-updates", + "scenario": "tracks/ecommerce/scenarios/01-buying.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "The live catalogue handler preserves a stale Espresso Machine stock projection while applying all other item updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"items:update\", (data: ItemT[]) => setItems(data));", + "replace": " socket.on(\"items:update\", (data: ItemT[]) => setItems((previous) => data.map((item) => item.name === \"Espresso Machine\" ? { ...item, stock: previous.find((old) => old.id === item.id)?.stock ?? item.stock } : item)));" + } + ] + }, + { + "id": "restock-race-records-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: purchase receipt loses the authoritative price\n });" + } + ] + }, + { + "id": "purchase-order-uses-zero-price", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "A direct purchase records the item but stores a zero order total instead of the price paid.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: purchase receipt loses the authoritative price\n });" + } + ] + }, + { + "id": "reload-hydrates-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Cart hydration discards the persisted server response after reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const refreshCart = useCallback(async (tok: string) => {\n const data = await apiFetch(\"/api/cart\", tok);\n setCart(data);\n }, []);", + "replace": " const refreshCart = useCallback(async (tok: string) => {\n await apiFetch(\"/api/cart\", tok);\n setCart({ items: [], total: 0 }); // mutant: persisted cart response is discarded\n }, []);" + } + ] + }, + { + "id": "shared-cart-live-events-ignored", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.live-state.shared-cart.4c" + ], + "desc": "An already-open second session ignores committed cart update events.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"cart:update\", (data: CartT) => setCart(data));", + "replace": " socket.on(\"cart:update\", (data: CartT) => setCart(current => current.items.length === 0 ? current : data)); // mutant: an empty second-session cart ignores its first remote update" + } + ] + }, + { + "id": "review-comment-is-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Review submission persists an empty comment rather than the customer's submitted text.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },", + "replace": " { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: \"\" }," + } + ] + }, + { + "id": "repeat-review-uses-a-new-owner-key", + "scenario": "tracks/ecommerce/scenarios/01-review-uniqueness.json", + "targets": [ + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "desc": "Each review submission is stored under a fresh owner key, bypassing the one-review-per-customer constraint.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { itemId, userId: user._id },\n { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },", + "replace": " { itemId, userId: new Types.ObjectId() },\n { itemId, userId: new Types.ObjectId(), username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" }," + } + ] + }, + { + "id": "live-review-average-uses-an-extra-divisor", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "The live review event divides the rating sum by one more review than actually exists.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / reviews.length : 0;", + "replace": "async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / (reviews.length + 1) : 0;" + } + ] + }, + { + "id": "warehouse-view-omits-one-location", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "The admin warehouse projection truncates the final item-location row.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {overview.locations.map((loc) => (", + "replace": " {overview.locations.slice(0, -1).map((loc) => (" + } + ] + }, + { + "id": "unauthenticated-purchase-defaults-to-admin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "The purchase endpoint drops authentication and assigns sessionless purchases to the seeded administrator.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/items/:id/buy\", requireAuth, async (req, res) => {", + "replace": "app.post(\"/api/items/:id/buy\", async (req, res) => {" + }, + { + "find": " const user = (req as any).user;\n const order = await Order.create({", + "replace": " const user = (req as any).user || await User.findOne({ username: \"admin\" });\n const order = await Order.create({" + } + ] + }, + { + "id": "direct-purchase-total-ignores-store-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "The direct purchase creates one order but records a zero total rather than the store's current price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: direct purchase ignores the authoritative price\n });" + } + ] + }, + { + "id": "cart-hydration-loses-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "Reload hydration discards the account's persisted cart response.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const refreshCart = useCallback(async (tok: string) => {\n const data = await apiFetch(\"/api/cart\", tok);\n setCart(data);\n }, []);", + "replace": " const refreshCart = useCallback(async (tok: string) => {\n await apiFetch(\"/api/cart\", tok);\n setCart({ items: [], total: 0 }); // mutant: account state is discarded on hydration\n }, []);" + } + ] + }, + { + "id": "reconnect-hydration-loses-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "The initial account cart loads correctly, but after network restoration the client ignores both refreshed and pushed cart state.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " useEffect(() => {\n const socket = io({ auth: token ? { token } : {} });", + "replace": " useEffect(() => {\n const clearAccountOffline = () => {\n setCurrentUser(null);\n setCart({ items: [], total: 0 });\n };\n window.addEventListener(\"offline\", clearAccountOffline, { once: true });\n const socket = io({ auth: token ? { token } : {} });" + } + ] + }, + { + "id": "order-history-is-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Order history returns every customer's orders instead of filtering by the authenticated owner.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const orders = await Order.find({ userId }).sort({ createdAt: -1 });", + "replace": " const orders = await Order.find({}).sort({ createdAt: -1 });" + } + ] + }, + { + "id": "revenue-aggregation-ignores-order-totals", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "The admin revenue aggregation counts every order as zero regardless of its stored total.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { $group: { _id: null, total: { $sum: { $subtract: [\"$total\", { $ifNull: [\"$refundTotal\", 0] }] } } } },", + "replace": " { $group: { _id: null, total: { $sum: 0 } } }," + } + ] + }, + { + "id": "unpurchased-review-is-accepted", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "The review endpoint bypasses its completed-purchase eligibility check.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }", + "replace": " if (false && !hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }" + } + ] + }, + { + "id": "purchased-review-ui-does-not-submit", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b" + ], + "desc": "The review form discards valid customer submissions. Direct authorized and unauthorized review calls retain their normal behavior.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"\");", + "replace": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"Review submission unavailable\");\n return;" + } + ] + }, + { + "id": "external-stock-polling-disabled", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "The server stops reconciling direct database stock writes into live catalogue events.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " broadcastItems().catch((err) => console.error(\"broadcastItems poll failed\", err));", + "replace": " // mutant: direct database stock changes are never reconciled" + } + ] + }, + { + "id": "server-restart-disables-catalog-recovery", + "scenario": "tracks/ecommerce/scenarios/01-external-server-restart-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901c" + ], + "desc": "After a socket disconnect, the existing page ignores both reconnect refreshes and later catalogue snapshots.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const socketRef = useRef(null);\n\n const saveSession", + "replace": " const socketRef = useRef(null);\n const acceptCatalogRecovery = useRef(true);\n\n const saveSession" + }, + { + "find": " socket.on(\"connect\", () => {\n refreshItems().catch((err) => console.error(err));", + "replace": " socket.on(\"disconnect\", () => { acceptCatalogRecovery.current = false; });\n socket.on(\"connect\", () => {\n if (acceptCatalogRecovery.current) refreshItems().catch((err) => console.error(err));" + }, + { + "find": " socket.on(\"items:update\", (data: ItemT[]) => setItems(data));", + "replace": " socket.on(\"items:update\", (data: ItemT[]) => { if (acceptCatalogRecovery.current) setItems(data); });" + } + ] + }, + { + "id": "reconnect-generation-ignores-current-catalog", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "After the browser goes offline, the existing page ignores reconnect refreshes and subsequent catalogue events.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const socketRef = useRef(null);\n\n const saveSession", + "replace": " const socketRef = useRef(null);\n const acceptCatalogUpdates = useRef(true);\n useEffect(() => {\n const stopCatalogRecovery = () => { acceptCatalogUpdates.current = false; };\n window.addEventListener(\"offline\", stopCatalogRecovery);\n return () => window.removeEventListener(\"offline\", stopCatalogRecovery);\n }, []);\n\n const saveSession" + }, + { + "find": " setItems(data.items);", + "replace": " if (acceptCatalogUpdates.current) setItems(data.items);" + }, + { + "find": " socket.on(\"items:update\", (data: ItemT[]) => setItems(data));", + "replace": " socket.on(\"items:update\", (data: ItemT[]) => {\n if (acceptCatalogUpdates.current) setItems(data);\n });" + } + ] + }, + { + "id": "open-review-list-ignores-live-update", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "The already-open review list ignores a committed review update from another client.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItemDetail((prev) => (prev && prev.id === payload.itemId ? { ...prev, reviews: payload.reviews, average: payload.average } : prev));", + "replace": " void payload; // mutant: the already-open review list ignores committed updates" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Cancellation changes order state but skips restoration of its recorded warehouse allocations.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});", + "replace": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});" + } + ] + }, + { + "id": "cancellation-accounting-loses-stock-restoration", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});", + "replace": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c" + ], + "desc": "Cancellation changes order state but skips restoration, so a fresh client reads the persisted shortfall.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});", + "replace": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "Cancellation restores allocations but writes pending back to order history.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " value.status = \"cancelled\";\n await value.save({session});", + "replace": " value.status = \"pending\";\n await value.save({session});" + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Admin revenue includes cancelled orders even though cancellation otherwise succeeds.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },", + "replace": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: {} }," + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-invariant", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Admin revenue includes cancelled orders even though cancellation otherwise succeeds.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },", + "replace": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: {} }," + } + ] + }, + { + "id": "operator-authorization-allows-customer-transfer", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "The transfer route keeps authentication but drops its administrator role gate.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/admin/transfer\", requireAuth, requireAdmin, async (req, res) => {", + "replace": "app.post(\"/api/admin/transfer\", requireAuth, async (req, res) => {" + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping route keeps authentication but drops its staff role gate.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {", + "replace": "app.post(\"/api/fulfilment/ship\", requireAuth, async (req, res) => {" + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation retains authentication and pending-state validation but drops order ownership.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const value = orderId ? await Order.findOne({ _id: orderId, userId: user._id }).session(session) : null;\n if (!value) return null;\n if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");", + "replace": " const value = orderId ? await Order.findOne({ _id: orderId }).session(session) : null;\n if (!value) return null;\n if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");" + } + ] + }, + { + "id": "queue-depth-lags-one-order", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1a" + ], + "desc": "The queue renders every order but its visible depth remains one behind.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "Orders waiting: {queue.depth}", + "replace": "Orders waiting: {Math.max(0, queue.depth - 1)}" + } + ] + }, + { + "id": "ship-acknowledges-without-changing-status", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-ship.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1c" + ], + "desc": "Shipping returns success but writes pending back to the order, leaving both live views unchanged.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { $set: { status: \"shipped\" } },", + "replace": " { $set: { status: \"pending\" } }," + } + ] + }, + { + "id": "customer-sees-fulfilment-navigation", + "scenario": "tracks/ecommerce/scenarios/02-features.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {(currentUser?.isStaff || currentUser?.isAdmin) && (\n ", + "replace": " setPreference(value => ({ ...value, order: !value.order }))} />" + }, + { + "find": "function SupportTicket({ ticket, user, orders, act }: any) {", + "replace": "function OrderNotificationToggle({ on, onToggle }: { on: boolean; onToggle: () => void }) {\n const [shown] = useState(on); // mutant: the toggle keeps the state loaded when the card opened\n return ;\n}\n\nfunction SupportTicket({ ticket, user, orders, act }: any) {" + } + ] + }, + { + "id": "role-editor-snaps-back-to-stored-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.progression.staff-roles.staff-roles.621c" + ], + "desc": "Pressing Save persists the role but resets the dropdown to the role that was stored before the save, so the assignment is not visible until a reload.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " ", + "replace": " " + } + ] + }, + { + "id": "queue-ignores-live-fulfilment-updates", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.spec.live-state.fulfilment-queue.1a" + ], + "desc": "The open staff queue ignores live fulfilment events, so a new order appears only after a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => setFulfilmentQueue(data));", + "replace": " socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => { void data; }); // mutant: the open staff queue ignores live fulfilment updates" + } + ] + }, + { + "id": "low-stock-boundary-excludes-ten-live", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "The low-stock view uses a strict boundary, so an item that falls to exactly ten units never joins the list.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " .filter((it) => it.stock <= 10)", + "replace": " .filter((it) => it.stock < 10)" + } + ] + }, + { + "id": "live-admin-updates-keep-stale-category-totals", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.spec.live-state.sales-dashboard.5b" + ], + "desc": "Live admin updates keep the category totals loaded at page load, so a purchase does not move units or revenue until a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"admin:update\", (data: AdminOverviewT) => setAdminOverview(data));", + "replace": " socket.on(\"admin:update\", (data: AdminOverviewT) => setAdminOverview((previous) => previous ? { ...data, categories: previous.categories } : data)); // mutant: live admin updates keep the category totals loaded at page load" + } + ] + }, + { + "id": "overdraw-transfer-is-accepted", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c" + ], + "desc": "The atomic source debit no longer requires sufficient quantity, so an overdrawn transfer succeeds and moves both warehouse totals.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "{ item_id: itemId, warehouse_id: fromWarehouseId, quantity: { $gte: qty } }", + "replace": "{ item_id: itemId, warehouse_id: fromWarehouseId }" + } + ] + }, + { + "id": "transfer-totals-omit-destination-credit-live", + "scenario": "tracks/ecommerce/scenarios/02-transfer-totals.json", + "targets": [ + "ecommerce.spec.live-state.stock-transfers.2b" + ], + "desc": "A transfer debits the source but adds zero to the destination, so the two live warehouse totals do not move in opposite directions.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: qty } },\n { upsert: true }\n );", + "replace": " await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );" + } + ] + }, + { + "id": "credit-checkout-ignores-wallet", + "desc": "A credit checkout pays entirely externally despite available credit.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.feature.store-credit.store-credit-750.750a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " const creditMinor = useCredit ? Math.min(user.creditMinor, totalMinor) : 0;", + "replace": " const creditMinor = 0;" + } + ] + }, + { + "id": "credit-grant-replay-increments-balance", + "desc": "Replaying a grant applies its credit to the wallet again.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-752.752a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (existing.amountMinor !== amountMinor) throw new Error('Reference already identifies another grant');\n return;", + "replace": " if (existing.amountMinor !== amountMinor) throw new Error('Reference already identifies another grant');\n await User.updateOne({ _id: accountId }, { $inc: { creditMinor: amountMinor } }, { session });\n return;" + } + ] + }, + { + "id": "customer-can-grant-credit", + "desc": "Customer authentication is accepted without staff authorization.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-753.753a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " app.post('/api/staff/credit', auth, staff, async (req, res) => {", + "replace": " app.post('/api/staff/credit', auth, async (req, res) => {" + } + ] + }, + { + "id": "credit-checkout-retains-purchased-cart", + "desc": "A second checkout can reuse the purchased cart and create another order.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-754.754a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;", + "replace": " // mutant: purchased cart lines remain" + } + ] + }, + { + "id": "split-refund-does-not-restore-credit", + "desc": "The refund is recorded but its original wallet credit is not restored.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a" + ], + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await refundCredit(order, session);", + "replace": " // mutant: omit wallet restoration" + } + ] + }, + { + "id": "split-refund-duplicates-credit", + "desc": "A refund credits the wallet twice while recording one refund.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.spec.split-tender-refunds.production-756.756a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": "{ $inc: { creditMinor: delta } }, { session });", + "replace": "{ $inc: { creditMinor: delta * 2 } }, { session });" + } + ] + }, + { + "id": "subscription-skips-due-purchase", + "desc": "Due deliveries are recorded as skipped although stock is available.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.feature.subscriptions.subscriptions-760.760a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " const allocation = await reserveStock(row.itemId, row.quantity, session);", + "replace": " const allocation = row.quantity < 0 ? await reserveStock(row.itemId, row.quantity, session) : null;" + } + ] + }, + { + "id": "subscription-allows-foreign-cancellation", + "desc": "A customer can cancel another customer subscription.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-762.762a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " if (!row || String(row.userId) !== String((req as any).user._id)) return false;", + "replace": " if (!row) return false;" + } + ] + }, + { + "id": "subscription-pause-is-not-recorded", + "desc": "Pause acknowledges the request but the subscription remains active.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-763.763a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " row.status = 'paused'; row.pausedAt = new Date();", + "replace": " row.status = 'active'; row.pausedAt = new Date();" + } + ] + }, + { + "id": "credit-balance-is-cleared-at-startup", + "desc": "Restart clears an issued wallet balance while leaving accounts present.", + "file": "server/src/index.ts", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-755.755a" + ], + "edits": [ + { + "find": " await seed();", + "replace": " await seed();\n await User.updateMany({}, { $set: { creditMinor: 0 } });" + } + ] + }, + { + "id": "pending-subscriptions-are-cleared-at-startup", + "desc": "Restart erases pending subscription work while preserving ordinary timer execution.", + "file": "server/src/index.ts", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-761.761a" + ], + "edits": [ + { + "find": " await seed();", + "replace": " await seed();\n await mongoose.connection.collection(\"purchasesubscriptions\").deleteMany({ status: \"active\" });" + } + ] + }, + { + "id": "bundle-definition-loses-component-quantity", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.feature.product-bundles.product-bundles.740a" + ], + "desc": "definition loses component quantity", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "quantity: component.quantity });", + "replace": "quantity: 1 });" + } + ] + }, + { + "id": "bundle-catalog-write-allows-customers", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-743.743a" + ], + "desc": "catalog write allows customers", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "if (!actor.isAdmin && !actor.roles?.includes('catalog'))", + "replace": "if (false)" + } + ] + }, + { + "id": "bundle-checkout-price-not-snapshot", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.feature.bundle-checkout.bundle-checkout.741a" + ], + "desc": "checkout price not snapshot", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "bundlePrice: bundle.price,", + "replace": "bundlePrice: bundle.price + 1," + } + ] + }, + { + "id": "bundle-expiry-does-not-release-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-746.746a" + ], + "desc": "expiry does not release components", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "await releaseBundle(line.componentAllocations as Allocation[], session);\n line.componentAllocations = [] as any;", + "replace": "line.componentAllocations = [] as any; // mutant: component holds leak" + } + ] + }, + { + "id": "bundle-return-loses-original-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.feature.bundle-returns.bundle-returns.742a" + ], + "desc": "return loses original components", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "for (const line of bundles) { await releaseBundle(line.componentAllocations as Allocation[], session); line.returned = true; }", + "replace": "for (const line of bundles) { line.returned = true; }" + } + ] + }, + { + "id": "bundle-return-replay-restocks-again", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-742.742b" + ], + "desc": "return replay restocks again", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "line.isBundle && !line.returned", + "replace": "line.isBundle" + } + ] + }, + { + "id": "bundle-return-crosses-account-boundary", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-748.748a" + ], + "desc": "return crosses account boundary", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "{ _id: req.params.orderId, userId, status: { $in: ['shipped', 'delivered'] } }", + "replace": "{ _id: req.params.orderId, status: { $in: ['shipped', 'delivered'] } }" + } + ] + }, + { + "id": "bundle-components-can-overdraw", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-744.744a", + "ecommerce.spec.bundle-integrity.bundle-745.745a" + ], + "desc": "components can overdraw", + "file": "server/src/stock-reservations.ts", + "edits": [ + { + "find": "{ item_id: itemId, quantity: { $gte: 1 } }", + "replace": "{ item_id: itemId }" + } + ] + }, + { + "id": "bundle-checkout-reuses-reservation", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-747.747a" + ], + "desc": "checkout reuses reservation", + "file": "server/src/credit.ts", + "edits": [ + { + "find": "cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;", + "replace": "// mutant: active cart survives checkout" + } + ] + }, + { + "id": "return-after-support-refund-is-blocked", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757a" + ], + "desc": "Reject a valid physical return after a financial refund.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!line || line.returned) throw new Error('No returnable item found');", + "replace": " if (!line || line.returned || value.refundTotal > 0) throw new Error('No returnable item found');" + } + ] + }, + { + "id": "support-refund-after-return-pays-twice", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757b" + ], + "desc": "Pay the full order again after a physical return.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " order.refundTotal = order.total;", + "replace": " order.refundTotal += order.total;" + } + ] + }, + { + "id": "support-history-leaks-to-signed-out-visitors", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Disclose support tickets to signed-out visitors while leaving signed-in ownership filtering unchanged.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "const ticketFilter = isStaff ? {} : user ? { userId: user._id } : { _id: null };", + "replace": "const ticketFilter = isStaff ? {} : user ? { userId: user._id } : {};" + } + ] + }, + { + "id": "checkout-crash-integrity", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeOrderData(mongoose.connection);", + "replace": " await initializeOrderData(mongoose.connection);\n for (const cart of await Cart.find()) { const order = await Order.findOne({ userId: cart.userId }); if (order?.items[0]) await Cart.updateOne({ _id: cart._id }, { $set: { items: [{ itemId: order.items[0].itemId, quantity: 2 }] } }); }" + } + ] + }, + { + "id": "checkout-crash-sibling-stock", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A recovery callback removes stock for Coffee Grinder while leaving cart and order data intact. The second-product stock discrepancy must fail crash atomicity.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeOrderData(mongoose.connection);", + "replace": " await initializeOrderData(mongoose.connection);\n if (await Order.exists({})) { const item = await Item.findOne({ name: 'Coffee Grinder' }); if (item) await Stock.updateOne({ item_id: item._id }, { $inc: { quantity: -1 } }); }" + } + ] + }, + { + "id": "checkout-crash-durability", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-durability.910b" + ], + "desc": "A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeOrderData(mongoose.connection);", + "replace": " await initializeOrderData(mongoose.connection);\n await Order.updateMany({}, { $set: { total: 0 } });" + } + ] + }, + { + "id": "review-script-unsafe-render", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Render stored review text as HTML so it executes in another customer session.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "
{r.comment}
", + "replace": "
" + } + ] + }, + { + "id": "review-script-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Reject all nonempty reviews, including the required benign review; absence of script is insufficient.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const { rating, comment } = req.body || {};", + "replace": "const { rating, comment } = req.body || {};\n if (String(comment).length > 0) { res.status(400).json({ error: \"Review rejected\" }); return; }" + } + ] + }, + { + "id": "review-owner-trust-username", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Trust the submitted buyer username instead of the authenticated caller.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const user = (req as any).user;\n const hasPurchased = await Order.exists", + "replace": "const user = req.body?.username ? await User.findOne({ username: req.body.username }) : (req as any).user;\n const hasPurchased = await Order.exists" + } + ] + }, + { + "id": "review-owner-deny-after-write", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Replace the buyer review under a forged username, then return a refusal.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const user = (req as any).user;\n const hasPurchased = await Order.exists", + "replace": "const user = req.body?.username ? await User.findOne({ username: req.body.username }) : (req as any).user;\n const hasPurchased = await Order.exists" + }, + { + "find": "await broadcastReviews(String(itemId));", + "replace": "if (req.body?.username) return res.status(403).json({ error: \"Refused after write\" });\n await broadcastReviews(String(itemId));" + } + ] + }, + { + "id": "review-owner-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Reject the legitimate buyer review as well as unauthorized reviews.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const { rating, comment } = req.body || {};", + "replace": "const { rating, comment } = req.body || {};\n if (String(comment).length > 0) { res.status(403).json({ error: \"Reject all reviews\" }); return; }" + } + ] + }, + { + "id": "password-verification-truncates-utf8", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Hash and verify only the first 72 UTF-8 bytes, accepting a different password suffix.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "scrypt(password, salt, 64,", + "replace": "scrypt(Buffer.from(password).subarray(0, 72), salt, 64," + } + ] + }, + { + "id": "password-verification-rejects-every-login", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Refuse even a correct password; refusal alone must not earn credit.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "tampered-session-can-purchase", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Accept a modified session credential and allow its purchase.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "jwt.verify(token, JWT_SECRET) as { sub: string }", + "replace": "jwt.decode(token) as { sub: string }" + } + ] + }, + { + "id": "tampered-session-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Commit the forged purchase but return an access refusal; stored-state evidence must catch it.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "jwt.verify(token, JWT_SECRET) as { sub: string }", + "replace": "jwt.decode(token) as { sub: string }" + }, + { + "find": "res.json({ order: order.toJSON() });\n});\n\n// ---------------------------------------------------------------------------\n// Cart routes", + "replace": "try { jwt.verify(extractToken(req)!, JWT_SECRET); } catch { res.status(401); }\n res.json({ order: order.toJSON() });\n});\n\n// ---------------------------------------------------------------------------\n// Cart routes" + } + ] + }, + { + "id": "restock-trusts-submitted-admin-role", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust an administrator role supplied by staff in the restock request. Normal staff requests are still refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/admin/restock\", requireAuth, requireAdmin, async (req, res) => {", + "replace": "app.post(\"/api/admin/restock\", requireAuth, (req, res, next) => req.body?.role === \"admin\" ? next() : requireAdmin(req, res, next), async (req, res) => {" + } + ] + }, + { + "id": "direct-purchase-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Ordinary purchases use the catalog price; an extra price field overrides the booked order and line price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const user = (req as any).user;\n const order = await Order.create({", + "replace": " const user = (req as any).user;\n if (req.body?.price !== undefined) item.price = req.body.price;\n const order = await Order.create({" + } + ] + }, + { + "id": "order-cache-survives-account-switch", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Merge order updates into a browser cache that is never cleared when the account changes. Separate-browser ownership still works.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [orders, setOrders] = useState([]);", + "replace": " const [orders, setStoredOrders] = useState([]);\n const setOrders: typeof setStoredOrders = update => setStoredOrders(previous => {\n const incoming = typeof update === 'function' ? update(previous) : update;\n return [...new Map([...previous, ...incoming].map(order => [order.id, order])).values()];\n });" + } + ] + }, + { + "id": "checkout-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Book the submitted checkout price instead of the stored price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const order = await checkoutAtomic(userId);", + "replace": " const order = await checkoutAtomic(userId);\n if (req.body?.price !== undefined) { order.total = req.body.price; await order.save(); }" + } + ] + }, + { + "id": "checkout-missing-sibling-line", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit the cart checkout with one product line missing while retaining the other product and the full order total.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const order = await checkoutAtomic(userId);", + "replace": " const order = await checkoutAtomic(userId);\n order.items = order.items.filter(line => line.name !== 'Coffee Grinder') as any; await order.save();" + } + ] + }, + { + "id": "shipping-cancellation-cancel-shipped", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation accepts a shipped order and restores its stock after shipping.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (value.status !== \"pending\")", + "replace": "if (false)" + } + ] + }, + { + "id": "shipping-cancellation-resurrect-cancelled", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Shipping accepts a cancelled order after its stock was restored.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (order.status !== \"pending\")", + "replace": "if (false)" + }, + { + "find": " { _id: orderId, status: \"pending\" },", + "replace": " { _id: orderId }," + } + ] + }, + { + "id": "shipping-cancellation-combined-race", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation restores stock but records a shipped status, combining mutually exclusive effects.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (value.status !== \"pending\")", + "replace": "if (false)" + }, + { + "find": "value.status = \"cancelled\";", + "replace": "value.status = \"shipped\";" + } + ] + }, + { + "id": "shipping-cancellation-reject-both", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Both cancellation and shipping refuse pending orders; useful progress must remain required.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (value.status !== \"pending\")", + "replace": "if (true)" + }, + { + "find": "if (order.status !== \"pending\")", + "replace": "if (true)" + } + ] + }, + { + "id": "shipping-cancellation-queue-retains-removed-order", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "The live staff queue accepts new orders but retains rows after cancellation; the database state remains correct.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => setFulfilmentQueue(data));", + "replace": " socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => setFulfilmentQueue(previous => data.orders.length < previous.orders.length ? previous : data));" + } + ] + }, + { + "id": "opposing-transfer-ignored", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is acknowledged without moving stock.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body || {};", + "replace": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body || {};\n if (Number(quantity) === 7) return res.json({});" + } + ] + }, + { + "id": "opposing-transfer-rejected", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is refused despite ample source stock.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body || {};", + "replace": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body || {};\n if (Number(quantity) === 7) return res.status(409).json({ error: \"Transfer disabled\" });" + } + ] + }, + { + "id": "overlapping-cart-add-ignored", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is ignored while ordinary cart setup still works.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);", + "replace": " if ((req as any).user.username.includes(\"checkout-overlap\") && item.name === \"Coffee Grinder\") return res.json({});\n const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);" + } + ] + }, + { + "id": "overlapping-cart-add-rejected", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is rejected while ordinary cart setup still works.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);", + "replace": " if ((req as any).user.username.includes(\"checkout-overlap\") && item.name === \"Coffee Grinder\") return res.status(409).json({ error: \"Cart add disabled\" });\n const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);" + } + ] + }, + { + "id": "mixed-history-lost-transfer", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: lost-transfer. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: qty } },\n { upsert: true }\n );", + "replace": " await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );" + } + ] + }, + { + "id": "mixed-history-wrong-price", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-price. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: direct purchase ignores the authoritative price\n });" + } + ] + }, + { + "id": "mixed-history-ignored-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: ignored-write. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);", + "replace": " if ((req as any).user.username.includes(\"history-\") && item.name === \"Coffee Grinder\") return res.json({});\n const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);" + } + ] + }, + { + "id": "mixed-history-reject-valid-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: reject-valid-write. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);", + "replace": " if ((req as any).user.username.includes(\"history-\") && item.name === \"Coffee Grinder\") return res.status(409).json({ error: \"Cart add disabled\" });\n const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);" + } + ] + }, + { + "id": "mixed-history-repeat-cancel", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: repeat-cancel. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");", + "replace": " if (![\"pending\", \"cancelled\"].includes(value.status)) throw new Error(\"Order cannot be cancelled\");" + } + ] + }, + { + "id": "mixed-history-wrong-owner", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-owner. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " userId: user._id,\n items: [", + "replace": " userId: (await User.findOne({ _id: { $ne: user._id } }))!._id,\n items: [" + } + ] + }, + { + "id": "checkout-empty-replay-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Allow retrying the consumed empty cart to create another order after a lost reply.", + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!cart?.items.length) throw new Error('Cart is empty');", + "replace": " if (!cart) throw new Error('Cart is missing');" + }, + { + "find": " if (!active.length) throw new Error('Reservation expired');", + "replace": "// Defect: an empty consumed cart may create another order." + } + ] + }, + { + "id": "checkout-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Reject every valid checkout; refusal cannot earn duplicate protection credit.", + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!cart?.items.length) throw new Error('Cart is empty');", + "replace": " if (!cart?.items.length) throw new Error('Cart is empty');\n throw new Error('Checkout disabled');" + } + ] + }, + { + "id": "checkout-replay-loses-prior-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Retrying the consumed cart deletes an earlier acknowledged order.", + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!cart?.items.length) throw new Error('Cart is empty');", + "replace": " if (!cart?.items.length) { const prior = await Order.findOne({ userId }).sort({ _id: 1 }).session(session); if (prior) await prior.deleteOne({ session }); return null; }" + } + ] + }, + { + "id": "checkout-clears-cart-without-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout returns success and empties the cart without recording an order.", + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!cart?.items.length) throw new Error('Cart is empty');", + "replace": " if (!cart?.items.length) throw new Error('Cart is empty');\n if (cart.items.length > 0) { cart.items = [] as any; await cart.save({ session }); return null; }" + } + ] + }, + { + "id": "revenue-aggregation-truncates-order-population", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Calculate revenue from only the first 256 stored orders while preserving all acknowledged purchases.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([", + "replace": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $limit: 256 }," + } + ] + }, + { + "id": "catalog-truncates-created-population", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Omit committed products after the first 256 search results.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".slice(searchPage * CATALOG_PAGE_SIZE,", + "replace": ".slice(0, 256).slice(searchPage * CATALOG_PAGE_SIZE," + } + ] + }, + { + "id": "catalog-discards-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Acknowledge catalog creation without committing a product.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " router.post(\"/catalog\", auth, staff, async (req: Request, res) => {", + "replace": " router.post(\"/catalog\", auth, staff, async (req: Request, res) => {\n if (req.body) { res.json({ok:true}); return; } // mutant: successful response without product" + } + ] + }, + { + "id": "catalog-rejects-all-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Reject every catalog creation while keeping sign-in and catalog reads available.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " router.post(\"/catalog\", auth, staff, async (req: Request, res) => {", + "replace": " router.post(\"/catalog\", auth, staff, async (req: Request, res) => {\n res.status(403).json({error:'Catalog writes refused'}); return;" + } + ] + }, + { + "id": "shipping-trusts-submitted-staff-role", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Trust a staff role supplied by a customer; ordinary customers remain refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {", + "replace": "app.post(\"/api/fulfilment/ship\", requireAuth, (req, res, next) => req.body?.role === \"staff\" ? next() : requireStaff(req, res, next), async (req, res) => {" + } + ] + }, + { + "id": "shipping-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Reject legitimate staff shipping as well as customer requests.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {", + "replace": "app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {\n return res.status(403).json({ error: \"Shipping refused\" });" + } + ] + }, + { + "id": "cancel-trusts-submitted-owner", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Trust a username submitted by another signed-in customer as the order owner.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (req as any).user;", + "replace": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (typeof req.body?.username === 'string' ? await User.findOne({username:req.body.username}) : null) ?? (req as any).user;" + } + ] + }, + { + "id": "cancel-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Reject legitimate cancellation as well as foreign requests.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (req as any).user;", + "replace": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (req as any).user;\n return res.status(403).json({error:\"Cancellation refused\"});" + } + ] + }, + { + "id": "account-password-in-public-data", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Expose the actual submitted account password through data available to other clients.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "const app = express();", + "replace": "const app = express();\nconst leakedPasswords: string[] = [];\napp.use((_req, res, next) => { const json = res.json.bind(res); res.json = (body: any) => json({ ...body, debugCredentials: leakedPasswords }); next(); });" + }, + { + "file": "server/src/index.ts", + "find": "const passwordHash = await hashPassword(password);", + "replace": "const passwordHash = await hashPassword(password);\n leakedPasswords.push(password);" + } + ] + }, + { + "id": "account-disclosure-rejects-login", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Reject every valid login; absent disclosures alone must not earn a pass.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "signout-keeps-session", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Show a signed-out interface while retaining the browser credential and its purchase authority.", + "edits": [ + { + "file": "client/src/App.tsx", + "find": " const clearSession = () => {\n localStorage.removeItem(TOKEN_KEY);", + "replace": " const clearSession = () => {\n // mutant: retain the current request token after signout" + } + ] + }, + { + "id": "query-like-password-bypasses-signin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Interpret the query-like password as a login bypass while retaining normal password verification.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const valid = await verifyPassword(password, user.passwordHash);", + "replace": " const valid = password === \"' OR '1'='1\" || await verifyPassword(password, user.passwordHash);" + } + ] + }, + { + "id": "purchase-session-rejects-returning-login", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Refuse every returning login; blocking all access after logout must not pass the session check.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "rejected-query-login-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Create an order during the query-like login, leave stock unchanged and still refuse access; a refusal and stock-only observation must not earn a pass.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const valid = await verifyPassword(password, user.passwordHash);", + "replace": " if (password === \"' OR '1'='1\") {\n const item = await Item.findOne({ name: \"Bluetooth Speaker\" });\n if (!item) throw new Error(\"mutation requires the seeded item\");\n await Order.create({ userId: user._id, items: [{ itemId: item._id, name: item.name, price: item.price, quantity: 1, allocations: [] }], total: item.price });\n }\n const valid = await verifyPassword(password, user.passwordHash);" + } + ] + }, + { + "id": "account-password-in-script", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in the public JavaScript bundle while normal signup and login still work.", + "edits": [ + { + "file": "client/src/main.tsx", + "find": "import React from \"react\";", + "replace": "import React from \"react\";\nObject.assign(window, { debugPassword: 'sb-private-pw-76dc492ea153' });" + } + ] + }, + { + "id": "account-password-in-style", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in a public stylesheet while normal signup and login still work.", + "edits": [ + { + "file": "client/src/index.css", + "find": ":root {", + "replace": ":root {\n --debug-password: \"sb-private-pw-76dc492ea153\";" + } + ] + }, + { + "id": "signup-accepts-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust a submitted signup administrator claim and grant protected stock-write authority.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "User.create({ username: username.trim(), passwordHash, isAdmin: false })", + "replace": "User.create({ username: username.trim(), passwordHash, isAdmin: req.body.role === 'admin' })" + } + ] + }, + { + "id": "admin-boundary-rejects-all-signups", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Refuse every new registration; blocking useful signup must not pass the authority check.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "app.post(\"/api/auth/signup\", async (req, res) => {", + "replace": "app.post(\"/api/auth/signup\", async (req, res) => {\n return res.status(403).json({ error: \"Registration disabled\" });" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/mutations/postgres-ecommerce.json b/tools/stack-bench/grader/mutations/postgres-ecommerce.json new file mode 100644 index 00000000000..5539680a6bd --- /dev/null +++ b/tools/stack-bench/grader/mutations/postgres-ecommerce.json @@ -0,0 +1,3501 @@ +{ + "schemaVersion": 3, + "fixtureSha256": "d7c162ae456010e5d3b334ac839c445ff5c9c889d83bbc94b9d840b0042534a5", + "backend": "postgres", + "track": "ecommerce", + "note": "Mutation definitions for the PostgreSQL ecommerce reference.", + "mutations": [ + { + "id": "recommendation-dismissal-lost-on-restart", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.spec.state-durability.recommendation-feedback-restart.504c" + ], + "desc": "Erase saved recommendation dismissals when the application starts again.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeProgressionSchema(pool);", + "replace": " await initializeProgressionSchema(pool);\n await pool.query('DELETE FROM recommendation_dismissal');" + } + ] + }, + { + "id": "pending-order-item-return-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-order-return-boundary.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3f" + ], + "desc": "Accept a pending order return and restore its stock before shipment.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (!['shipped', 'delivered'].includes(orderRow.rows[0].status)) {", + "replace": "if (!['pending', 'shipped', 'delivered'].includes(orderRow.rows[0].status)) {" + } + ] + }, + { + "id": "staff-admin-access-survives-role-removal", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Keep administrator access after changing the assigned role back to staff.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "is_admin = ($1 = 'admin')", + "replace": "is_admin = (is_admin OR $1 = 'admin')" + } + ] + }, + { + "id": "shipping-counts-sale-twice", + "scenario": "tracks/ecommerce/scenarios/progression-shipping-accounting.json", + "targets": [ + "ecommerce.inventory-operations.shipping-accounting.202e" + ], + "desc": "Shipping succeeds but doubles the completed sale value in authoritative revenue.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "`UPDATE orders SET status = 'shipped', shipped_at = now()\n WHERE id = $1 AND status = 'pending' RETURNING account_id`", + "replace": "`UPDATE orders SET status = 'shipped', shipped_at = now(), total = total * 2\n WHERE id = $1 AND status = 'pending' RETURNING account_id`" + } + ] + }, + { + "id": "signup-ui-does-not-enter-created-account", + "scenario": "tracks/ecommerce/scenarios/01-account-create.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Create the account successfully but discard the returned signed-in identity in the client.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const r = await api<{ account: Account }>(\"/api/auth/signup\", {\n method: \"POST\",\n body: JSON.stringify({ username, password }),\n });\n setAccount(r.account);", + "replace": " const r = await api<{ account: Account }>(\"/api/auth/signup\", {\n method: \"POST\",\n body: JSON.stringify({ username, password }),\n });\n void r;\n setAccount(null);" + } + ] + }, + { + "id": "duplicate-signup-authenticates-existing-account", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "Treat a duplicate signup as a successful session for the pre-existing account.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (existing.length > 0) {\n res.status(409).json({ error: \"username already taken\" });\n return;\n }", + "replace": " if (existing.length > 0) {\n const token = newToken();\n await db.insert(session).values({ id: token, accountId: existing[0].id });\n res.cookie(\"sid\", token, { httpOnly: true, sameSite: \"lax\", path: \"/\" });\n res.json({ account: { id: existing[0].id, username, isAdmin: false, isStaff: false } });\n return;\n }" + } + ] + }, + { + "id": "signin-skips-password-verification", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Accept an existing account regardless of its password; valid login remains possible.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (rows.length === 0 || !(await verifyPassword(password, rows[0].passwordHash))) {", + "replace": " if (rows.length === 0) {" + } + ] + }, + { + "id": "correct-signin-is-refused", + "scenario": "tracks/ecommerce/scenarios/01-account-signout.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Preserve wrong-password refusal but reject an otherwise valid sign-in, preventing a signed-out account from returning.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const acc = rows[0];\n const token = newToken();", + "replace": " const acc = rows[0];\n if (username === acc.username) {\n res.status(401).json({ error: \"sign in is unavailable\" });\n return;\n }\n const token = newToken();" + } + ] + }, + { + "id": "reload-discards-session-identity", + "scenario": "tracks/ecommerce/scenarios/01-account-reload.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "Ignore the authenticated identity returned during initial page hydration.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setAccount(me.account);", + "replace": " setAccount(null);" + } + ] + }, + { + "id": "purchase-does-not-broadcast-ranking", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "Commit the purchase but omit the catalog broadcast that updates already-open rankings.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });", + "replace": " lastCatalogJson = json;\n // mutant: changed catalog state is not broadcast" + } + ] + }, + { + "id": "signed-out-purchase-uses-default-account", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "Expose purchase controls to guests and let the purchase route charge the first stored account when no caller is authenticated.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const canBuy = !!account && !account.isAdmin && !account.isStaff;", + "replace": " const canBuy = !account || (!account.isAdmin && !account.isStaff);" + }, + { + "file": "server/src/index.ts", + "find": " \"/api/items/:id/buy\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": " \"/api/items/:id/buy\",\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account?.id ?? (await pool.query(`SELECT id FROM account ORDER BY id LIMIT 1`)).rows[0].id;" + } + ] + }, + { + "id": "purchase-stock-change-is-not-broadcast--01-buying", + "scenario": "tracks/ecommerce/scenarios/01-buying.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "Commit purchases without broadcasting their stock changes, breaking live stock and sold-out visibility.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });", + "replace": " lastCatalogJson = json;\n // mutant: changed stock is not broadcast" + } + ] + }, + { + "id": "purchase-stock-change-is-not-broadcast--stock-limit", + "scenario": "tracks/ecommerce/scenarios/progression-stock-limit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.stock-limit.3d" + ], + "desc": "Commit purchases without broadcasting their stock changes, breaking live stock and sold-out visibility.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });", + "replace": " lastCatalogJson = json;\n // mutant: changed stock is not broadcast" + } + ] + }, + { + "id": "restock-race-records-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [accountId, price]\n );", + "replace": " [accountId, Number(price) + 1]\n );" + } + ] + }, + { + "id": "direct-purchase-order-total-is-offset", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "Record a direct purchase one dollar above the stored price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [accountId, price]\n );", + "replace": " [accountId, Number(price) + 1]\n );" + } + ] + }, + { + "id": "reload-hydrates-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Return an empty cart from both reload hydration paths while preserving later live cart broadcasts.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const state = await buildCartState(req.account!.id);\n res.json(state);", + "replace": " await buildCartState(req.account!.id);\n res.json({ items: [], total: 0 });" + }, + { + "find": " const cartState = await buildCartState(acc.id);\n socket.emit(\"cart:update\", cartState);", + "replace": " await buildCartState(acc.id);\n socket.emit(\"cart:update\", { items: [], total: 0 });" + } + ] + }, + { + "id": "signed-out-visitors-do-not-see-reviews", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Hide an item's reviews whenever the viewer is signed out.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {reviews.length === 0 ? (", + "replace": " {!account || reviews.length === 0 ? (" + } + ] + }, + { + "id": "review-average-update-is-not-broadcast", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "Return the new average to the submitter but omit the live review update to other viewers.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " io.emit(\"review:update\", { itemId, reviews, average });", + "replace": " // mutant: other open review views do not receive the new average" + } + ] + }, + { + "id": "admin-warehouse-view-drops-one-location", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "Render only 23 of the 24 item-by-warehouse stock locations.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {admin.locations.map((loc) => {", + "replace": " {admin.locations.slice(0, 23).map((loc) => {" + } + ] + }, + { + "id": "unauthenticated-direct-purchase-uses-default-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Remove purchase authentication and attribute unauthenticated requests to a default account.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " \"/api/items/:id/buy\",\n requireAuth,", + "replace": " \"/api/items/:id/buy\"," + }, + { + "find": " const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();", + "replace": " const itemId = Number(req.params.id);\n const accountId = req.account?.id ?? 1;\n\n const client = await pool.connect();" + } + ] + }, + { + "id": "direct-purchase-is-attributed-to-previous-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-attribution.json", + "targets": [ + "ecommerce.spec.access-control.purchase-attribution.102a" + ], + "desc": "Create the direct-purchase order for the preceding account id rather than the authenticated caller.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();", + "replace": " const itemId = Number(req.params.id);\n const accountId = req.account!.id - 1;\n\n const client = await pool.connect();" + } + ] + }, + { + "id": "direct-purchase-uses-constant-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Create a direct-purchase order at a hard-coded price instead of the current stored price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [accountId, price]\n );", + "replace": " [accountId, \"1.00\"]\n );" + } + ] + }, + { + "id": "account-state-reload-discards-session", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "Discard the authenticated account during reload hydration, making its cart and orders unavailable.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setAccount(me.account);", + "replace": " setAccount(null);" + } + ] + }, + { + "id": "offline-event-clears-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "Treat a temporary offline event as a sign-out and clear the account and cart state.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " useEffect(() => {\n const socket = io({ path: \"/socket.io\" });", + "replace": " useEffect(() => {\n const clearAccountOffline = () => {\n setAccount(null);\n setCart({ items: [], total: 0 });\n };\n window.addEventListener(\"offline\", clearAccountOffline, { once: true });\n const socket = io({ path: \"/socket.io\" });" + } + ] + }, + { + "id": "purchase-does-not-decrement-warehouse-stock", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107b" + ], + "desc": "Create purchase orders without decrementing their selected warehouse stock row.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " UPDATE stock s SET quantity = quantity - 1\n FROM target t", + "replace": " UPDATE stock s SET quantity = quantity\n FROM target t" + } + ] + }, + { + "id": "review-route-skips-purchase-eligibility", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "Allow review creation even when the caller has never purchased the item.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (purchased.rowCount === 0) {", + "replace": " if (false && purchased.rowCount === 0) {" + } + ] + }, + { + "id": "only-shipped-orders-earn-review-eligibility", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "Incorrectly require an order to be shipped before its buyer may review the item. The same restriction also rejects the required successful buyer control in 108a; it does not independently test non-buyer denial.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " WHERE o.account_id = $1 AND oi.item_id = $2 LIMIT 1`,", + "replace": " WHERE o.account_id = $1 AND oi.item_id = $2 AND o.status = 'shipped' LIMIT 1`," + } + ] + }, + { + "id": "cart-update-accepts-negative-quantity", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109b" + ], + "desc": "Accept a negative cart quantity update and persist it instead of refusing the named action.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!Number.isInteger(quantity) || quantity < 1) {", + "replace": " if (!Number.isInteger(quantity)) {" + } + ] + }, + { + "id": "oversell-no-row-lock", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Drop the row lock so simultaneous buyers can select the same remaining units.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " FOR UPDATE\n LIMIT 1", + "replace": " LIMIT 1" + } + ] + }, + { + "id": "purchase-read-write-loses-concurrent-stock", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Replace atomic stock reservation with an unlocked read and absolute write. A fixed pause widens scheduling overlap; serial purchases and restocks retain their stock effects. Concurrent reservations or restocking can lose updates.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const decrement = await client.query(\n `WITH target AS (\n SELECT item_id, warehouse_id FROM stock\n WHERE item_id = $1 AND quantity > 0\n ORDER BY warehouse_id\n FOR UPDATE\n LIMIT 1\n )\n UPDATE stock s SET quantity = quantity - 1\n FROM target t\n WHERE s.item_id = t.item_id AND s.warehouse_id = t.warehouse_id\n RETURNING s.item_id, s.warehouse_id`,\n [itemId]\n );\n", + "replace": " const snapshot = await client.query(\n `SELECT item_id, warehouse_id, quantity FROM stock\n WHERE item_id = $1 AND quantity > 0 ORDER BY warehouse_id LIMIT 1`, [itemId]\n );\n // Mutant: widen the unlocked read/write window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n const decrement = snapshot.rowCount === 0 ? snapshot : await client.query(\n `UPDATE stock SET quantity = $3 WHERE item_id = $1 AND warehouse_id = $2\n RETURNING item_id, warehouse_id`,\n [itemId, snapshot.rows[0].warehouse_id, snapshot.rows[0].quantity - 1]\n );\n" + } + ] + }, + { + "id": "external-stock-polling-disabled", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "Stop reconciling direct database changes while the server remains online.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " broadcastCatalog().catch((err) => console.error(\"poll broadcast failed\", err));", + "replace": " // mutant: direct database catalog changes are never reconciled" + } + ] + }, + { + "id": "server-restart-does-not-resynchronize-catalog", + "scenario": "tracks/ecommerce/scenarios/01-external-server-restart-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901c" + ], + "desc": "After a server restart, omit both connection-time catalog hydration and periodic authoritative reconciliation.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const catalog = await buildCatalog();\n socket.emit(\"items:update\", { items: catalog });", + "replace": " // mutant: reconnecting clients retain their pre-restart catalog" + }, + { + "find": " broadcastCatalog().catch((err) => console.error(\"poll broadcast failed\", err));", + "replace": " // mutant: restart recovery does not reconcile authoritative catalog state" + } + ] + }, + { + "id": "reconnect-does-not-send-current-catalog", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "Follow catalog changes until the browser goes offline, then ignore updates after restoration.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const cartObservationRef = useRef(0);\n\n function applyCartResponse", + "replace": " const cartObservationRef = useRef(0);\n const acceptCatalogUpdates = useRef(true);\n useEffect(() => {\n const stopCatalogRecovery = () => { acceptCatalogUpdates.current = false; };\n window.addEventListener(\"offline\", stopCatalogRecovery);\n return () => window.removeEventListener(\"offline\", stopCatalogRecovery);\n }, []);\n\n function applyCartResponse" + }, + { + "find": " socket.on(\"items:update\", (payload: { items: Item[] }) => setItems(payload.items));", + "replace": " socket.on(\"items:update\", (payload: { items: Item[] }) => {\n if (acceptCatalogUpdates.current) setItems(payload.items);\n });" + } + ] + }, + { + "id": "open-review-list-ignores-live-update", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "Ignore committed review updates in a detail view that is already open.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItemDetail({ reviews: payload.reviews, average: payload.average });", + "replace": " // mutant: the already-open review list ignores committed updates" + } + ] + }, + { + "id": "open-review-list-renders-each-review-twice", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "Render every committed review twice in the already-open list.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " reviews.map((r) => (", + "replace": " [...reviews, ...reviews].map((r) => (" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Cancellation commits but restores zero units to each recorded warehouse row.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);", + "replace": " [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);" + } + ] + }, + { + "id": "cancellation-accounting-loses-stock-restoration", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);", + "replace": " [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c" + ], + "desc": "Cancellation commits but restores zero units, so a fresh client reads the persisted shortfall.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);", + "replace": " [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "Cancellation restores allocations but writes pending back to order history.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);", + "replace": "await client.query(`UPDATE orders SET status = 'pending' WHERE id = $1`, [orderId]);" + } + ] + }, + { + "id": "operator-authorization-allows-customer-transfer", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "The transfer route replaces its administrator gate with ordinary authentication.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/admin/transfer\",\n requireAdmin,", + "replace": "app.post(\n \"/api/admin/transfer\",\n requireAuth," + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping route replaces its staff gate with ordinary authentication.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/fulfilment/ship\",\n requireStaff,", + "replace": "app.post(\n \"/api/fulfilment/ship\",\n requireAuth," + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation retains authentication and pending-state validation but drops order ownership.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const orderRow = await client.query(`SELECT * FROM orders WHERE id = $1 FOR UPDATE`, [orderId]);\n if (orderRow.rowCount === 0 || orderRow.rows[0].account_id !== accountId) {", + "replace": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const orderRow = await client.query(`SELECT * FROM orders WHERE id = $1 FOR UPDATE`, [orderId]);\n if (orderRow.rowCount === 0) {" + } + ] + }, + { + "id": "queue-depth-lags-one-order", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1a" + ], + "desc": "The queue renders every order but its visible depth remains one behind.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "{queue.depth}", + "replace": "{Math.max(0, queue.depth - 1)}" + } + ] + }, + { + "id": "customer-sees-fulfilment-navigation", + "scenario": "tracks/ecommerce/scenarios/02-features.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {account && (account.isStaff || account.isAdmin) && (\n
)}", + "replace": "
)}" + } + ] + }, + { + "id": "purchase-does-not-broadcast-fulfilment-queue", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.spec.live-state.fulfilment-queue.1a" + ], + "desc": "A direct purchase commits its pending order but omits the fulfilment queue broadcast, so an open staff queue never learns about the new order.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await broadcastCatalog();\n await broadcastOrders(accountId);\n await broadcastFulfilment();\n await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes ----------", + "replace": " await broadcastCatalog();\n await broadcastOrders(accountId);\n // mutant: the new pending order is not pushed to open fulfilment queues\n await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes ----------" + } + ] + }, + { + "id": "admin-state-change-is-not-broadcast", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "Changed admin dashboard state is never broadcast to open admin views, so a customer purchase that drops an item to ten units does not re-enter the low-stock list live.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " lastAdminJson = adminJson;\n io.to(\"admin\").emit(\"admin:update\", adminState);", + "replace": " lastAdminJson = adminJson;\n // mutant: changed admin state is not broadcast" + } + ] + }, + { + "id": "admin-sockets-do-not-join-admin-room", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.spec.live-state.sales-dashboard.5b" + ], + "desc": "Admin sockets receive their dashboard state on connection but never join the admin room, so a customer purchase does not update the open category totals live.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (acc.isAdmin) socket.join(\"admin\");", + "replace": " // mutant: admin sockets never join the admin room" + } + ] + }, + { + "id": "transfer-overdraft-guard-skips-bulk-transfers", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c" + ], + "desc": "The insufficient-stock guard is only evaluated for transfers under 1000 units, so a bulk transfer that overdraws the source warehouse commits instead of being refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (available < qty) {", + "replace": " if (available < qty && qty < 1000) {" + } + ] + }, + { + "id": "transfer-does-not-publish-warehouse-totals", + "scenario": "tracks/ecommerce/scenarios/02-transfer-totals.json", + "targets": [ + "ecommerce.spec.live-state.stock-transfers.2b" + ], + "desc": "A transfer commits but answers with the pre-transfer admin snapshot and admin state is never broadcast, so the open warehouse totals do not move.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n // Lock both warehouse rows", + "replace": " const state = await buildAdminState();\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n // Lock both warehouse rows" + }, + { + "find": " await broadcastCatalog();\n const state = await buildAdminState();\n res.json(state);\n })\n);\n\napp.post(\n \"/api/admin/price\",", + "replace": " await broadcastCatalog();\n res.json(state);\n })\n);\n\napp.post(\n \"/api/admin/price\"," + }, + { + "find": " lastAdminJson = adminJson;\n io.to(\"admin\").emit(\"admin:update\", adminState);", + "replace": " lastAdminJson = adminJson;\n // mutant: changed admin state is not broadcast" + } + ] + }, + { + "id": "credit-checkout-ignores-wallet", + "desc": "A credit checkout pays entirely externally despite available credit.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.feature.store-credit.store-credit-750.750a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " const creditMinor = Math.min(Number(account.rows[0].credit_minor), totalMinor);", + "replace": " const creditMinor = 0;" + } + ] + }, + { + "id": "credit-grant-replay-increments-balance", + "desc": "Replaying a grant applies its credit to the wallet again.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-752.752a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!existing.rows.length) {", + "replace": " if (existing.rows.length) await client.query('UPDATE account SET credit_minor=credit_minor+$1 WHERE id=$2', [amountMinor, accountId]);\n if (!existing.rows.length) {" + } + ] + }, + { + "id": "customer-can-grant-credit", + "desc": "Customer authentication is accepted without staff authorization.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-753.753a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " app.post('/api/staff/credit', auth, staff, async (req, res) => {", + "replace": " app.post('/api/staff/credit', auth, async (req, res) => {" + } + ] + }, + { + "id": "split-refund-does-not-restore-credit", + "desc": "The refund is recorded but its original wallet credit is not restored.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a" + ], + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await refundCredit(client, order.rows[0]);", + "replace": " // mutant: omit wallet restoration" + } + ] + }, + { + "id": "split-refund-duplicates-credit", + "desc": "A refund credits the wallet twice while recording one refund.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.spec.split-tender-refunds.production-756.756a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": "[delta, order.account_id]);", + "replace": "[delta * 2, order.account_id]);" + } + ] + }, + { + "id": "subscription-skips-due-purchase", + "desc": "Due deliveries are recorded as skipped although stock is available.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.feature.subscriptions.subscriptions-760.760a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " if (stock.rows.reduce((sum, row) => sum + row.quantity, 0) >= subscription.quantity) {", + "replace": " if (false && stock.rows.reduce((sum, row) => sum + row.quantity, 0) >= subscription.quantity) {" + } + ] + }, + { + "id": "subscription-allows-foreign-cancellation", + "desc": "A customer can cancel another customer subscription.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-762.762a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " if (!row || row.account_id !== req.account!.id) {", + "replace": " if (!row) {" + } + ] + }, + { + "id": "subscription-pause-is-not-recorded", + "desc": "Pause acknowledges the request but the subscription remains active.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-763.763a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": "UPDATE purchase_subscription SET status='paused',paused_at=now() WHERE id=$1", + "replace": "UPDATE purchase_subscription SET status='active',paused_at=now() WHERE id=$1" + } + ] + }, + { + "id": "credit-checkout-retains-purchased-cart", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-754.754a" + ], + "desc": "The purchased cart remains available instead of being consumed by checkout.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);", + "replace": " // Mutation: keep checked-out cart lines." + } + ] + }, + { + "id": "pending-subscriptions-are-cleared-at-startup", + "desc": "Restart erases pending subscription work while preserving ordinary timer execution.", + "file": "server/src/subscriptions.ts", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-761.761a" + ], + "edits": [ + { + "find": " `);\n}\n\nexport function registerSubscriptions", + "replace": " `);\n await pool.query(\"UPDATE purchase_subscription SET status='cancelled' WHERE status='active'\");\n}\n\nexport function registerSubscriptions" + } + ] + }, + { + "id": "credit-balance-is-cleared-at-startup", + "desc": "Restart clears an issued wallet balance while leaving accounts present.", + "file": "server/src/credit.ts", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-755.755a" + ], + "edits": [ + { + "find": " `);\n}\n\nexport async function spendCredit", + "replace": " `);\n await pool.query(\"UPDATE account SET credit_minor=0\");\n}\n\nexport async function spendCredit" + } + ] + }, + { + "id": "bundle-definition-loses-component-quantity", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.feature.product-bundles.product-bundles.740a" + ], + "desc": "definition loses component quantity", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "values.push({ ...component, itemId: item.rows[0].id });", + "replace": "values.push({ ...component, quantity: 1, itemId: item.rows[0].id });" + } + ] + }, + { + "id": "bundle-catalog-write-allows-customers", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-743.743a" + ], + "desc": "catalog write allows customers", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "if (!actor?.is_admin && actor?.staff_role !== 'catalog')", + "replace": "if (false)" + } + ] + }, + { + "id": "bundle-checkout-price-not-snapshot", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.feature.bundle-checkout.bundle-checkout.741a" + ], + "desc": "checkout price not snapshot", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "[cart.rows[0].id, bundleId, bundle.price, JSON.stringify(allocations), JSON.stringify(bundle.bundle_components)]", + "replace": "[cart.rows[0].id, bundleId, bundle.price + 1, JSON.stringify(allocations), JSON.stringify(bundle.bundle_components)]" + } + ] + }, + { + "id": "bundle-expiry-does-not-release-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-746.746a" + ], + "desc": "expiry does not release components", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "await releaseBundle(client, bundle.rows[0].component_allocations);", + "replace": "/* mutant: component holds leak after expiration */" + } + ] + }, + { + "id": "bundle-return-loses-original-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.feature.bundle-returns.bundle-returns.742a" + ], + "desc": "return loses original components", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "await releaseBundle(client, line.component_allocations);", + "replace": "/* mutant: purchased components are not restored */" + } + ] + }, + { + "id": "bundle-return-replay-restocks-again", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-742.742b" + ], + "desc": "return replay restocks again", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "WHERE order_id=$1 AND is_bundle AND NOT returned FOR UPDATE", + "replace": "WHERE order_id=$1 AND is_bundle FOR UPDATE" + } + ] + }, + { + "id": "bundle-return-crosses-account-boundary", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-748.748a" + ], + "desc": "return crosses account boundary", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "WHERE id=$1 AND account_id=$2 AND status IN ('shipped','delivered')", + "replace": "WHERE id=$1 AND $2::integer=$2::integer AND status IN ('shipped','delivered')" + } + ] + }, + { + "id": "bundle-components-can-overdraw", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-744.744a", + "ecommerce.spec.bundle-integrity.bundle-745.745a" + ], + "desc": "components can overdraw", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "if (rows.rows.reduce((sum, row) => sum + row.quantity, 0) < component.quantity) throw new Error('A component is unavailable');", + "replace": "// mutant: incomplete component reservation is accepted" + } + ] + }, + { + "id": "bundle-checkout-reuses-reservation", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-747.747a" + ], + "desc": "checkout reuses reservation", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);", + "replace": "// mutant: cart survives checkout" + } + ] + }, + { + "id": "return-after-support-refund-is-blocked", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757a" + ], + "desc": "Reject a valid physical return after a financial refund.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (lineRow.rows[0].returned) {", + "replace": " if (lineRow.rows[0].returned || Number(orderRow.rows[0].refund_total) > 0) {" + } + ] + }, + { + "id": "support-refund-after-return-pays-twice", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757b" + ], + "desc": "Pay the full order again after a physical return.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await client.query(`UPDATE orders SET refund_total = total, status = CASE", + "replace": " await client.query(`UPDATE orders SET refund_total = refund_total + total, status = CASE" + } + ] + }, + { + "id": "progression-support-history-anonymous-leak", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Return private support subjects to signed-out visitors while preserving authenticated ownership filtering.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (!account) return [];\n const where = account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = account.isAdmin || account.isStaff ? [] : [account.id];", + "replace": " const where = !account || account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = !account || account.isAdmin || account.isStaff ? [] : [account.id];" + } + ] + }, + { + "id": "checkout-crash-integrity", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeProgressionSchema(pool);", + "replace": " await initializeProgressionSchema(pool);\n await pool.query(`INSERT INTO cart_item (cart_id, item_id, quantity) SELECT c.id, MIN(oi.item_id), 2 FROM cart c JOIN orders o ON o.account_id=c.account_id JOIN order_item oi ON oi.order_id=o.id GROUP BY c.id ON CONFLICT (cart_id,item_id) DO UPDATE SET quantity=2`);" + } + ] + }, + { + "id": "checkout-crash-sibling-stock", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A recovery callback removes stock for Coffee Grinder while leaving cart and order data intact. The second-product stock discrepancy must fail crash atomicity.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeProgressionSchema(pool);", + "replace": " await initializeProgressionSchema(pool);\n await pool.query(`UPDATE stock SET quantity=quantity-1 WHERE item_id=(SELECT id FROM item WHERE name='Coffee Grinder') AND EXISTS (SELECT 1 FROM orders)`);" + } + ] + }, + { + "id": "checkout-crash-durability", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-durability.910b" + ], + "desc": "A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeProgressionSchema(pool);", + "replace": " await initializeProgressionSchema(pool);\n await pool.query('UPDATE orders SET total=0');" + } + ] + }, + { + "id": "review-script-unsafe-render", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Render stored review text as HTML so it executes in another customer session.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "
{r.comment}
", + "replace": "
" + } + ] + }, + { + "id": "review-script-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Reject all nonempty reviews, including the required benign review; absence of script is insufficient.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const { rating, comment } = req.body ?? {};", + "replace": "const { rating, comment } = req.body ?? {};\n if (String(comment).length > 0) { res.status(400).json({ error: \"Review rejected\" }); return; }" + } + ] + }, + { + "id": "review-owner-trust-username", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Trust the submitted buyer username instead of the authenticated caller.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.body?.username ? (await pool.query(\"SELECT id FROM account WHERE username = $1\", [req.body.username])).rows[0]?.id : req.account!.id;" + } + ] + }, + { + "id": "review-owner-deny-after-write", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Replace the buyer review under a forged username, then return a refusal.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.body?.username ? (await pool.query(\"SELECT id FROM account WHERE username = $1\", [req.body.username])).rows[0]?.id : req.account!.id;" + }, + { + "find": "const { reviews, average } = await buildItemReviews(itemId);\n io.emit(\"review:update\", { itemId, reviews, average });", + "replace": "if (req.body?.username) { res.status(403).json({ error: \"Refused after write\" }); return; }\n const { reviews, average } = await buildItemReviews(itemId);\n io.emit(\"review:update\", { itemId, reviews, average });" + } + ] + }, + { + "id": "review-owner-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Reject the legitimate buyer review as well as unauthorized reviews.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const { rating, comment } = req.body ?? {};", + "replace": "const { rating, comment } = req.body ?? {};\n if (String(comment).length > 0) { res.status(403).json({ error: \"Reject all reviews\" }); return; }" + } + ] + }, + { + "id": "password-verification-truncates-utf8", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Hash and verify only the first 72 UTF-8 bytes, accepting a different password suffix.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "scrypt(password, salt, 64,", + "replace": "scrypt(Buffer.from(password).subarray(0, 72), salt, 64," + } + ] + }, + { + "id": "password-verification-rejects-every-login", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Refuse even a correct password; refusal alone must not earn credit.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "tampered-session-can-purchase", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Accept a modified session credential and allow its purchase.", + "file": "server/src/index.ts", + "edits": [ + { + "find": ".where(eq(session.id, token))\n .limit(1)", + "replace": ".where(sql`true`)\n .limit(1)" + } + ] + }, + { + "id": "restock-trusts-submitted-admin-role", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust an administrator role supplied by staff in the restock request. Normal staff requests are still refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/admin/restock\",\n requireAdmin,", + "replace": "app.post(\n \"/api/admin/restock\",\n requireAuth,\n (req, res, next) => req.body?.role === \"admin\" ? next() : requireAdmin(req, res, next)," + } + ] + }, + { + "id": "restock-admin-claim-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust the submitted administrator role, change stored stock, then return 403.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/admin/restock\",\n requireAdmin,", + "replace": "app.post(\n \"/api/admin/restock\",\n requireAuth,\n (req, res, next) => req.body?.role === \"admin\" ? next() : requireAdmin(req, res, next)," + }, + { + "find": " await processImmediateRestock(itemId);\n await broadcastCatalog();\n const state = await buildAdminState();\n res.json(state);", + "replace": " await processImmediateRestock(itemId);\n await broadcastCatalog();\n const state = await buildAdminState();\n if (req.body?.role === \"admin\") { res.status(403).json({ error: \"refused after write\" }); return; }\n res.json(state);" + } + ] + }, + { + "id": "restock-rejects-administrator-control", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Reject every direct restock, including the legitimate administrator control.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/admin/restock\",\n requireAdmin,", + "replace": "app.post(\n \"/api/admin/restock\",\n (_req, res) => { res.status(403).json({ error: \"restock disabled\" }); }," + } + ] + }, + { + "id": "direct-purchase-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Ordinary purchases use the catalog price; an extra price field overrides the booked order and line price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { name, price } = itemRow.rows[0];", + "replace": " const { name } = itemRow.rows[0];\n const price = req.body?.price ?? itemRow.rows[0].price;" + } + ] + }, + { + "id": "direct-purchase-price-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "A request with an extra price field commits an otherwise correct purchase, then returns 403.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes", + "replace": " await broadcastRecommended(accountId);\n if (req.body?.price !== undefined) { res.status(403).json({ error: \"Price denied\" }); return; }\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes" + } + ] + }, + { + "id": "direct-purchase-rejects-all", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Reject every direct purchase, including the valid purchase controls.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " \"/api/items/:id/buy\",\n requireAuth,", + "replace": " \"/api/items/:id/buy\",\n (_req, res) => { res.status(403).json({ error: \"Purchases denied\" }); },\n requireAuth," + } + ] + }, + { + "id": "order-cache-survives-account-switch", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Merge order updates into a browser cache that is never cleared when the account changes. Separate-browser ownership still works.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [orders, setOrders] = useState([]);", + "replace": " const [orders, setStoredOrders] = useState([]);\n const setOrders: typeof setStoredOrders = update => setStoredOrders(previous => {\n const incoming = typeof update === 'function' ? update(previous) : update;\n return [...new Map([...previous, ...incoming].map(order => [order.id, order])).values()];\n });" + } + ] + }, + { + "id": "order-history-is-always-empty", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Hide all orders, including the required own-account positive views.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "FROM orders WHERE account_id = $1 ORDER BY created_at DESC`,", + "replace": "FROM orders WHERE account_id = $1 AND FALSE ORDER BY created_at DESC`," + } + ] + }, + { + "id": "checkout-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Book the submitted checkout price instead of the stored price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await checkoutReservedCart(accountId);", + "replace": " await checkoutReservedCart(accountId);\n if (req.body?.price !== undefined) await pool.query('UPDATE orders SET total=$1 WHERE id=(SELECT MAX(id) FROM orders WHERE account_id=$2)', [req.body.price, accountId]);" + } + ] + }, + { + "id": "checkout-commits-then-refuses-price", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit checkout before refusing the extra price field.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await checkoutReservedCart(accountId);", + "replace": " await checkoutReservedCart(accountId);\n if (req.body?.price !== undefined) { res.status(400).json({ error: \"Invalid price field\" }); return; }" + } + ] + }, + { + "id": "checkout-rejects-all-price-control", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Refuse all checkouts, including the normal positive path.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " \"/api/checkout\",\n requireAuth,", + "replace": " \"/api/checkout\",\n (_req, res) => { res.status(400).json({ error: \"Checkout denied\" }); },\n requireAuth," + } + ] + }, + { + "id": "checkout-missing-sibling-line", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit the cart checkout with one product line missing while retaining the other product and the full order total.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await checkoutReservedCart(accountId);", + "replace": " await checkoutReservedCart(accountId);\n await pool.query(\"DELETE FROM order_item WHERE order_id=(SELECT MAX(id) FROM orders WHERE account_id=$1) AND item_name='Coffee Grinder'\", [accountId]);" + } + ] + }, + { + "id": "shipping-cancellation-cancel-shipped", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation accepts a shipped order and restores its stock after shipping.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (orderRow.rows[0].status !== \"pending\")", + "replace": "if (false)" + } + ] + }, + { + "id": "shipping-cancellation-resurrect-cancelled", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Shipping accepts a cancelled order after its stock was restored.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "WHERE id = $1 AND status = 'pending' RETURNING account_id", + "replace": "WHERE id = $1 RETURNING account_id" + } + ] + }, + { + "id": "shipping-cancellation-combined-race", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation restores stock but records a shipped status, combining mutually exclusive effects.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (orderRow.rows[0].status !== \"pending\")", + "replace": "if (false)" + }, + { + "find": "UPDATE orders SET status = 'cancelled' WHERE id = $1", + "replace": "UPDATE orders SET status = 'shipped' WHERE id = $1" + } + ] + }, + { + "id": "shipping-cancellation-reject-both", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Both cancellation and shipping refuse pending orders; useful progress must remain required.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (orderRow.rows[0].status !== \"pending\")", + "replace": "if (true)" + }, + { + "find": "WHERE id = $1 AND status = 'pending' RETURNING account_id", + "replace": "WHERE id = $1 AND false RETURNING account_id" + } + ] + }, + { + "id": "shipping-cancellation-queue-retains-removed-order", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "The live staff queue accepts new orders but retains rows after cancellation; the database state remains correct.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"queue:update\", (payload: QueueState) => setQueue(payload));", + "replace": " socket.on(\"queue:update\", (payload: QueueState) => setQueue(previous => payload.queue.length < previous.queue.length ? previous : payload));" + } + ] + }, + { + "id": "opposing-transfer-ignored", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is acknowledged without moving stock.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body ?? {};", + "replace": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body ?? {};\n if (Number(quantity) === 7) return res.json({});" + } + ] + }, + { + "id": "opposing-transfer-rejected", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is refused despite ample source stock.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body ?? {};", + "replace": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body ?? {};\n if (Number(quantity) === 7) return res.status(409).json({ error: \"Transfer disabled\" });" + } + ] + }, + { + "id": "overlapping-cart-add-ignored", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is ignored while ordinary cart setup still works.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " try {\n await reserveCartItem(accountId, itemId, qty);", + "replace": " if (req.account!.username.includes(\"checkout-overlap\") && (await db.select().from(item).where(eq(item.id, itemId)))[0]?.name === \"Coffee Grinder\") return res.json({});\n try {\n await reserveCartItem(accountId, itemId, qty);" + } + ] + }, + { + "id": "overlapping-cart-add-rejected", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is rejected while ordinary cart setup still works.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " try {\n await reserveCartItem(accountId, itemId, qty);", + "replace": " if (req.account!.username.includes(\"checkout-overlap\") && (await db.select().from(item).where(eq(item.id, itemId)))[0]?.name === \"Coffee Grinder\") return res.status(409).json({ error: \"Cart add disabled\" });\n try {\n await reserveCartItem(accountId, itemId, qty);" + } + ] + }, + { + "id": "mixed-history-lost-transfer", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: lost-transfer. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, qty]\n );", + "replace": " await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, 0]\n );" + } + ] + }, + { + "id": "mixed-history-wrong-price", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-price. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [accountId, price]\n );", + "replace": " [accountId, \"1.00\"]\n );" + } + ] + }, + { + "id": "mixed-history-ignored-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: ignored-write. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " try {\n await reserveCartItem(accountId, itemId, qty);", + "replace": " if (req.account!.username.includes(\"history-\") && (await db.select().from(item).where(eq(item.id, itemId)))[0]?.name === \"Coffee Grinder\") return res.json({});\n try {\n await reserveCartItem(accountId, itemId, qty);" + } + ] + }, + { + "id": "mixed-history-reject-valid-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: reject-valid-write. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " try {\n await reserveCartItem(accountId, itemId, qty);", + "replace": " if (req.account!.username.includes(\"history-\") && (await db.select().from(item).where(eq(item.id, itemId)))[0]?.name === \"Coffee Grinder\") return res.status(409).json({ error: \"Cart add disabled\" });\n try {\n await reserveCartItem(accountId, itemId, qty);" + } + ] + }, + { + "id": "mixed-history-repeat-cancel", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: repeat-cancel. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (orderRow.rows[0].status !== \"pending\") {", + "replace": " if (![\"pending\", \"cancelled\"].includes(orderRow.rows[0].status)) {" + } + ] + }, + { + "id": "mixed-history-wrong-owner", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-owner. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();", + "replace": " const itemId = Number(req.params.id);\n const accountId = req.account!.id - 1;\n\n const client = await pool.connect();" + } + ] + }, + { + "id": "checkout-empty-replay-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Allow retrying the consumed empty cart to create another order after a lost reply.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");", + "replace": "// Defect: an empty consumed cart may create another order." + } + ] + }, + { + "id": "checkout-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Reject every valid checkout; refusal cannot earn duplicate protection credit.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");", + "replace": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");\n throw new Error('Checkout disabled');" + } + ] + }, + { + "id": "checkout-replay-loses-prior-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Retrying the consumed cart deletes an earlier acknowledged order.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");", + "replace": " if (lines.rows.length === 0) { await client.query(\"DELETE FROM orders WHERE id = (SELECT min(id) FROM orders WHERE account_id = $1)\", [accountId]); await client.query(\"COMMIT\"); return 0; }" + } + ] + }, + { + "id": "checkout-clears-cart-without-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout returns success and empties the cart without recording an order.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");", + "replace": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");\n if (lines.rows.length > 0) { await client.query(\"DELETE FROM cart_item WHERE cart_id = $1\", [cart.rows[0].id]); await client.query(\"COMMIT\"); return 0; }" + } + ] + }, + { + "id": "revenue-aggregation-truncates-order-population", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Calculate revenue from only the first 256 stored orders while preserving all acknowledged purchases.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "COALESCE((SELECT SUM(total - refund_total) FROM orders WHERE status != 'cancelled'), 0)", + "replace": "COALESCE((SELECT SUM(total - refund_total) FROM (SELECT * FROM orders WHERE status != 'cancelled' ORDER BY id LIMIT 256) limited_orders), 0)" + } + ] + }, + { + "id": "catalog-truncates-created-population", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Omit committed products after the first 256 search results.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".slice(searchPage * CATALOG_PAGE_SIZE,", + "replace": ".slice(0, 256).slice(searchPage * CATALOG_PAGE_SIZE," + } + ] + }, + { + "id": "catalog-discards-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Acknowledge catalog creation without committing a product.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " app.post(\"/api/catalog/products\", dependencies.requireStaff, asyncRoute(async (req, res) => {", + "replace": " app.post(\"/api/catalog/products\", dependencies.requireStaff, asyncRoute(async (req, res) => {\n if (req.body) { res.json({ok:true}); return; } // mutant: successful response without product" + } + ] + }, + { + "id": "catalog-rejects-all-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Reject every catalog creation while keeping sign-in and catalog reads available.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " app.post(\"/api/catalog/products\", dependencies.requireStaff, asyncRoute(async (req, res) => {", + "replace": " app.post(\"/api/catalog/products\", dependencies.requireStaff, asyncRoute(async (req, res) => {\n res.status(403).json({error:'Catalog writes refused'}); return;" + } + ] + }, + { + "id": "shipping-trusts-submitted-staff-role", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Trust a staff role supplied by a customer; ordinary customers remain refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/fulfilment/ship\",\n requireStaff,", + "replace": "app.post(\n \"/api/fulfilment/ship\",\n (req, res, next) => req.body?.role === \"staff\" ? requireAuth(req, res, next) : requireStaff(req, res, next)," + } + ] + }, + { + "id": "shipping-claim-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Ship the order using the claimed role, then return 403. A refusal response cannot hide the stored effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/fulfilment/ship\",\n requireStaff,", + "replace": "app.post(\n \"/api/fulfilment/ship\",\n (req, res, next) => req.body?.role === \"staff\" ? requireAuth(req, res, next) : requireStaff(req, res, next)," + }, + { + "find": " const accountId = result.rows[0].account_id;", + "replace": " if (req.body?.role === \"staff\") { res.status(403).json({error:\"Shipping refused\"}); return; }\n const accountId = result.rows[0].account_id;" + } + ] + }, + { + "id": "shipping-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Reject legitimate staff shipping as well as customer requests.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/fulfilment/ship\",\n requireStaff,", + "replace": "app.post(\n \"/api/fulfilment/ship\",\n (_req, res) => { res.status(403).json({ error: \"Shipping refused\" }); }," + } + ] + }, + { + "id": "cancel-trusts-submitted-owner", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Trust a username submitted by another signed-in customer as the order owner.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const claimed = typeof req.body?.username === 'string' ? (await pool.query('SELECT id FROM account WHERE username = $1', [req.body.username])).rows[0] : null;\n const accountId = claimed?.id ?? req.account!.id;" + } + ] + }, + { + "id": "cancel-claim-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancel using the submitted owner, then return 403; fresh state must expose the unauthorized effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const claimed = typeof req.body?.username === 'string' ? (await pool.query('SELECT id FROM account WHERE username = $1', [req.body.username])).rows[0] : null;\n const accountId = claimed?.id ?? req.account!.id;" + }, + { + "find": " await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);\n await client.query(\"COMMIT\");", + "replace": " await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);\n await client.query(\"COMMIT\");\n if (req.body?.username) { res.status(403).json({error:\"Cancellation refused\"}); return; }" + } + ] + }, + { + "id": "cancel-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Reject legitimate cancellation as well as foreign requests.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "app.post(\n \"/api/orders/:id/cancel\",\n (_req, res) => { res.status(403).json({error:\"Cancellation refused\"}); },\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;" + } + ] + }, + { + "id": "purchase-trusts-cross-origin-cookie", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Remove the origin check so a foreign page can make a purchase with the browsers ambient session cookie.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " return URL.parse(req.headers.origin)?.host === req.headers.host\n ? next() : res.status(403).json({ error: \"cross-origin request refused\" });", + "replace": " return next(); // mutant: accept another origins cookie-backed POST" + } + ] + }, + { + "id": "account-password-in-public-data", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Expose the actual submitted account password through data available to other clients.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "const app = express();", + "replace": "const app = express();\nconst leakedPasswords: string[] = [];\napp.use((_req, res, next) => { const json = res.json.bind(res); res.json = (body: any) => json({ ...body, debugCredentials: leakedPasswords }); next(); });" + }, + { + "file": "server/src/index.ts", + "find": "const passwordHash = await hashPassword(password);", + "replace": "const passwordHash = await hashPassword(password);\n leakedPasswords.push(password);" + } + ] + }, + { + "id": "account-disclosure-rejects-login", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Reject every valid login; absent disclosures alone must not earn a pass.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "signout-keeps-session", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Show a signed-out interface while retaining the browser credential and its purchase authority.", + "edits": [ + { + "file": "client/src/App.tsx", + "find": " await api(\"/api/auth/signout\", { method: \"POST\" });", + "replace": " // mutant: clear the UI but retain the valid session cookie" + } + ] + }, + { + "id": "query-like-password-bypasses-signin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Interpret the query-like password as a login bypass while retaining normal password verification.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (rows.length === 0 || !(await verifyPassword(password, rows[0].passwordHash))) {", + "replace": " if (rows.length === 0 || !(password === \"' OR '1'='1\" || await verifyPassword(password, rows[0].passwordHash))) {" + } + ] + }, + { + "id": "purchase-session-rejects-returning-login", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Refuse every returning login; blocking all access after logout must not pass the session check.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "account-password-in-script", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in the public JavaScript bundle while normal signup and login still work.", + "edits": [ + { + "file": "client/src/main.tsx", + "find": "import React from \"react\";", + "replace": "import React from \"react\";\nObject.assign(window, { debugPassword: 'sb-private-pw-76dc492ea153' });" + } + ] + }, + { + "id": "account-password-in-style", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in a public stylesheet while normal signup and login still work.", + "edits": [ + { + "file": "client/src/index.css", + "find": ":root {", + "replace": ":root {\n --debug-password: \"sb-private-pw-76dc492ea153\";" + } + ] + }, + { + "id": "signup-accepts-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust a submitted signup administrator claim and grant protected stock-write authority.", + "edits": [ + { + "file": "server/src/index.ts", + "find": ".values({ username, passwordHash, isAdmin: false, isStaff: false })", + "replace": ".values({ username, passwordHash, isAdmin: req.body.role === 'admin', isStaff: false })" + } + ] + }, + { + "id": "admin-boundary-rejects-all-signups", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Refuse every new registration; blocking useful signup must not pass the authority check.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "\"/api/auth/signup\",\n asyncHandler(async (req, res) => {", + "replace": "\"/api/auth/signup\",\n asyncHandler(async (req, res) => {\n res.status(403).json({ error: \"Registration disabled\" }); return;" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/mutations/spacetime-ecommerce.json b/tools/stack-bench/grader/mutations/spacetime-ecommerce.json new file mode 100644 index 00000000000..b8c19561b0d --- /dev/null +++ b/tools/stack-bench/grader/mutations/spacetime-ecommerce.json @@ -0,0 +1,3346 @@ +{ + "schemaVersion": 3, + "fixtureSha256": "bfe2d180ce0f62cf667c5a067d47c90ed28fb2478ef0cd315c9f9e0fc14ad67c", + "backend": "spacetime", + "track": "ecommerce", + "note": "Mutation definitions for the SpacetimeDB ecommerce reference.", + "mutations": [ + { + "id": "recommendation-dismissal-lost-on-reconnect", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.spec.state-durability.recommendation-feedback-restart.504c" + ], + "desc": "Erase saved recommendation dismissals when a browser reconnects. This controls reconnect persistence, not backend restart alone.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onConnect = spacetimedb.clientConnected((ctx) => { for (const row of ctx.db.recommendationDismissal.iter()) ctx.db.recommendationDismissal.id.delete(row.id); });\n// --- views ---" + } + ] + }, + { + "id": "pending-order-item-return-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-order-return-boundary.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3f" + ], + "desc": "Accept a pending order return and restore its stock before shipment.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!['shipped', 'delivered'].includes(order.status)) throw new SenderError('Order has not shipped yet.');", + "replace": "if (!['pending', 'shipped', 'delivered'].includes(order.status)) throw new SenderError('Order has not shipped yet.');" + } + ] + }, + { + "id": "staff-admin-access-survives-role-removal", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Keep administrator access after changing the assigned role back to staff.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.account.id.update({ ...target, isAdmin: role === 'admin' });", + "replace": "ctx.db.account.id.update({ ...target, isAdmin: target.isAdmin || role === 'admin' });" + } + ] + }, + { + "id": "shipping-counts-sale-twice", + "scenario": "tracks/ecommerce/scenarios/progression-shipping-accounting.json", + "targets": [ + "ecommerce.inventory-operations.shipping-accounting.202e" + ], + "desc": "Shipping succeeds but doubles the completed sale value in authoritative revenue.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });", + "replace": " ctx.db.customerOrder.id.update({ ...order, status: 'shipped', total: order.total * 2 });" + } + ] + }, + { + "id": "restock-client-snapshot-overwrites-concurrent-purchases", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Compute absolute restock quantity when the administrator edits the form and store that captured value in the reducer. A fixed 500 ms submission delay widens the stale-write window. Serial purchases and restocks still work; intervening purchases can be overwritten. This is a stale-form lost update, not a race inside an atomic reducer.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n for (const alert of", + "replace": "ctx.db.stock.insert({ ...existing, quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n for (const alert of" + }, + { + "file": "client/src/components/AdminPanel.tsx", + "find": "onChange={(e) => setRestockInputs((v) => ({ ...v, [k]: e.target.value }))}", + "replace": "onChange={(e) => setRestockInputs((v) => ({ ...v, [k]: String(stockOf(item.id, wh.id) + Number(e.target.value)) }))}" + }, + { + "file": "client/src/App.tsx", + "find": " await conn?.reducers.adminRestock({ itemId, warehouseId, quantity });", + "replace": " // Mutant: widen the stale form submission window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n await conn?.reducers.adminRestock({ itemId, warehouseId, quantity });" + } + ] + }, + { + "id": "signup-binds-the-new-account-to-the-admin-session", + "scenario": "tracks/ecommerce/scenarios/01-account-create.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Create the requested account but bind the new browser session to the administrator, so account creation no longer signs the visitor in as the account it created.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": " bindSession(tx, account.id);\n return true;\n });\n }\n);\n\nexport const signIn", + "replace": " bindSession(tx, tx.db.account.username.find('admin')!.id);\n return true;\n });\n }\n);\n\nexport const signIn" + } + ] + }, + { + "id": "duplicate-signup-is-silently-ignored", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "Return success for a taken username without creating a session or surfacing the required refusal.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "if (tx.db.account.username.find(name) || tx.db.account.passwordSalt.find(passwordSalt)) return false;", + "replace": "if (tx.db.account.username.find(name) || tx.db.account.passwordSalt.find(passwordSalt)) return true;" + } + ] + }, + { + "id": "signin-does-not-verify-the-password", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Accept a known username without comparing the supplied password hash.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account;" + } + ] + }, + { + "id": "signout-keeps-the-account-session", + "scenario": "tracks/ecommerce/scenarios/01-account-signout.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Leave the current account session in place when the visitor signs out.", + "file": "client/src/components/AuthWidget.tsx", + "edits": [ + { + "find": " await connection.reducers.signOut({});\n clearToken(); location.reload();", + "replace": " location.reload(); // mutant: keep the account and credential" + } + ] + }, + { + "id": "session-token-is-not-persisted-for-reload", + "scenario": "tracks/ecommerce/scenarios/01-account-reload.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "Discard the connection token instead of persisting it, so a reload receives a new identity with no account session.", + "file": "client/src/main.tsx", + "edits": [ + { + "find": ".onConnect((_connection, _identity, token) => saveToken(token));", + "replace": ".onConnect((_connection, _identity, token) => { saveToken(token); sessionStorage.clear(); });" + } + ] + }, + { + "id": "catalog-seeds-the-wrong-air-purifier-price", + "scenario": "tracks/ecommerce/scenarios/01-catalog-values.json", + "targets": [ + "ecommerce.feature.catalog.catalog-values.2a" + ], + "desc": "Seed Air Purifier with an incorrect stored price while leaving the rest of the catalog intact.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ['Air Purifier', 189.0, 60, 40, 'Home'],", + "replace": " ['Air Purifier', 999.0, 60, 40, 'Home']," + } + ] + }, + { + "id": "catalog-tie-breaks-in-reverse-alphabetical-order--01-catalog-ranking", + "scenario": "tracks/ecommerce/scenarios/01-catalog-ranking.json", + "targets": [ + "ecommerce.feature.catalog.catalog-ranking.2b" + ], + "desc": "Reverse the specified alphabetical tie-breaker. This necessarily breaks both the initial sequence and the post-purchase sequence in the same scenario.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " return a.name.localeCompare(b.name);", + "replace": " return b.name.localeCompare(a.name);" + } + ] + }, + { + "id": "catalog-tie-breaks-in-reverse-alphabetical-order--01-core", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "Reverse the specified alphabetical tie-breaker. This necessarily breaks both the initial sequence and the post-purchase sequence in the same scenario.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " return a.name.localeCompare(b.name);", + "replace": " return b.name.localeCompare(a.name);" + } + ] + }, + { + "id": "purchase-does-not-update-ranking-count", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "Complete the purchase but leave its popularity count unchanged, so open storefronts cannot rank the purchased item first.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " bumpPurchaseCount(ctx, itemId, quantity);", + "replace": " // mutant: buy-now never advances the ranking count" + } + ] + }, + { + "id": "signed-out-purchase-bypasses-account-check", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "Expose the guest purchase button and accept its purchase as the existing administrator. The stock observation then exercises the broken account boundary; normal signed-in purchases remain unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = requireAccount(ctx);", + "replace": "export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = getAccountId(ctx) === null ? ctx.db.account.username.find('admin')! : requireAccount(ctx);" + }, + { + "file": "client/src/components/ItemCard.tsx", + "find": " {isSignedIn && (", + "replace": " {true && (" + } + ] + }, + { + "id": "buy-now-creates-orders-without-reserving-stock--01-buying", + "scenario": "tracks/ecommerce/scenarios/01-buying.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "Create purchase orders without reserving inventory. The same defect necessarily breaks live purchase stock and the sell-out portion of the stock-limit check.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const allocations = decrementStockTracked(ctx, itemId, quantity);", + "replace": " const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];" + } + ] + }, + { + "id": "buy-now-creates-orders-without-reserving-stock--stock-limit", + "scenario": "tracks/ecommerce/scenarios/progression-stock-limit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.stock-limit.3d" + ], + "desc": "Create purchase orders without reserving inventory. The same defect necessarily breaks live purchase stock and the sell-out portion of the stock-limit check.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const allocations = decrementStockTracked(ctx, itemId, quantity);", + "replace": " const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];" + } + ] + }, + { + "id": "restock-race-records-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: Math.round(price * 100) * quantity / 100,\n status: 'pending',", + "replace": " total: Math.round(price * 100) * quantity / 100 + 1,\n status: 'pending'," + } + ] + }, + { + "id": "buy-now-records-the-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "Record a completed buy-now order one dollar above the stored item price.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: Math.round(price * 100) * quantity / 100,\n status: 'pending',", + "replace": " total: Math.round(price * 100) * quantity / 100 + 1,\n status: 'pending'," + } + ] + }, + { + "id": "existing-cart-line-does-not-increment-basic-cart", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4a" + ], + "desc": "Write an existing cart line back without incrementing its quantity.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.cartItem.id.update({ ...existing, quantity: existing.quantity + 1 });", + "replace": "ctx.db.cartItem.id.update({ ...existing, quantity: existing.quantity });" + } + ] + }, + { + "id": "cart-is-deleted-when-owner-disconnects", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Delete the account cart on transport disconnect. Reload loses stored cart contents even after the same account signs in again; account and session records remain intact.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onDisconnect = spacetimedb.clientDisconnected((ctx) => {\n const accountId = getAccountId(ctx);\n if (accountId !== null) for (const row of [...ctx.db.cartItem.byAccountItem.filter(accountId)]) ctx.db.cartItem.id.delete(row.id);\n});\n// --- views ---" + } + ] + }, + { + "id": "signin-binds-the-second-client-to-a-different-account", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.live-state.shared-cart.4c" + ], + "desc": "Authenticate valid credentials but bind the second connection to the administrator account, so two sessions for one customer do not share the customer's cart.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": " if (!current || current.passwordHash !== account.passwordHash) return false;\n bindSession(tx, account.id);", + "replace": " if (!current || current.passwordHash !== account.passwordHash) return false;\n bindSession(tx, tx.db.account.username.find('admin')!.id);" + } + ] + }, + { + "id": "checkout-does-not-empty-the-basic-cart", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Leave completed checkout lines in the durable cart.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": " // mutant: checked-out cart lines remain" + } + ] + }, + { + "id": "new-review-is-accepted-without-being-stored", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Accept an eligible new review but omit its durable insert, so neither author nor visitor can see it.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.review.insert({\n id: 0n,\n itemId,\n accountId: acc.id,\n rating,\n comment,\n createdAt: ctx.timestamp,\n });", + "replace": " // mutant: accepted review is not persisted" + } + ] + }, + { + "id": "repeat-review-inserts-a-second-row", + "scenario": "tracks/ecommerce/scenarios/01-review-uniqueness.json", + "targets": [ + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "desc": "Insert a second review row instead of updating the customer's existing item review.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.review.id.update({ ...existing, rating, comment, createdAt: ctx.timestamp });", + "replace": " ctx.db.review.insert({ id: 0n, itemId, accountId: acc.id, rating, comment, createdAt: ctx.timestamp });" + } + ] + }, + { + "id": "review-average-counts-rows-instead-of-ratings", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "Compute the live average from a constant per row rather than each stored rating.", + "file": "client/src/components/ItemDetail.tsx", + "edits": [ + { + "find": " : reviews.reduce((sum, r) => sum + r.rating, 0) / reviews.length;", + "replace": " : reviews.reduce((sum) => sum + 1, 0) / reviews.length;" + } + ] + }, + { + "id": "every-signed-in-customer-is-treated-as-an-admin", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-area-boundary.7a" + ], + "desc": "Use account presence instead of the server-provided administrator flag to expose the admin area.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isAdmin = currentUser?.isAdmin ?? false;", + "replace": " const isAdmin = isSignedIn;" + } + ] + }, + { + "id": "warehouse-view-omits-west", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "Filter one real warehouse out of the administrator's inventory view.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " warehouses={warehouses}", + "replace": " warehouses={warehouses.filter((warehouse) => warehouse.name !== 'West')}" + } + ] + }, + { + "id": "guest-purchase-falls-back-to-the-admin-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Allow an unauthenticated direct purchase by attributing missing sessions to the administrator account, while preserving the authorized control path.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = requireAccount(ctx);", + "replace": "export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const accountId = getAccountId(ctx);\n const acc = (accountId === null ? null : ctx.db.account.id.find(accountId))\n ?? ctx.db.account.username.find('admin')!;" + } + ] + }, + { + "id": "direct-purchases-are-attributed-to-the-system-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-attribution.json", + "targets": [ + "ecommerce.spec.access-control.purchase-attribution.102a" + ], + "desc": "Store every buy-now order under the administrator instead of the authenticated caller.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " accountId,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100,", + "replace": " accountId: ctx.db.account.username.find('admin')!.id,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100," + } + ] + }, + { + "id": "direct-restock-does-not-require-an-admin", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Remove the server-side administrator check from the restock reducer.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n requireAdmin(ctx);", + "replace": "export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n // mutant: no administrator check" + } + ] + }, + { + "id": "direct-purchase-ignores-the-stored-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Create the direct purchase order one dollar above the authoritative stored price.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: Math.round(price * 100) * quantity / 100,\n status: 'pending',", + "replace": " total: 1,\n status: 'pending'," + } + ] + }, + { + "id": "account-state-token-is-not-restored-after-reload", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "Discard the saved token after connection, preserving signup but losing access to the account, cart and orders on the next reload.", + "file": "client/src/main.tsx", + "edits": [ + { + "find": ".onConnect((_connection, _identity, token) => saveToken(token));", + "replace": ".onConnect((_connection, _identity, token) => { saveToken(token); sessionStorage.clear(); });" + } + ] + }, + { + "id": "reconnect-discards-the-visible-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "Keep normal reload recovery but discard the client account projection when the browser comes back online.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const currentUser = currentUserRows[0] ?? null;", + "replace": " const [discardAccountAfterReconnect, setDiscardAccountAfterReconnect] = useState(false);\n useEffect(() => {\n const discardAccount = () => setDiscardAccountAfterReconnect(true);\n window.addEventListener('online', discardAccount);\n return () => window.removeEventListener('online', discardAccount);\n }, []);\n const currentUser = discardAccountAfterReconnect ? null : currentUserRows[0] ?? null;" + } + ] + }, + { + "id": "order-views-return-every-customers-orders", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Remove account filters from both order views, exposing another customer's order and its line items.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const rows = [...ctx.db.customerOrder.accountId.filter(accountId)];", + "replace": " const rows = [...ctx.db.customerOrder.iter()];" + }, + { + "find": " for (const o of ctx.db.customerOrder.accountId.filter(accountId)) {", + "replace": " for (const o of ctx.db.customerOrder.iter()) {" + } + ] + }, + { + "id": "admin-revenue-double-counts-every-order", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Count every completed order twice in the administrator revenue projection.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total += o.total - o.refundedTotal;", + "replace": " total += (o.total - o.refundedTotal) * 2;" + } + ] + }, + { + "id": "purchases-do-not-leave-the-warehouses", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107b" + ], + "desc": "Create normal orders and revenue while leaving warehouse stock unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const allocations = decrementStockTracked(ctx, itemId, quantity);", + "replace": " const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];" + } + ] + }, + { + "id": "review-purchase-eligibility-is-not-checked", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "Allow a signed-in customer to review an item with no matching purchase.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!bought) throw new SenderError('You can only review items you have purchased.');", + "replace": " // mutant: purchase eligibility is not checked" + } + ] + }, + { + "id": "eligible-review-is-accepted-without-being-stored", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "Keep the non-buyer refusal but omit the insert for a buyer's eligible new review. Both eligibility criteria require a successfully stored eligible review as a positive control; this does not establish a non-buyer authorization defect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.review.insert({\n id: 0n,\n itemId,\n accountId: acc.id,\n rating,\n comment,\n createdAt: ctx.timestamp,\n });", + "replace": " // mutant: eligible review is acknowledged but not persisted" + } + ] + }, + { + "id": "cart-line-lookup-ignores-cart-ownership", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109a" + ], + "desc": "Find an existing cart line by item alone, so the same named add action from another customer increments the owner's line instead of that customer's cart.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "function findCartLine(ctx: Ctx, accountId: bigint, itemId: bigint) {\n for (const row of ctx.db.cartItem.byAccountItem.filter([accountId, itemId])) {\n return row;\n }\n return null;\n}", + "replace": "function findCartLine(ctx: Ctx, _accountId: bigint, itemId: bigint) {\n for (const row of ctx.db.cartItem.iter()) {\n if (row.itemId === itemId) return row;\n }\n return null;\n}" + } + ] + }, + { + "id": "purchase-does-not-reserve-stock-last-unit", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Create purchase orders without reserving stock, proving the focused last-unit stock, order-count, and revenue consequences.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const allocations = decrementStockTracked(ctx, itemId, quantity);", + "replace": " const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];" + } + ] + }, + { + "id": "existing-cart-line-does-not-increment", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a" + ], + "desc": "Render the old cart quantity after concurrent adds.", + "file": "client/src/components/CartPanel.tsx", + "edits": [ + { + "find": " value={line.quantity}", + "replace": " value={1}" + } + ] + }, + { + "id": "checkout-does-not-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Keep checked-out cart lines so the next serialized checkout creates a duplicate order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": " // mutant: checked-out lines remain in the cart" + } + ] + }, + { + "id": "stock-subscription-snapshotted-once", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "Render the first non-empty stock snapshot forever instead of following committed subscription updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [stocks] = useTable(tables.stock);", + "replace": " const [liveStocks] = useTable(tables.stock);\n const initialStocks = useRef(null);\n if (initialStocks.current === null && liveStocks.length > 0) {\n initialStocks.current = liveStocks;\n }\n const stocks = initialStocks.current ?? liveStocks;" + } + ] + }, + { + "id": "stock-view-ignores-update-across-app-server-stop", + "scenario": "tracks/ecommerce/scenarios/01-external-server-restart-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901c" + ], + "desc": "Persist the first stock quantities in browser session storage and keep rendering them after app-server restart, including any frontend reload. Initial stock remains correct. This validates the stale-view oracle, not SpacetimeDB storage durability.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [stocks] = useTable(tables.stock);", + "replace": " const [liveStocks] = useTable(tables.stock);\n const cacheKey = 'stale-stock-quantities';\n let savedQuantities = sessionStorage.getItem(cacheKey);\n if (!savedQuantities && liveStocks.length > 0) {\n savedQuantities = JSON.stringify(Object.fromEntries(liveStocks.map(row => [`${row.itemId}-${row.warehouseId}`, row.quantity])));\n sessionStorage.setItem(cacheKey, savedQuantities);\n }\n const quantities: Record = JSON.parse(savedQuantities ?? '{}');\n const stocks = liveStocks.map(row => ({ ...row, quantity: quantities[`${row.itemId}-${row.warehouseId}`] ?? row.quantity }));" + } + ] + }, + { + "id": "stock-view-keeps-pre-reconnect-snapshot", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "Continue following stock until the browser goes offline, then retain the last online snapshot after network restoration.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [stocks] = useTable(tables.stock);", + "replace": " const [liveStocks] = useTable(tables.stock);\n const [freezeStockAfterOffline, setFreezeStockAfterOffline] = useState(false);\n const lastOnlineStocks = useRef(liveStocks);\n useEffect(() => {\n const freezeStock = () => setFreezeStockAfterOffline(true);\n window.addEventListener('offline', freezeStock);\n return () => window.removeEventListener('offline', freezeStock);\n }, []);\n if (!freezeStockAfterOffline) {\n lastOnlineStocks.current = liveStocks;\n }\n const stocks = freezeStockAfterOffline ? lastOnlineStocks.current : liveStocks;" + } + ] + }, + { + "id": "open-review-list-snapshots-on-selection", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "Snapshot the selected item's reviews when the detail opens instead of following later subscription updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const selectedItemReviews = selectedItemId !== null ? reviewsByItem.get(selectedItemId) ?? [] : [];", + "replace": " const openedReviewItem = useRef(null);\n const openedReviews = useRef<(typeof reviews)[number][]>([]);\n if (selectedItemId !== openedReviewItem.current) {\n openedReviewItem.current = selectedItemId;\n openedReviews.current = selectedItemId !== null ? reviewsByItem.get(selectedItemId) ?? [] : [];\n }\n const selectedItemReviews = openedReviews.current;" + } + ] + }, + { + "id": "open-review-list-renders-each-review-twice", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "Render every committed review twice in the already-open list.", + "file": "client/src/components/ItemDetail.tsx", + "edits": [ + { + "find": " {reviews.map((r) => (", + "replace": " {[...reviews, ...reviews].map((r) => (" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "The serialized cancellation reducer changes order state and purchase counts but skips allocation restoration.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);", + "replace": " // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);" + } + ] + }, + { + "id": "cancellation-accounting-loses-stock-restoration", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);", + "replace": " // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c" + ], + "desc": "The serialized cancellation reducer skips allocation restoration, so a fresh client reads the persisted shortfall.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);", + "replace": " // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "The serialized cancellation reducer restores allocations but writes pending back to order history.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.customerOrder.id.update({ ...order, status: 'cancelled' });", + "replace": " ctx.db.customerOrder.id.update({ ...order, status: 'pending' });" + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "The admin revenue view includes cancelled orders even though cancellation otherwise succeeds.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!isOrderCounted(o)) continue;\n total += o.total - o.refundedTotal;", + "replace": " total += o.total - o.refundedTotal;" + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-invariant", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "The admin revenue view includes cancelled orders even though cancellation otherwise succeeds.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!isOrderCounted(o)) continue;\n total += o.total - o.refundedTotal;", + "replace": " total += o.total - o.refundedTotal;" + } + ] + }, + { + "id": "operator-authorization-allows-customer-transfer", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "The transfer reducer drops its administrator role gate.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, { itemId, fromWarehouseId, toWarehouseId, quantity }) => {\n requireAdmin(ctx);", + "replace": " (ctx, { itemId, fromWarehouseId, toWarehouseId, quantity }) => {\n // mutant: no administrator role check" + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping reducer drops its staff role check while retaining pending-state validation.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);", + "replace": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check" + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation bypasses the owner helper while retaining missing-order and pending-state validation.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = requireOrderOwner(ctx, orderId);", + "replace": "export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = ctx.db.customerOrder.id.find(orderId);\n if (!order) throw new SenderError('Order not found.');" + } + ] + }, + { + "id": "queue-depth-lags-one-order", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1a" + ], + "desc": "The reactive queue renders every order but its visible depth remains one behind.", + "file": "client/src/components/FulfilmentPanel.tsx", + "edits": [ + { + "find": "Waiting: {queue.length}", + "replace": "Waiting: {Math.max(0, queue.length - 1)}" + } + ] + }, + { + "id": "ship-acknowledges-without-changing-status", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-ship.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1c" + ], + "desc": "The serialized shipping reducer accepts the call but writes pending back to the order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });", + "replace": " ctx.db.customerOrder.id.update({ ...order, status: 'pending' });" + } + ] + }, + { + "id": "customer-sees-fulfilment-navigation", + "scenario": "tracks/ecommerce/scenarios/02-features.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isStaff = currentUser?.isStaff ?? false;", + "replace": " const isStaff = isSignedIn;" + } + ] + }, + { + "id": "progression-customer-sees-fulfilment-content", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-access.json", + "targets": [ + "ecommerce.spec.access-control.fulfilment-area-boundary.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isStaff = currentUser?.isStaff ?? false;", + "replace": " const isStaff = isSignedIn;" + } + ] + }, + { + "id": "operator-authorization-allows-customer-shipping", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping reducer drops the staff role check while retaining the pending-order guard.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);", + "replace": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check" + } + ] + }, + { + "id": "transfer-debits-source-without-crediting-existing-destination", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202a", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "A transfer debits the source row but writes the existing destination quantity back unchanged, violating both directional movement and total conservation inside the serialized reducer. It also breaks 201a's authorized-transfer positive control; that coupled failure is not independent evidence of an authorization defect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity });" + } + ] + }, + { + "id": "transfer-warehouse-totals-omit-destination-credit", + "scenario": "tracks/ecommerce/scenarios/02-transfer-totals.json", + "targets": [ + "ecommerce.inventory-operations.warehouse-transfer.2b" + ], + "desc": "The serialized transfer debits the source but writes the existing destination quantity back unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity });" + } + ] + }, + { + "id": "transfer-overdraft-guard-removed", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.inventory-operations.warehouse-transfer.2c" + ], + "desc": "The serialized reducer no longer rejects insufficient source stock and commits negative source quantity.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (available < quantity) {\n throw new SenderError(`Not enough stock in source warehouse: only ${available} available.`);\n }", + "replace": " // mutant: insufficient source stock is not rejected" + } + ] + }, + { + "id": "low-stock-excludes-boundary-ten", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5a" + ], + "desc": "The reactive low-stock view uses a strict boundary and omits items with exactly ten units.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)", + "replace": " .filter((i) => (stockByItem.get(i.id) ?? 0) < LOW_STOCK_THRESHOLD)" + } + ] + }, + { + "id": "category-totals-ignore-pending-purchases", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5b" + ], + "desc": "The category totals view includes only shipped orders, so a newly accepted pending purchase is absent.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const order of ctx.db.customerOrder.iter()) {\n if (!isOrderCounted(order)) continue;", + "replace": " for (const order of ctx.db.customerOrder.iter()) {\n if (order.status !== 'shipped') continue;" + } + ] + }, + { + "id": "recommendations-ignore-pending-purchases", + "scenario": "tracks/ecommerce/scenarios/02-operational-recommendations.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5c" + ], + "desc": "The personal recommendation view derives categories only from shipped orders, so a new pending purchase has no influence.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const order of ctx.db.customerOrder.accountId.filter(accountId)) {\n if (!isOrderCounted(order)) continue;", + "replace": " for (const order of ctx.db.customerOrder.accountId.filter(accountId)) {\n if (order.status !== 'shipped') continue;" + } + ] + }, + { + "id": "purchases-do-not-affect-best-sellers", + "scenario": "tracks/ecommerce/scenarios/02-operational-best-sellers.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5d" + ], + "desc": "Rank signed-out recommendations without purchase counts.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const purchaseCountOf = (id: bigint) => ctx.db.itemStats.itemId.find(id)?.purchaseCount ?? 0;", + "replace": "const purchaseCountOf = (_id: bigint) => 0;" + } + ] + }, + { + "id": "queue-warehouse-reports-west", + "scenario": "tracks/ecommerce/scenarios/02-queue-warehouse.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1b" + ], + "desc": "The queue renders West for the deterministic Desk Lamp allocation even though the order reserved stock in East.", + "file": "client/src/components/FulfilmentPanel.tsx", + "edits": [ + { + "find": "{name}: {order.warehouseNames[i]}", + "replace": "{name}: West" + } + ] + }, + { + "id": "transfer-creates-stock-during-race", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "The transfer reducer credits the destination one unit more than it debits from the source, so the item's total after a transfer racing a purchase is the starting total rather than one less. A stored conservation defect; it does not model a lost-update interleaving because SpacetimeDB reducer execution is atomically serialized.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity + 1 });" + } + ] + }, + { + "id": "catalog-search-ignores-the-query", + "scenario": "tracks/ecommerce/scenarios/01-catalog-search.json", + "targets": [ + "ecommerce.feature.catalog.catalog-search.2d" + ], + "desc": "A non-empty catalog query filters out every product instead of matching names.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".filter(item => !q || item.name.toLowerCase().includes(q))", + "replace": ".filter(() => !q) // mutant: non-empty searches return no products" + } + ] + }, + { + "id": "admin-total-stock-is-not-rendered", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json", + "targets": [ + "ecommerce.spec.live-state.warehouse-stock.7c" + ], + "desc": "The staff stock total always renders zero after a warehouse restock.", + "file": "client/src/components/AdminPanel.tsx", + "edits": [ + { + "find": "{totalStockOf(item.id)}", + "replace": "{0}" + } + ] + }, + { + "id": "customers-can-schedule-restocks", + "scenario": "tracks/ecommerce/scenarios/03-deferred-access.json", + "targets": [ + "ecommerce.l3.deferred-access.scheduled-work-access.317a" + ], + "desc": "Scheduling a restock no longer checks that the caller is an administrator.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, input) => {\n requireAdmin(ctx);\n const itemName = input.item.trim();\n const warehouseName = input.warehouse.trim();\n const item = [...ctx.db.item].find(row => row.name === itemName);\n const warehouse = [...ctx.db.warehouse].find(row => row.name === warehouseName);", + "replace": " (ctx, input) => {\n // mutant: any signed-in or anonymous caller can schedule work\n const itemName = input.item.trim();\n const warehouseName = input.warehouse.trim();\n const item = [...ctx.db.item].find(row => row.name === itemName);\n const warehouse = [...ctx.db.warehouse].find(row => row.name === warehouseName);" + } + ] + }, + { + "id": "scheduled-restock-execution-queue-is-process-local", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.311a" + ], + "desc": "Keep manual restock execution IDs only in the V8 process. Ordinary timers work, but restart loses the execution queue while pending rows remain. Isolate replacement can also lose this queue.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const scheduleRestock = spacetimedb.reducer(", + "replace": "const pendingRestockExecution = new Set();\n\nexport const scheduleRestock = spacetimedb.reducer(" + }, + { + "find": " ctx.db.scheduledRestock.insert({\n id: 0n,\n itemId: item.id,\n warehouseId: warehouse.id,\n quantity: input.quantity,\n dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,\n status: 'pending',\n reorderRuleId: undefined,\n });", + "replace": " const pending = ctx.db.scheduledRestock.insert({\n id: 0n,\n itemId: item.id,\n warehouseId: warehouse.id,\n quantity: input.quantity,\n dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,\n status: 'pending',\n reorderRuleId: undefined,\n });\n pendingRestockExecution.add(pending.id);" + }, + { + "find": " if (pending.status !== 'pending' || pending.dueMicros > now) continue;\n restoreStock(ctx, pending.itemId, pending.warehouseId, pending.quantity);", + "replace": " if (pending.status !== 'pending' || pending.dueMicros > now) continue;\n if (pending.reorderRuleId === undefined && !pendingRestockExecution.delete(pending.id)) continue;\n restoreStock(ctx, pending.itemId, pending.warehouseId, pending.quantity);" + } + ] + }, + { + "id": "reservation-is-delayed-past-the-durability-window", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.314a" + ], + "desc": "A reservation is persisted with a ten-minute lifetime instead of ninety seconds.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const expiresMicros = nowMicros(ctx) + 90n * SECOND;", + "replace": "const expiresMicros = nowMicros(ctx) + 600n * SECOND;" + } + ] + }, + { + "id": "completed-restock-remains-pending", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.exactly-once.311a" + ], + "desc": "A completed restock remains pending and is applied again by later maintenance ticks.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.scheduledRestock.id.update({ ...pending, status: 'complete' });", + "replace": "ctx.db.scheduledRestock.id.update({ ...pending, status: 'pending' });" + } + ] + }, + { + "id": "reservation-expiry-restores-stock-twice", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.stock-conservation.313a" + ], + "desc": "Reservation expiry returns twice the quantity that was reserved.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity);\n ctx.db.reservation.id.update({ ...row, expired: true });", + "replace": "restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity * 2);\n ctx.db.reservation.id.update({ ...row, expired: true });" + } + ] + }, + { + "id": "checkout-takes-reserved-stock-again", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.stock-conservation.314a" + ], + "desc": "Checkout decrements stock after the cart reservation already took it.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const allocations = held.map(row => ({ warehouseId: row.warehouseId, quantity: row.quantity, stockItemId: row.stockItemId }));\n const orderItemRow", + "replace": "const allocations = decrementStockTracked(ctx, p.itemId, p.quantity);\n const orderItemRow" + } + ] + }, + { + "id": "reservation-does-not-decrement-stock", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.301a" + ], + "desc": "Creating a reservation leaves the public stock total unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.stock.insert({ ...row, quantity: row.quantity - allocation.quantity });", + "replace": "ctx.db.stock.insert({ ...row, quantity: row.quantity });" + } + ] + }, + { + "id": "reservation-timer-is-static", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.305a" + ], + "desc": "The cart always renders ninety seconds instead of a decreasing reservation timer.", + "file": "client/src/components/CartPanel.tsx", + "edits": [ + { + "find": "const seconds = Math.max(0, Number((reservation.expiresMicros - BigInt(Date.now()) * 1000n) / 1_000_000n));", + "replace": "const seconds = 90;" + } + ] + }, + { + "id": "checkout-leaves-cart-lines", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.306a" + ], + "desc": "Checkout creates an order but leaves the purchased lines in the cart.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": "for (const line of lines) void line;" + } + ] + }, + { + "id": "expired-reservation-is-still-marked-live", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.307a" + ], + "desc": "Expired reservations keep their live flag, so the cart does not mark them expired.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.reservation.id.update({ ...row, expired: true });", + "replace": "ctx.db.reservation.id.update({ ...row, expired: false });" + } + ] + }, + { + "id": "renewed-reservation-expires-too-soon", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.308a" + ], + "desc": "Renewed quantities receive only a twenty-second reservation window.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "reserveUnits(ctx, accountId, itemId, quantity);", + "replace": "reserveUnits(ctx, accountId, itemId, quantity);\n for (const renewed of findReservations(ctx, accountId, itemId)) {\n ctx.db.reservation.id.update({ ...renewed, expiresMicros: nowMicros(ctx) + 20n * SECOND });\n }" + } + ] + }, + { + "id": "pending-restock-timer-is-static", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restocks.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a" + ], + "desc": "The pending restock UI always renders ninety seconds instead of the server due time.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": "{Math.max(0, Number((row.dueMicros - BigInt(Date.now()) * 1000n) / 1_000_000n))}", + "replace": "{90}" + } + ] + }, + { + "id": "due-restock-omits-ledger-entry", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-apply.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a" + ], + "desc": "A due restock updates stock but does not create its stock ledger record.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.stockLedger.insert({\n id: 0n,\n itemId: pending.itemId,\n warehouseId: pending.warehouseId,\n quantity: pending.quantity,\n createdMicros: now,\n source: 'scheduled restock',\n });", + "replace": "// mutant: due restocks are not recorded in the ledger" + } + ] + }, + { + "id": "cancelled-restock-remains-pending", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a" + ], + "desc": "Cancelling a restock leaves it pending, so maintenance later applies it.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.scheduledRestock.id.update({ ...row, status: 'cancelled' });", + "replace": "ctx.db.scheduledRestock.id.update({ ...row, status: 'pending' });" + } + ] + }, + { + "id": "restart-restock-runs-early", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.312a" + ], + "desc": "A scheduled restock ignores its requested delay and becomes due after one second.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,", + "replace": "dueMicros: nowMicros(ctx) + SECOND," + } + ] + }, + { + "id": "closed-browser-reservation-never-expires", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.313a" + ], + "desc": "Server maintenance skips reservation expiry when no customer browser is present.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const row of [...ctx.db.reservation.iter()]) {\n if (row.expired || row.expiresMicros > now) continue;", + "replace": "for (const row of [...ctx.db.reservation.iter()].filter(() => false)) {\n if (row.expired || row.expiresMicros > now) continue;" + } + ] + }, + { + "id": "catalog-product-is-not-published", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-management.json", + "targets": [ + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b" + ], + "desc": "The public product card omits the submitted name. Both create visibility and variant navigation locate the submitted product card by its name, so the hidden name prevents both required observations.", + "file": "client/src/components/ItemCard.tsx", + "edits": [ + { + "find": "{item.name}", + "replace": "{item.name === 'Travel Mug' ? '' : item.name}" + } + ] + }, + { + "id": "catalog-variants-are-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-management.json", + "targets": [ + "ecommerce.progression.catalog-management.catalog-management.622b" + ], + "desc": "Catalog management discards every submitted product variant.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const variantName of variants.split(',').map(value => value.trim()).filter(Boolean)) {\n ctx.db.itemVariant.insert({ id: 0n, itemId: product.id, name: variantName });\n }", + "replace": "void variants; // mutant: submitted variants are discarded" + } + ] + }, + { + "id": "profile-is-lost-on-fresh-account-login", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.state-durability.customer-profile-reload.620a" + ], + "desc": "Delete a saved profile when its owner signs in from a new transport identity. Initial save, same-identity reload, and the independent privacy owner remain intact.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": " if (!current || current.passwordHash !== account.passwordHash) return false;\n bindSession(tx, account.id);", + "replace": " if (!current || current.passwordHash !== account.passwordHash) return false;\n tx.db.customerProfile.accountId.delete(account.id);\n bindSession(tx, account.id);" + } + ] + }, + { + "id": "customer-profile-view-leaks-another-account", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.access-control.customer-profile-privacy.620b" + ], + "desc": "The customer profile view returns the first stored profile without checking its owner.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "return accountId === null ? undefined : ctx.db.customerProfile.accountId.find(accountId) ?? undefined;", + "replace": "return accountId === null ? undefined : [...ctx.db.customerProfile.iter()][0];" + } + ] + }, + { + "id": "faceted-search-ignores-category", + "scenario": "tracks/ecommerce/scenarios/progression-faceted-filters.json", + "targets": [ + "ecommerce.progression.faceted-search.faceted-search.401a" + ], + "desc": "Faceted search applies price and stock filters but ignores the selected category.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".filter(item => !categoryFilter || categoryByItem.get(item.id) === categoryFilter)", + "replace": ".filter(() => true) // mutant: category filter is ignored" + } + ] + }, + { + "id": "active-search-uses-purchase-ranking", + "scenario": "tracks/ecommerce/scenarios/progression-search-ordering.json", + "targets": [ + "ecommerce.spec.search-ordering.search-ordering.402b" + ], + "desc": "Keep purchase ranking when search text or filters are active.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "const catalogItems = showingSearch ? filteredSearchResults : rankedItems;", + "replace": "const catalogItems = rankedItems;" + } + ] + }, + { + "id": "faceted-search-next-page-does-not-advance", + "scenario": "tracks/ecommerce/scenarios/progression-faceted-pagination.json", + "targets": [ + "ecommerce.progression.faceted-search.faceted-search.402a" + ], + "desc": "The next-page control keeps the search on its current page.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "onClick={() => setSearchPage(page => page + 1)}>Next", + "replace": "onClick={() => setSearchPage(page => page)}>Next" + } + ] + }, + { + "id": "managed-support-leaks-and-accepts-cross-account-replies", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-privacy.json", + "targets": [ + "ecommerce.spec.access-control.managed-support-privacy.613b" + ], + "desc": "Managed support tickets are visible across accounts and replayed replies are accepted.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))", + "replace": ".filter(() => true)" + }, + { + "file": "backend/spacetimedb/src/index.ts", + "find": "if (!actor.isAdmin && !actor.isStaff && ticket.accountId !== actor.id) {\n throw new SenderError('That support ticket is private.');\n }", + "replace": "// mutant: any signed-in account can access any support ticket" + } + ] + }, + { + "id": "managed-support-replies-are-empty", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared.json", + "targets": [ + "ecommerce.spec.live-state.managed-support.613a", + "ecommerce.progression.managed-support.managed-support.613c" + ], + "desc": "Managed support stores replies without their message body. Both the ordinary reply and shared-live reply assertions require the stored message body; neither can pass an empty reply.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "body: body.trim(),\n createdMicros: nowMicros(ctx),", + "replace": "body: '',\n createdMicros: nowMicros(ctx)," + } + ] + }, + { + "id": "managed-support-live-replies-stay-at-initial-snapshot", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared.json", + "targets": [ + "ecommerce.spec.live-state.managed-support.613a" + ], + "desc": "Keep the initial subscribed reply snapshot on each page. Reducers still save replies, and a reload shows them, but later replies do not reach the rendered conversation live.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [supportReplyRows] = useTable(tables.visibleSupportReplies);", + "replace": " const [liveSupportReplyRows, supportRepliesReady] = useTable(tables.visibleSupportReplies);\n const supportReplyRows = useMemo(() => [...liveSupportReplyRows], [supportRepliesReady]);" + } + ] + }, + { + "id": "notification-preferences-are-not-saved", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.state-durability.notification-preferences-reload.630a" + ], + "desc": "Saving notification preferences discards the selected values.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (existing) ctx.db.notificationPreference.accountId.update(row);\n else ctx.db.notificationPreference.insert(row);", + "replace": "void existing;\n void row; // mutant: notification preferences are discarded" + } + ] + }, + { + "id": "notification-preferences-leak-across-accounts", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.access-control.notification-preferences-privacy.630b" + ], + "desc": "The preference view returns another account's first stored choice.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const row = ctx.db.notificationPreference.accountId.find(accountId);\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;", + "replace": "const row = [...ctx.db.notificationPreference.iter()][0];\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;" + } + ] + }, + { + "id": "checkout-records-zero-payment", + "scenario": "tracks/ecommerce/scenarios/progression-core-business.json", + "targets": [ + "ecommerce.progression.payment-records.payment-records.623a" + ], + "desc": "Checkout records a paid payment with a zero amount.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {", + "replace": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: 0, status: 'paid' });\n if (promo) {" + } + ] + }, + { + "id": "checkout-records-duplicate-payments", + "scenario": "tracks/ecommerce/scenarios/progression-core-business.json", + "targets": [ + "ecommerce.spec.transactional-integrity.payment-deduplication.623b" + ], + "desc": "Checkout inserts two payment records for one order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {", + "replace": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total + 0.01, status: 'paid' });\n if (promo) {" + } + ] + }, + { + "id": "active-promotion-does-not-discount-checkout", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-checkout.json", + "targets": [ + "ecommerce.progression.promotion-checkout.promotion-checkout-active.621a" + ], + "desc": "Checkout ignores an active promotion when it calculates the discount.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const discount = promo ? total * (promo.discountPercent / 100) : 0;", + "replace": "const discount = 0;" + } + ] + }, + { + "id": "expired-promotion-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-checkout.json", + "targets": [ + "ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b" + ], + "desc": "Promotion application does not reject a promotion after its end time.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {", + "replace": "if (!promo || promo.startMicros > now || promo.redemptions >= promo.usageLimit) {" + } + ] + }, + { + "id": "exhausted-promotion-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-checkout.json", + "targets": [ + "ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c" + ], + "desc": "Promotion application does not reject a promotion at its usage limit.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {", + "replace": "if (!promo || promo.startMicros > now || promo.endMicros < now) {" + } + ] + }, + { + "id": "customers-can-create-promotions", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.spec.access-control.promotion-management-boundary.620b" + ], + "desc": "Promotion creation does not require staff access.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, input) => {\n requireStaffOrAdmin(ctx);\n if (input.discountPercent <= 0 || input.discountPercent > 100) {", + "replace": " (ctx, input) => {\n if (input.discountPercent <= 0 || input.discountPercent > 100) {" + } + ] + }, + { + "id": "promotion-rule-stores-the-wrong-discount", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.progression.promotion-rules.promotion-rule-values.620a" + ], + "desc": "Promotion creation stores a one-percent discount instead of the submitted value.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.promotion.insert({ id: 0n, ...input, code: input.code.trim(), redemptions: 0 });", + "replace": "ctx.db.promotion.insert({ id: 0n, ...input, discountPercent: 1, code: input.code.trim(), redemptions: 0 });" + } + ] + }, + { + "id": "staff-cannot-open-staff-tools", + "scenario": "tracks/ecommerce/scenarios/progression-staff-access.json", + "targets": [ + "ecommerce.progression.staff-access.staff-access.601a" + ], + "desc": "Deny administrators entry to staff tools while keeping ordinary staff access working.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "{(isStaff || isAdmin) && (\n ({", + "replace": "return [...ctx.db.notification.iter()].map(row => ({" + } + ] + }, + { + "id": "support-history-is-lost-on-fresh-account-login", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.state-durability.support-history-reload.612a" + ], + "desc": "Resolve customer support ownership by transport identity only, omitting the account ownership path. Initial submission, same-identity reload, and stored tickets remain intact; fresh account login cannot recover the history.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "!!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))", + "replace": "!!actor && (actor.isAdmin || actor.isStaff || row.creatorIdentity.toHexString() === sender))" + } + ] + }, + { + "id": "support-history-leaks-across-customers", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Return all support tickets, exposing them to other customers and signed-out visitors.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))", + "replace": ".filter(() => true)" + } + ] + }, + { + "id": "visitor-support-reference-is-hidden", + "scenario": "tracks/ecommerce/scenarios/progression-support-intake.json", + "targets": [ + "ecommerce.progression.support-intake.support-intake.610a" + ], + "desc": "A visitor can create a support ticket but the returned reference is not rendered.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": "
{supportReference}
", + "replace": "
" + } + ] + }, + { + "id": "support-assignment-is-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage.json", + "targets": [ + "ecommerce.progression.support-triage.support-assignment.611a" + ], + "desc": "Support triage saves status and priority but discards the assignee.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });", + "replace": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: undefined, priority, status });" + } + ] + }, + { + "id": "support-priority-is-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage.json", + "targets": [ + "ecommerce.progression.support-triage.support-priority.611b" + ], + "desc": "Support triage always saves normal priority instead of the submitted value.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });", + "replace": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority: 'normal', status });" + } + ] + }, + { + "id": "support-status-is-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage.json", + "targets": [ + "ecommerce.progression.support-triage.support-status.611c" + ], + "desc": "Support triage preserves the old status instead of the submitted value.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });", + "replace": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status: ticket.status });" + } + ] + }, + { + "id": "nonpositive-cart-quantity-is-treated-as-removal", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109b" + ], + "desc": "Accept a negative quantity and remove the cart line instead of refusing the request.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (quantity < 1) throw new SenderError('Quantity must be at least 1.');\n const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n if (existing.bundlePrice > 0) throw new SenderError('Remove and re-add a whole bundle.');\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });", + "replace": " const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n if (quantity < 1) {\n ctx.db.cartItem.id.delete(existing.id);\n return;\n }\n if (existing.bundlePrice > 0) throw new SenderError('Remove and re-add a whole bundle.');\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });" + } + ] + }, + { + "id": "admin-restock-preserves-existing-stock", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json", + "targets": [ + "ecommerce.spec.live-state.warehouse-stock.7c" + ], + "desc": "Accept an administrator restock but write the existing quantity back unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...existing, quantity: existing.quantity });" + } + ] + }, + { + "id": "operator-authorization-allows-customer-price-change", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201b" + ], + "desc": "The price reducer drops its administrator role gate.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, { itemId, price }) => {\n requireAdmin(ctx);", + "replace": " (ctx, { itemId, price }) => {\n // mutant: no administrator role check" + } + ] + }, + { + "id": "fulfilment-queue-allows-customer-shipping", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1e" + ], + "desc": "The shipping reducer drops the staff role check while retaining the pending-order guard.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);", + "replace": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check" + } + ] + }, + { + "id": "catalog-search-keeps-pre-change-price", + "scenario": "tracks/ecommerce/scenarios/02-live-price.json", + "targets": [ + "ecommerce.returns-pricing.price-history.4b" + ], + "desc": "The search result renderer caches each item's first visible price and ignores later live price updates.", + "file": "client/src/components/ItemCard.tsx", + "edits": [ + { + "find": "import { ItemRow } from '../types';", + "replace": "import { useRef } from 'react';\nimport { ItemRow } from '../types';" + }, + { + "find": " const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;", + "replace": " const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;\n // mutant: the card retains the first price it renders\n const firstPrice = useRef(item.price);" + }, + { + "find": " {formatMoney(item.price)}", + "replace": " {formatMoney(firstPrice.current)}" + } + ] + }, + { + "id": "open-cart-keeps-pre-change-price", + "scenario": "tracks/ecommerce/scenarios/progression-price-cart-checkout.json", + "targets": [ + "ecommerce.returns-pricing.price-history.4c" + ], + "desc": "The open-cart memo ignores reactive item-table price updates while checkout still reads the current server price.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " [cartRows, items, stockByItem]", + "replace": " [cartRows, stockByItem]" + } + ] + }, + { + "id": "catalog-price-rewrites-receipts", + "scenario": "tracks/ecommerce/scenarios/02-paid-price-history.json", + "targets": [ + "ecommerce.returns-pricing.price-history.4a" + ], + "desc": "Changing a catalog price cascades into saved order lines and recomputes historical order totals inside the reducer transaction.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.item.id.update({ ...it, price });", + "replace": " ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }" + } + ] + }, + { + "id": "catalog-price-rewrites-earned-revenue", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203b" + ], + "desc": "Changing a catalog price cascades into saved order lines and recomputes historical order totals and revenue inside the reducer transaction.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.item.id.update({ ...it, price });", + "replace": " ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }" + } + ] + }, + { + "id": "returned-line-marker-omitted", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3c" + ], + "desc": "A returned order line keeps its persisted returned state, restored stock, and adjusted revenue but omits the visible returned marker.", + "file": "client/src/components/OrdersPanel.tsx", + "edits": [ + { + "find": "{item.returned && Returned}", + "replace": "{false && Returned}" + } + ] + }, + { + "id": "direct-review-access-is-not-checked", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "The direct review action accepts a review from a customer who did not buy the item.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!bought) throw new SenderError('You can only review items you have purchased.');", + "replace": " // mutant: purchase eligibility is not checked" + } + ] + }, + { + "id": "support-history-rows-are-hidden", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.progression.support-history.support-history.612c", + "ecommerce.spec.state-durability.support-history-reload.612a", + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Hide submitted support ticket rows while leaving the submission reference available. History, reload, privacy, and logout require the owner to see the ticket first. This breaks those positive observations; it does not create a privacy leak or remove stored data.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": "data-role=\"support-ticket\"", + "replace": "data-role=\"support-ticket\" style={{ display: \"none\" }}" + } + ] + }, + { + "id": "authorized-restock-does-not-change-stock", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Accept an administrator restock without changing warehouse stock.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (existing) {\n ctx.db.stock.by_item_warehouse.delete([itemId, warehouseId]);\n ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n", + "replace": " // mutant: accept restock without changing stock\n" + } + ] + }, + { + "id": "low-stock-threshold-is-two-units", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5e", + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "The dashboard lists only items with two units or fewer, so the seeded three-unit item is missing from the low-stock view. The live check in the same scenario opens with the identical observation and necessarily fails with it.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "const LOW_STOCK_THRESHOLD = 10;", + "replace": "const LOW_STOCK_THRESHOLD = 2;" + } + ] + }, + { + "id": "category-totals-count-only-since-the-dashboard-opened", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5f" + ], + "desc": "The category table shows units and revenue accumulated since the dashboard was opened instead of the stored totals, so a reload resets both to zero and the totals recorded before the reload are not reproduced. Live movement within one open dashboard is still correct, so the live check is unaffected.", + "file": "client/src/components/AdminPanel.tsx", + "edits": [ + { + "find": "import { useState } from 'react';", + "replace": "import { useRef, useState } from 'react';" + }, + { + "find": " return (\n
", + "replace": " const openingTotals = useRef | null>(null);\n if (openingTotals.current === null && categoryTotals.length > 0) {\n openingTotals.current = new Map(\n categoryTotals.map((cat): [bigint, { units: number; revenue: number }] => [\n cat.categoryId,\n { units: cat.unitsSold, revenue: cat.revenue },\n ])\n );\n }\n const sessionTotals = categoryTotals.map((cat) => {\n const opening = openingTotals.current?.get(cat.categoryId);\n return {\n ...cat,\n unitsSold: cat.unitsSold - (opening?.units ?? 0),\n revenue: cat.revenue - (opening?.revenue ?? 0),\n };\n });\n\n return (\n
" + }, + { + "find": " {categoryTotals.map((cat) => (", + "replace": " {sessionTotals.map((cat) => (" + } + ] + }, + { + "id": "profile-summary-ignores-a-profile-saved-this-session", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.progression.customer-profile.customer-profile.620c" + ], + "desc": "Hide the profile summary immediately after saving in the current view. Stored profile data and a reopened or reloaded view remain correct, so fresh-login durability and privacy positive controls remain observable.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": " const [profileName, setProfileName] = useState(profile?.name ?? '');", + "replace": " const [profileName, setProfileName] = useState(profile?.name ?? '');\n const [profileSavedHere, setProfileSavedHere] = useState(false);" + }, + { + "find": "onClick={() => reducers?.saveProfile({ name: profileName, address: profileAddress })}", + "replace": "onClick={() => { setProfileSavedHere(true); return reducers?.saveProfile({ name: profileName, address: profileAddress }); }}" + }, + { + "find": "
{profile?.name} {profile?.address}
", + "replace": "
{!profileSavedHere && <>{profile?.name} {profile?.address}}
" + } + ] + }, + { + "id": "stored-support-replies-are-hidden-after-reload", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared.json", + "targets": [ + "ecommerce.progression.managed-support.managed-support.613c" + ], + "desc": "Replies already stored when the page loads are hidden and only replies that arrive while the page is open are shown, so a reloaded customer or staff member cannot see the earlier exchange. The live shared-case check exchanges only new replies and is unaffected.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [supportReplyRows] = useTable(tables.visibleSupportReplies);", + "replace": " const [liveSupportReplyRows, supportRepliesReady] = useTable(tables.visibleSupportReplies);\n const openedSupportReplies = useRef | null>(null);\n if (openedSupportReplies.current === null && supportRepliesReady) {\n openedSupportReplies.current = new Set(\n liveSupportReplyRows.map((row) => `${row.ticketId}:${row.author}:${row.body}`)\n );\n }\n const supportReplyRows = liveSupportReplyRows.filter(\n (row) => !openedSupportReplies.current?.has(`${row.ticketId}:${row.author}:${row.body}`)\n );" + } + ] + }, + { + "id": "saving-notification-preferences-resets-the-toggles", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.progression.notification-preferences.notification-preferences.630c" + ], + "desc": "Saving sends the chosen preferences but resets both toggles to off and stops the form from following the stored row for the rest of the session, so the saved choice cannot be seen until a reload. The reload and cross-account checks read the stored row on a fresh page and are unaffected.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": " useEffect(() => {\n setOrderEnabled(preferences?.orderEnabled ?? false);\n setStockEnabled(preferences?.stockEnabled ?? false);\n }, [preferences?.orderEnabled, preferences?.stockEnabled]);", + "replace": " const [preferencesSubmitted, setPreferencesSubmitted] = useState(false);\n useEffect(() => {\n if (preferencesSubmitted) return;\n setOrderEnabled(preferences?.orderEnabled ?? false);\n setStockEnabled(preferences?.stockEnabled ?? false);\n }, [preferences?.orderEnabled, preferences?.stockEnabled, preferencesSubmitted]);" + }, + { + "find": "onClick={() => reducers?.saveNotificationPreferences({ orderEnabled, stockEnabled })}", + "replace": "onClick={() => { reducers?.saveNotificationPreferences({ orderEnabled, stockEnabled }); setPreferencesSubmitted(true); setOrderEnabled(false); setStockEnabled(false); }}" + } + ] + }, + { + "id": "saving-a-staff-role-snaps-the-input-back-to-the-stored-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.progression.staff-roles.staff-roles.621c" + ], + "desc": "Saving a role sends the new role to the server but snaps the visible input back to the role stored before the save for the rest of the session, so the administrator cannot see the assignment take. A reload renders the stored role, so the durability and boundary checks are unaffected.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": " ", + "replace": " " + } + ] + }, + { + "id": "fulfilment-queue-is-frozen-at-page-load", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.spec.live-state.fulfilment-queue.1a" + ], + "desc": "The fulfilment queue renders the rows delivered with the page's initial subscription and ignores later updates, so an order placed while the queue is open never appears without a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [queueRows] = useTable(tables.fulfilmentQueue);", + "replace": " const [liveQueueRows, queueReady] = useTable(tables.fulfilmentQueue);\n const openedQueueRows = useRef(null);\n if (openedQueueRows.current === null && queueReady) openedQueueRows.current = liveQueueRows;\n const queueRows = openedQueueRows.current ?? liveQueueRows;" + } + ] + }, + { + "id": "low-stock-list-is-frozen-at-page-load", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "The low-stock list is computed once from the first complete stock snapshot and never recomputed, so items no longer enter or leave it as stock is restocked or sold. The seeded low item is in that first snapshot, so the static listing check is unaffected.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const lowStockItems = useMemo(\n () =>\n [...items]\n .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)\n .sort((a, b) => (stockByItem.get(a.id) ?? 0) - (stockByItem.get(b.id) ?? 0)),\n [items, stockByItem]\n );", + "replace": " const liveLowStockItems = useMemo(\n () =>\n [...items]\n .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)\n .sort((a, b) => (stockByItem.get(a.id) ?? 0) - (stockByItem.get(b.id) ?? 0)),\n [items, stockByItem]\n );\n const openedLowStockItems = useRef(null);\n if (openedLowStockItems.current === null && items.length > 0 && stocks.length > 0) {\n openedLowStockItems.current = liveLowStockItems;\n }\n const lowStockItems = openedLowStockItems.current ?? liveLowStockItems;" + } + ] + }, + { + "id": "category-totals-are-frozen-at-page-load", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.spec.live-state.sales-dashboard.5b" + ], + "desc": "The category totals render the first non-empty row set received and ignore later updates, so a purchase does not move units or revenue while the dashboard is open. A reload receives the stored totals, so the reload check is unaffected.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [categoryTotalRows] = useTable(tables.categoryTotals);", + "replace": " const [liveCategoryTotalRows] = useTable(tables.categoryTotals);\n const openedCategoryTotalRows = useRef(null);\n if (openedCategoryTotalRows.current === null && liveCategoryTotalRows.length > 0) {\n openedCategoryTotalRows.current = liveCategoryTotalRows;\n }\n const categoryTotalRows = openedCategoryTotalRows.current ?? liveCategoryTotalRows;" + } + ] + }, + { + "id": "warehouse-totals-are-frozen-at-page-load", + "scenario": "tracks/ecommerce/scenarios/02-transfer-totals.json", + "targets": [ + "ecommerce.spec.live-state.stock-transfers.2b" + ], + "desc": "The per-warehouse totals are computed once from the first stock snapshot and never recomputed, so a transfer moves neither warehouse figure while the dashboard is open.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const stockByWarehouse = useMemo(() => {\n const map = new Map();\n for (const row of stocks) {\n map.set(row.warehouseId, (map.get(row.warehouseId) ?? 0) + row.quantity);\n }\n return map;\n }, [stocks]);", + "replace": " const liveStockByWarehouse = useMemo(() => {\n const map = new Map();\n for (const row of stocks) {\n map.set(row.warehouseId, (map.get(row.warehouseId) ?? 0) + row.quantity);\n }\n return map;\n }, [stocks]);\n const openedStockByWarehouse = useRef(null);\n if (openedStockByWarehouse.current === null && liveStockByWarehouse.size > 0) {\n openedStockByWarehouse.current = liveStockByWarehouse;\n }\n const stockByWarehouse = openedStockByWarehouse.current ?? liveStockByWarehouse;" + } + ] + }, + { + "id": "transfer-skips-the-source-holding-check", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c" + ], + "desc": "The serialized transfer reducer no longer checks that the source warehouse holds the requested quantity, so an overdraw is accepted instead of refused and the source quantity wraps below zero.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (available < quantity) {\n throw new SenderError(`Not enough stock in source warehouse: only ${available} available.`);\n }", + "replace": " // mutant: the source warehouse holding is not checked" + } + ] + }, + { + "id": "credit-checkout-ignores-wallet", + "desc": "A credit checkout pays entirely externally despite available credit.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.feature.store-credit.store-credit-750.750a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const creditMinor = useCredit ? Math.min(wallet?.amountMinor ?? 0,totalMinor) : 0;", + "replace": " const creditMinor = 0;" + } + ] + }, + { + "id": "credit-grant-replay-increments-balance", + "desc": "Replaying a grant applies its credit to the wallet again.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-752.752a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (existing.amountMinor !== amountMinor) throw new SenderError('Reference identifies another grant.');\n return;", + "replace": " if (existing.amountMinor !== amountMinor) throw new SenderError('Reference identifies another grant.');\n const wallet = ctx.db.creditWallet.accountId.find(accountId)!;\n ctx.db.creditWallet.accountId.update({ ...wallet, amountMinor: wallet.amountMinor + amountMinor });\n return;" + } + ] + }, + { + "id": "customer-can-grant-credit", + "desc": "Customer authentication is accepted without staff authorization.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-753.753a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, { accountId, amountMinor, reference }) => {\n requireStaffOrAdmin(ctx);", + "replace": " (ctx, { accountId, amountMinor, reference }) => {\n requireAccount(ctx);" + } + ] + }, + { + "id": "split-refund-does-not-restore-credit", + "desc": "The refund is recorded but its original wallet credit is not restored.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " refundOrderCredit(ctx, order, order.total);", + "replace": " // mutant: omit wallet restoration" + } + ] + }, + { + "id": "split-refund-duplicates-credit", + "desc": "A refund credits the wallet twice while recording one refund.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.spec.split-tender-refunds.production-756.756a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "amountMinor: wallet.amountMinor + delta", + "replace": "amountMinor: wallet.amountMinor + delta * 2" + } + ] + }, + { + "id": "subscription-skips-due-purchase", + "desc": "Due deliveries are recorded as skipped although stock is available.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.feature.subscriptions.subscriptions-760.760a" + ], + "file": "backend/spacetimedb/src/subscriptions.ts", + "edits": [ + { + "find": " const orderId = purchase(row.accountId, row.itemId, row.quantity, row.price);", + "replace": " const orderId: bigint | null = null;" + } + ] + }, + { + "id": "subscription-allows-foreign-cancellation", + "desc": "A customer can cancel another customer subscription.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-762.762a" + ], + "file": "backend/spacetimedb/src/subscriptions.ts", + "edits": [ + { + "find": " if (!row || row.accountId !== accountId) throw new SenderError('Subscription access denied.');", + "replace": " if (!row) throw new SenderError('Subscription access denied.');" + } + ] + }, + { + "id": "subscription-pause-is-not-recorded", + "desc": "Pause acknowledges the request but the subscription remains active.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-763.763a" + ], + "file": "backend/spacetimedb/src/subscriptions.ts", + "edits": [ + { + "find": " ctx.db.purchaseSubscription.id.update({ ...row, status: 'paused', pausedMicros: now });", + "replace": " ctx.db.purchaseSubscription.id.update({ ...row, status: 'active', pausedMicros: now });" + } + ] + }, + { + "id": "credit-checkout-retains-purchased-cart", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-754.754a" + ], + "desc": "The purchased cart remains available instead of being consumed by checkout.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": " // mutant: checked-out lines remain in the cart" + } + ] + }, + { + "id": "reconnection-erases-stored-credit", + "desc": "A new connection erases stored wallet credit; the fresh view after restart must catch the data loss.", + "file": "backend/spacetimedb/src/index.ts", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-755.755a" + ], + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onConnect = spacetimedb.clientConnected(ctx => { for (const row of ctx.db.creditWallet.iter()) ctx.db.creditWallet.accountId.update({ ...row, amountMinor: 0 }); });\n// --- views ---" + } + ] + }, + { + "id": "reconnection-cancels-pending-subscriptions", + "desc": "A new connection cancels pending subscriptions; restarting and reconnecting must preserve this work.", + "file": "backend/spacetimedb/src/index.ts", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-761.761a" + ], + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onConnect = spacetimedb.clientConnected(ctx => { for (const row of ctx.db.purchaseSubscription.iter()) if (row.status === 'active') ctx.db.purchaseSubscription.id.update({ ...row, status: 'cancelled' }); });\n// --- views ---" + } + ] + }, + { + "id": "bundle-definition-loses-component-quantity", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.feature.product-bundles.product-bundles.740a" + ], + "desc": "definition loses component quantity", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "return { ...component, itemId: String(product.id) };", + "replace": "return { ...component, quantity: 1, itemId: String(product.id) };" + } + ] + }, + { + "id": "bundle-catalog-write-allows-customers", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-743.743a" + ], + "desc": "catalog write allows customers", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const actor = requireStaffOrAdmin(ctx);\n if (!actor.isAdmin && ctx.db.staffRole.accountId.find(actor.id)?.role !== 'catalog') {", + "replace": "const actor = requireAccount(ctx);\n if (false) {" + } + ] + }, + { + "id": "bundle-checkout-price-not-snapshot", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.feature.bundle-checkout.bundle-checkout.741a" + ], + "desc": "checkout price not snapshot", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "bundlePrice: product.price, bundleComponentsJson: definition.componentsJson", + "replace": "bundlePrice: product.price + 1, bundleComponentsJson: definition.componentsJson" + } + ] + }, + { + "id": "bundle-return-loses-original-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.feature.bundle-returns.bundle-returns.742a" + ], + "desc": "return loses original components", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const line of bundles) {\n restoreOrderItemStock(ctx, line);", + "replace": "for (const line of bundles) {\n // mutant: original stock is not restored" + } + ] + }, + { + "id": "bundle-return-replay-restocks-again", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-742.742b" + ], + "desc": "return replay restocks again", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => row.isBundle && !row.returned);", + "replace": ".filter(row => row.isBundle);" + } + ] + }, + { + "id": "bundle-return-crosses-account-boundary", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-748.748a" + ], + "desc": "return crosses account boundary", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!order || order.accountId !== account.id || !['shipped', 'delivered'].includes(order.status))", + "replace": "if (!order || !['shipped', 'delivered'].includes(order.status))" + } + ] + }, + { + "id": "bundle-components-can-overdraw", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-744.744a", + "ecommerce.spec.bundle-integrity.bundle-745.745a" + ], + "desc": "components can overdraw", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!allocations) throw new SenderError('Not enough stock to reserve.');", + "replace": "if (!allocations) { if (bundleId) return; throw new SenderError('Not enough stock to reserve.'); }" + } + ] + }, + { + "id": "bundle-checkout-reuses-reservation", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-747.747a" + ], + "desc": "checkout reuses reservation", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const row of held) ctx.db.reservation.id.delete(row.id);\n processReorderRules(ctx, p.itemId);", + "replace": "// mutant: reservations survive checkout\n processReorderRules(ctx, p.itemId);" + }, + { + "find": "for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": "// mutant: cart survives checkout" + } + ] + }, + { + "id": "bundle-expiry-does-not-release-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-746.746a" + ], + "desc": "Expired bundle reservations retain their component stock after restart.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (row.expired || row.expiresMicros > now) continue;\n restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity);", + "replace": "if (row.expired || row.expiresMicros > now) continue;\n if (!row.stockItemId) restoreStock(ctx, row.itemId, row.warehouseId, row.quantity);" + } + ] + }, + { + "id": "return-after-support-refund-is-blocked", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757a" + ], + "desc": "Reject a valid physical return after a financial refund.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (target.returned) throw new SenderError('Item already returned.');", + "replace": " if (target.returned || order.refundedTotal > 0) throw new SenderError('Item already returned.');" + } + ] + }, + { + "id": "support-refund-after-return-pays-twice", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757b" + ], + "desc": "Pay the full order again after the physical return already refunded it.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const amount = order.total - order.refundedTotal;", + "replace": " const amount = order.total;" + }, + { + "find": "refundedTotal: order.total, status: order.status", + "replace": "refundedTotal: order.refundedTotal + amount, status: order.status" + } + ] + }, + { + "id": "support-history-leaks-to-signed-out-visitors", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Disclose support tickets to signed-out visitors while leaving signed-in ownership filtering unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||", + "replace": ".filter(row => !actor || isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||" + } + ] + }, + { + "id": "checkout-crash-integrity", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { for (const row of ctx.db.cartItem.iter()) if ([...ctx.db.customerOrder.iter()].some(order => order.accountId === row.accountId)) ctx.db.cartItem.id.update({ ...row, quantity: 2 }); });\n// --- views ---" + } + ] + }, + { + "id": "checkout-crash-sibling-stock", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A disconnect callback removes second-product stock once after a cart and prior order exist. A private ledger marker prevents repeated damage to later setup. The control tests stock observation after a crash, not the time at which the defect was injected.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { if ([...ctx.db.stockLedger.iter()].some(row => row.source === 'stock fault control') || ![...ctx.db.cartItem.iter()].some(row => [...ctx.db.customerOrder.iter()].some(order => order.accountId === row.accountId))) return; const item = [...ctx.db.item.iter()].find(row => row.name === 'Coffee Grinder'); if (!item) return; const row = [...ctx.db.stock.by_item_warehouse.filter(item.id)][0]; if (row) { ctx.db.stock.by_item_warehouse.delete([row.item_id, row.warehouse_id]); ctx.db.stock.insert({ ...row, quantity: row.quantity - 1 }); ctx.db.stockLedger.insert({ id: 0n, itemId: row.item_id, warehouseId: row.warehouse_id, quantity: 1, createdMicros: ctx.timestamp.microsSinceUnixEpoch, source: 'stock fault control' }); } });\n// --- views ---" + } + ] + }, + { + "id": "checkout-crash-durability", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-durability.910b" + ], + "desc": "A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { for (const order of ctx.db.customerOrder.iter()) if ([...ctx.db.cartItem.iter()].some(row => row.accountId === order.accountId)) ctx.db.customerOrder.id.update({ ...order, total: 0 }); });\n// --- views ---" + } + ] + }, + { + "id": "review-script-unsafe-render", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Render stored review text as HTML so it executes in another customer session.", + "file": "client/src/components/ItemDetail.tsx", + "edits": [ + { + "find": "
{r.comment}
", + "replace": "
" + } + ] + }, + { + "id": "review-script-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Reject all nonempty reviews, including the required benign review; absence of script is insufficient.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "(ctx, { itemId, rating, comment }) => {", + "replace": "(ctx, { itemId, rating, comment }) => {\n if (comment.length > 0) throw new SenderError(\"Review rejected\");" + } + ] + }, + { + "id": "review-owner-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Reject the legitimate buyer review as well as unauthorized reviews.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "(ctx, { itemId, rating, comment }) => {", + "replace": "(ctx, { itemId, rating, comment }) => {\n if (comment.length > 0) throw new SenderError(\"Reject all reviews\");" + } + ] + }, + { + "id": "password-verification-truncates-utf8", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Hash and verify only the first 72 UTF-8 bytes, accepting a different password suffix.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "scrypt(password, salt, {", + "replace": "scrypt(new TextEncoder().encode(password).slice(0, 72), salt, {" + } + ] + }, + { + "id": "password-verification-rejects-every-login", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1c" + ], + "desc": "Refuse even a correct password; refusal alone must not earn credit.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account && sameHash(digest, account.passwordHash) && false;" + } + ] + }, + { + "id": "order-cache-survives-account-switch", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Persist order views in a browser cache that is not scoped to the account. Login reloads no longer clear the stale data.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const orderViews: OrderView[] = useMemo(", + "replace": " const freshOrderViews: OrderView[] = useMemo(" + }, + { + "find": " const revenue = revenueRows[0]?.total ?? 0;", + "replace": " const [orderCache] = useState(() => new Map(\n (JSON.parse(sessionStorage.getItem('order-cache') ?? '[]', (key, value) =>\n key === 'orderId' || key === 'itemId' ? BigInt(value) : key === 'createdAt' ? new Date(value) : value) as OrderView[])\n .map(order => [order.orderId, order])));\n for (const order of freshOrderViews) if (order.items.length) orderCache.set(order.orderId, order);\n const orderViews = [...orderCache.values()];\n sessionStorage.setItem('order-cache', JSON.stringify(orderViews, (_key, value) => typeof value === 'bigint' ? String(value) : value));\n\n const revenue = revenueRows[0]?.total ?? 0;" + } + ] + }, + { + "id": "checkout-records-wrong-total", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Store an incorrect checkout total while retaining the normal cart and item lines. Native extra arguments remain rejected.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: total - discount,\n status: 'pending',", + "replace": " total: 1,\n status: 'pending'," + } + ] + }, + { + "id": "checkout-missing-sibling-line", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit the cart checkout with one product line missing while retaining the other product and the full order total.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const p of priced) {", + "replace": " for (const p of priced) {\n if (p.name === 'Coffee Grinder') continue;" + } + ] + }, + { + "id": "shipping-cancellation-cancel-shipped", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation accepts a shipped order and restores its stock after shipping.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (order.status !== 'pending') throw new SenderError('Order has already shipped.');", + "replace": "// Permit the invalid order transition for this control." + } + ] + }, + { + "id": "shipping-cancellation-resurrect-cancelled", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Shipping accepts a cancelled order after its stock was restored.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (order.status !== 'pending') throw new SenderError('Order is not pending.');", + "replace": "// Permit the invalid order transition for this control." + } + ] + }, + { + "id": "shipping-cancellation-combined-race", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation restores stock but records a shipped status, combining mutually exclusive effects.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (order.status !== 'pending') throw new SenderError('Order has already shipped.');", + "replace": "// Permit the invalid order transition for this control." + }, + { + "find": "ctx.db.customerOrder.id.update({ ...order, status: 'cancelled' });", + "replace": "ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });" + } + ] + }, + { + "id": "shipping-cancellation-reject-both", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Both cancellation and shipping refuse pending orders; useful progress must remain required.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (order.status !== 'pending') throw new SenderError('Order has already shipped.');", + "replace": "if (order.status === 'pending') throw new SenderError('Cancellation disabled');" + }, + { + "find": "if (order.status !== 'pending') throw new SenderError('Order is not pending.');", + "replace": "if (order.status === 'pending') throw new SenderError('Shipping disabled');" + } + ] + }, + { + "id": "shipping-cancellation-queue-retains-removed-order", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "The live staff queue accepts new orders but retains rows after cancellation; the database state remains correct.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [queueRows] = useTable(tables.fulfilmentQueue);", + "replace": " const [liveQueueRows] = useTable(tables.fulfilmentQueue);\n const retainedQueueRows = useRef([]);\n if (liveQueueRows.length >= retainedQueueRows.current.length) retainedQueueRows.current = liveQueueRows;\n const queueRows = retainedQueueRows.current;" + } + ] + }, + { + "id": "opposing-transfer-ignored", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is acknowledged without moving stock.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (quantity < 1) throw new SenderError('Transfer quantity must be at least 1.');", + "replace": " if (quantity < 1) throw new SenderError('Transfer quantity must be at least 1.');\n if (quantity === 7) return;" + } + ] + }, + { + "id": "opposing-transfer-rejected", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is refused despite ample source stock.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (quantity < 1) throw new SenderError('Transfer quantity must be at least 1.');", + "replace": " if (quantity < 1) throw new SenderError('Transfer quantity must be at least 1.');\n if (quantity === 7) throw new SenderError('Transfer disabled');" + } + ] + }, + { + "id": "overlapping-cart-add-ignored", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is ignored while ordinary cart setup still works.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);", + "replace": " if (acc.username.includes('checkout-overlap') && it.name === 'Coffee Grinder') return;\n const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);" + } + ] + }, + { + "id": "overlapping-cart-add-rejected", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is rejected while ordinary cart setup still works.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);", + "replace": " if (acc.username.includes('checkout-overlap') && it.name === 'Coffee Grinder') throw new SenderError('Cart add disabled');\n const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);" + } + ] + }, + { + "id": "mixed-history-lost-transfer", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: lost-transfer. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity });" + } + ] + }, + { + "id": "mixed-history-wrong-price", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-price. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: Math.round(price * 100) * quantity / 100,\n status: 'pending',", + "replace": " total: 1,\n status: 'pending'," + } + ] + }, + { + "id": "mixed-history-ignored-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: ignored-write. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);", + "replace": " if (acc.username.includes('history-') && it.name === 'Coffee Grinder') return;\n const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);" + } + ] + }, + { + "id": "mixed-history-reject-valid-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: reject-valid-write. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);", + "replace": " if (acc.username.includes('history-') && it.name === 'Coffee Grinder') throw new SenderError('Cart add disabled');\n const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);" + } + ] + }, + { + "id": "mixed-history-repeat-cancel", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: repeat-cancel. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (order.status !== 'pending') throw new SenderError('Order has already shipped.');", + "replace": " if (!['pending', 'cancelled'].includes(order.status)) throw new SenderError('Order has already shipped.');" + } + ] + }, + { + "id": "mixed-history-wrong-owner", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-owner. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " accountId,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100,", + "replace": " accountId: ctx.db.account.username.find('admin')!.id,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100," + } + ] + }, + { + "id": "checkout-empty-replay-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Allow retrying the consumed empty cart to create another order after a lost reply.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (lines.length === 0) throw new SenderError('Your cart is empty.');", + "replace": "// Defect: an empty consumed cart may create another order." + } + ] + }, + { + "id": "checkout-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Reject every valid checkout; refusal cannot earn duplicate protection credit.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (lines.length === 0) throw new SenderError('Your cart is empty.');", + "replace": " if (lines.length === 0) throw new SenderError('Your cart is empty.');\n if (lines.length >= 0) throw new Error('Checkout disabled');" + } + ] + }, + { + "id": "checkout-replay-loses-prior-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Retrying the consumed cart deletes an earlier acknowledged order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (lines.length === 0) throw new SenderError('Your cart is empty.');", + "replace": " if (lines.length === 0) { const prior = [...ctx.db.customerOrder.iter()].filter(order => order.accountId === acc.id).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0)[0]; if (prior) ctx.db.customerOrder.id.delete(prior.id); return; }" + } + ] + }, + { + "id": "checkout-clears-cart-without-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout returns success and empties the cart without recording an order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (lines.length === 0) throw new SenderError('Your cart is empty.');", + "replace": " if (lines.length === 0) throw new SenderError('Your cart is empty.');\n if (lines.length > 0) { for (const line of lines) ctx.db.cartItem.id.delete(line.id); return; }" + } + ] + }, + { + "id": "revenue-aggregation-truncates-order-population", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Calculate revenue from only the first 256 stored orders while preserving all acknowledged purchases.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " let total = 0;\n for (const o of ctx.db.customerOrder.iter()) {", + "replace": " let total = 0;\n for (const o of [...ctx.db.customerOrder.iter()].slice(0, 256)) {" + } + ] + }, + { + "id": "catalog-truncates-created-population", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Omit committed products after the first 256 search results.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".slice(searchPage * CATALOG_PAGE_SIZE,", + "replace": ".slice(0, 256).slice(searchPage * CATALOG_PAGE_SIZE," + } + ] + }, + { + "id": "catalog-discards-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Acknowledge catalog creation without committing a product.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const product = ctx.db.item.insert({ id: 0n, name: name.trim(), price });", + "replace": " if (name.length >= 0) return; // mutant: successful reducer without product\n const product = ctx.db.item.insert({ id: 0n, name: name.trim(), price });" + } + ] + }, + { + "id": "catalog-rejects-all-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Reject every catalog creation while keeping sign-in and catalog reads available.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const product = ctx.db.item.insert({ id: 0n, name: name.trim(), price });", + "replace": " if (price > 0) throw new SenderError('Catalog writes refused');\n const product = ctx.db.item.insert({ id: 0n, name: name.trim(), price });" + } + ] + }, + { + "id": "shipping-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Reject legitimate staff shipping as well as customer requests.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);", + "replace": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);\n if (orderId >= 0n) throw new SenderError('Shipping refused.');" + } + ] + }, + { + "id": "cancel-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Reject legitimate cancellation as well as foreign requests.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = requireOrderOwner(ctx, orderId);", + "replace": "export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = requireOrderOwner(ctx, orderId);\n if (orderId >= 0n) throw new SenderError('Cancellation refused');" + } + ] + }, + { + "id": "account-password-in-public-data", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Expose the actual submitted account password through data available to other clients.", + "edits": [ + { + "file": "backend/spacetimedb/src/auth.ts", + "find": " isAdmin: false, isStaff: false });\n bindSession(tx, account.id);", + "replace": " isAdmin: false, isStaff: false });\n const publicItem = tx.db.item.id.find(1n);\n if (publicItem) tx.db.item.id.update({ ...publicItem, name: publicItem.name + ' ' + password });\n bindSession(tx, account.id);" + } + ] + }, + { + "id": "account-disclosure-rejects-login", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Reject every valid login; absent disclosures alone must not earn a pass.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account && sameHash(digest, account.passwordHash) && false;" + } + ] + }, + { + "id": "signout-keeps-session", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Show a signed-out interface while retaining the browser credential and its purchase authority.", + "edits": [ + { + "file": "client/src/components/AuthWidget.tsx", + "find": " const [error, setError] = useState(readAuthError);", + "replace": " const [error, setError] = useState(readAuthError);\n const [signedOut, setSignedOut] = useState(false);" + }, + { + "file": "client/src/components/AuthWidget.tsx", + "find": " await connection.reducers.signOut({});\n clearToken(); location.reload();", + "replace": " setSignedOut(true); // mutant: hide the account without removing its authority" + }, + { + "file": "client/src/components/AuthWidget.tsx", + "find": " {currentUsername ? <>", + "replace": " {currentUsername && !signedOut ? <>" + } + ] + }, + { + "id": "query-like-password-bypasses-signin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Interpret the query-like password as a login bypass while retaining normal password verification.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account && (password === \"' OR '1'='1\" || sameHash(digest, account.passwordHash));" + } + ] + }, + { + "id": "purchase-session-rejects-returning-login", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Refuse every returning login; blocking all access after logout must not pass the session check.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account && sameHash(digest, account.passwordHash) && false;" + } + ] + }, + { + "id": "account-password-in-script", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in the public JavaScript bundle while normal signup and login still work.", + "edits": [ + { + "file": "client/src/main.tsx", + "find": "import React from 'react';", + "replace": "import React from 'react';\nObject.assign(window, { debugPassword: 'sb-private-pw-76dc492ea153' });" + } + ] + }, + { + "id": "account-password-in-style", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in a public stylesheet while normal signup and login still work.", + "edits": [ + { + "file": "client/src/index.css", + "find": ":root {", + "replace": ":root {\n --debug-password: \"sb-private-pw-76dc492ea153\";" + } + ] + }, + { + "id": "signup-accepts-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust a submitted signup administrator claim and grant protected stock-write authority.", + "edits": [ + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "{ name: t.string(), password: t.string(), salt: t.string() }, t.bool(),", + "replace": "{ name: t.string(), password: t.string(), salt: t.string(), claims: t.object('SignupClaims', { role: t.string() }) }, t.bool()," + }, + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "(ctx, { name, password, salt }) => {", + "replace": "(ctx, { name, password, salt, claims }) => {" + }, + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "isAdmin: false, isStaff: false });", + "replace": "isAdmin: claims.role === 'admin', isStaff: false });" + }, + { + "file": "client/src/auth.ts", + "find": "mode === 'signup' ? [name, password, salt] : [name, password]", + "replace": "mode === 'signup' ? [name, password, salt, { role: 'customer' }] : [name, password]" + } + ] + }, + { + "id": "admin-boundary-rejects-all-signups", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Refuse every new registration; blocking useful signup must not pass the authority check.", + "edits": [ + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "if (!validInput(name, password) || !/^[a-f0-9]{64}$/.test(salt)) return false;", + "replace": "if (!validInput(name, password) || !/^[a-f0-9]{64}$/.test(salt) || name.length > 0) return false;" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/response-loss.ts b/tools/stack-bench/grader/response-loss.ts new file mode 100644 index 00000000000..fb28659c19e --- /dev/null +++ b/tools/stack-bench/grader/response-loss.ts @@ -0,0 +1,90 @@ +import { createHash } from 'node:crypto'; +import type { BrowserContext, WebSocketRoute } from 'playwright'; +import { evidenceNowMs } from '../src/evidence/evidence-timing.js'; + +// Install before the actor opens its connection. This gate changes delivery, +// never request contents, database state, or the application's retry policy. +export async function installResponseLoss(context: BrowserContext) { + if (context.serviceWorkers().length) throw new Error('response loss cannot observe an existing service worker'); + const sockets = new Set<{ page: WebSocketRoute; server: WebSocketRoute }>(); + const pending = new Set>(); + const events: { kind: 'http-request' | 'http-response' | 'ws-send' | 'ws-drop'; + atMs: number; bytes?: number; sha256?: string; status?: number; path: string }[] = []; + const errors = new Set(); + context.on('serviceworker', () => { errors.add('service worker bypasses response interception'); }); + let state: 'ready' | 'armed' | 'finished' = 'ready', armedAtMs: number | null = null; + let truncated = false, release!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const record = (kind: typeof events[number]['kind'], path: string, data?: string | Buffer, status?: number) => { + // Bounded evidence: an overflow is unmeasured, never a successful fault. + if (events.length === 256) { truncated = true; return; } + events.push({ kind, path, atMs: evidenceNowMs(), ...(data === undefined ? {} : { + bytes: Buffer.byteLength(data), sha256: createHash('sha256').update(data).digest('hex'), + }), ...(status === undefined ? {} : { status }) }); + }; + // The isolated actor loses write replies on all its HTTP paths and inbound + // WebSocket messages. This also covers SDK retries and proxy URLs unchanged. + await context.route('**/*', route => { + const task = (async () => { + if (state !== 'armed' || ['GET', 'HEAD', 'OPTIONS'].includes(route.request().method())) return route.continue(); + const path = new URL(route.request().url()).pathname; + record('http-request', path, route.request().postDataBuffer() ?? Buffer.alloc(0)); + // No redirects or transport retries: one intercepted write remains one write. + const response = await route.fetch({ maxRedirects: 0, maxRetries: 0, timeout: 10_000 }); + try { + record('http-response', path, undefined, response.status()); + await released; + await route.abort('connectionreset'); + } finally { await response.dispose(); } + })().catch(() => { errors.add('HTTP response interception failed'); }); + pending.add(task); + void task.finally(() => pending.delete(task)); + return task; + }); + await context.routeWebSocket('**/*', page => { + const server = page.connectToServer(), pair = { page, server }; + sockets.add(pair); + const path = new URL(page.url()).pathname; + page.onMessage(data => { + try { + if (state === 'armed') record('ws-send', path, data); + server.send(data); + } catch { errors.add('WebSocket request forwarding failed'); } + }); + server.onMessage(data => { + try { + if (state === 'armed') record('ws-drop', path, data); + else page.send(data); + } catch { errors.add('WebSocket response forwarding failed'); } + }); + page.onClose(async (code, reason) => { + sockets.delete(pair); + try { await server.close({ code, reason }); } + catch { errors.add('WebSocket server close failed'); } + }); + server.onClose(async (code, reason) => { + sockets.delete(pair); + try { await page.close({ code, reason }); } + catch { errors.add('WebSocket client close failed'); } + }); + }); + return { + arm() { + if (state !== 'ready') throw new Error('response-loss gate can be armed only once'); + state = 'armed'; armedAtMs = evidenceNowMs(); + }, + evidence() { return { state, armedAtMs, truncated, errors: [...errors], events: [...events] }; }, + async finish() { + if (state === 'finished') return; + state = 'finished'; + release(); + // Discard the lost reply permanently. New SDK connections can recover normally. + const closed = await Promise.allSettled([...sockets].flatMap(({ page, server }) => [ + page.close({ code: 1011, reason: 'connection interrupted' }), + server.close({ code: 1011, reason: 'connection interrupted' }), + ])); + if (closed.some(result => result.status === 'rejected')) errors.add('WebSocket fault cleanup failed'); + await Promise.all([...pending]); + }, + }; +} diff --git a/tools/stack-bench/grader/transport-frames.ts b/tools/stack-bench/grader/transport-frames.ts new file mode 100644 index 00000000000..94504293019 --- /dev/null +++ b/tools/stack-bench/grader/transport-frames.ts @@ -0,0 +1,95 @@ +import { brotliDecompressSync, gunzipSync } from 'node:zlib'; +import type { Page } from 'playwright'; +import { inconclusive } from '../src/actions/actor-action-runtime.js'; + +const MAX_RECEIVED_BYTES = 8 * 1024 * 1024; + +// A SpacetimeDB server frame carries a one-byte compression tag ahead of the +// message: 0 none, 1 brotli, 2 gzip, and the SDK compresses by default. The +// message text is inline UTF-8 once decoded, so a substring search finds it +// without the harness knowing the wire format. Any other frame is kept as it +// arrived. +export function transportFrameText(payload: string | Buffer): string { + if (typeof payload === 'string') return payload; + const bytes = Buffer.from(payload); + if (bytes.length > 1) { + try { + if (bytes[0] === 1) return brotliDecompressSync(bytes.subarray(1)).toString('utf8'); + if (bytes[0] === 2) return gunzipSync(bytes.subarray(1)).toString('utf8'); + } catch { /* not a compressed SpacetimeDB frame */ } + } + return bytes.toString('utf8'); +} + +// Bounded evidence must never turn dropped data into a privacy pass. +export class ReceivedTransport { + readonly chunks: string[] = []; + private bytes = 0; + private readonly incompleteCounts = { byteLimit: 0, bodyReadFailures: 0, unsupportedStreams: 0 }; + incomplete = false; + pending = 0; + + constructor(private readonly limit = MAX_RECEIVED_BYTES) {} + + markIncomplete(reason: keyof ReceivedTransport['incompleteCounts']): void { + this.incomplete = true; + this.incompleteCounts[reason]++; + } + + record(payload: string | Buffer): void { + const text = transportFrameText(payload); + // Receipt checks need presence, not frequency. Reloading an identical bundle + // adds no evidence and must not evict distinct data. + if (!text || this.chunks.includes(text)) return; + if (Buffer.byteLength(text) > this.limit) { + this.markIncomplete('byteLimit'); + return; + } + this.chunks.push(text); + this.bytes += Buffer.byteLength(text); + while (this.bytes > this.limit) { + this.markIncomplete('byteLimit'); + this.bytes -= Buffer.byteLength(this.chunks.shift()!); + } + } + + contains(needle: string, requireComplete = true): boolean { + if (this.chunks.some(chunk => chunk.includes(needle))) return true; + if (requireComplete && (this.incomplete || this.pending)) inconclusive('transport-incomplete', { + capture: { ...this.incompleteCounts, pendingBodies: this.pending, retainedBytes: this.bytes }, + }); + return false; + } +} + +export async function captureResponses(page: Page, received: ReceivedTransport): Promise { + page.on('response', async response => { + const type = response.headers()['content-type'] ?? ''; + // Native EventSource messages are captured below without waiting for stream closure. + if (/text\/event-stream/.test(type)) return; + // Public scripts and styles can contain secrets too. Keep the same bounded, + // fail-closed capture for data, rendered pages, assets and error responses. + if (!/(application\/(json|[^;]+\+json|x-ndjson|(?:x-)?(?:java|ecma)script)|text\/(plain|html|css|(?:java|ecma)script))/i.test(type)) return; + if (Number(response.headers()['content-length']) > MAX_RECEIVED_BYTES) { + received.markIncomplete('byteLimit'); + return; + } + received.pending++; + try { received.record(await response.text()); } + catch { received.markIncomplete('bodyReadFailures'); } + finally { received.pending--; } + }); + const session = await page.context().newCDPSession(page); + session.on('Network.eventSourceMessageReceived', event => received.record(event.data)); + session.on('Network.responseReceived', event => { + // Fetch streams have no EventSource events. Absence of an unseen body is not a pass. + if (event.response.mimeType === 'text/event-stream' && event.type !== 'EventSource') { + received.markIncomplete('unsupportedStreams'); + } + }); + await session.send('Network.enable'); + // Keep response bodies available when the app reloads immediately after reading them. + await session.send('Network.configureDurableMessages', { + maxTotalBufferSize: MAX_RECEIVED_BYTES, maxResourceBufferSize: MAX_RECEIVED_BYTES, + }); +} diff --git a/tools/stack-bench/linter/fixtures/agreed.html b/tools/stack-bench/linter/fixtures/agreed.html new file mode 100644 index 00000000000..c462174caf2 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/agreed.html @@ -0,0 +1,4 @@ +agreement fixture + +4 + diff --git a/tools/stack-bench/linter/fixtures/divergent.html b/tools/stack-bench/linter/fixtures/divergent.html new file mode 100644 index 00000000000..5d95d68cd79 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/divergent.html @@ -0,0 +1,12 @@ +divergence fixture + +
+ + + diff --git a/tools/stack-bench/linter/fixtures/mock-chat.html b/tools/stack-bench/linter/fixtures/mock-chat.html new file mode 100644 index 00000000000..2677b6feb48 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/mock-chat.html @@ -0,0 +1,71 @@ + + +Linter fixture — mock chat + + +
+ + +
+ + + + + + diff --git a/tools/stack-bench/linter/fixtures/mock-shop.html b/tools/stack-bench/linter/fixtures/mock-shop.html new file mode 100644 index 00000000000..5034271fdc9 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/mock-shop.html @@ -0,0 +1,297 @@ + +

Mock Shop

+ +
+ + + + + +
+ + + + + +
+ + + + + + + + + + + + + + diff --git a/tools/stack-bench/linter/fixtures/spec-accounts.html b/tools/stack-bench/linter/fixtures/spec-accounts.html new file mode 100644 index 00000000000..ddbb1ad5630 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/spec-accounts.html @@ -0,0 +1,29 @@ + +
+ + + + + +
+ + diff --git a/tools/stack-bench/linter/lint.ts b/tools/stack-bench/linter/lint.ts new file mode 100644 index 00000000000..9d04729e405 --- /dev/null +++ b/tools/stack-bench/linter/lint.ts @@ -0,0 +1,259 @@ +#!/usr/bin/env node +// Scenario-stage hooks require scenario setup and are not linted here. + +import { chromium } from 'playwright'; +import { attemptBrowserLaunchOptions } from '../container/browser-pipe.js'; +import type { Page } from 'playwright'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { loadTrack, DEFAULT_TRACK } from '../src/composition/tracks.js'; +import { emptyArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { harnessBrowserFailure } from '../src/evidence/harness-errors.js'; + +const CHECK_TIMEOUT = 5000; + +export interface LintHook { + id: string; + element: string; + stage: string; + check: 'visible' | 'attached'; + note: string; + revealedBy?: string; +} + +export interface LintResult { + id: string; + status: 'PASS' | 'FAIL' | 'BLOCKED' | 'SCENARIO' | 'HARNESS' | 'UNMEASURED'; + detail?: string; +} + +export interface LintArgs { + url?: string; + track: string; + level: number; + json: boolean; + headed: boolean; + out?: string; + label?: string; + parentAttemptId?: string; + credentialAliases?: unknown; + hooks?: string[]; +} + +export interface LintWalkContext { + page: Page; + args: LintArgs; + hooks: LintHook[]; + byStage(stage: string): LintHook[]; + blocked(stage: string): void; + checkHook(page: Page, hook: LintHook, results: LintResult[]): Promise; + results: LintResult[]; + uniq: string; + tid(id: string): string; + CHECK_TIMEOUT: number; +} + +function parseArgs(argv: string[]): LintArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + url: { type: 'string' }, track: { type: 'string' }, level: { type: 'string' }, + json: { type: 'boolean' }, out: { type: 'string' }, label: { type: 'string' }, + 'parent-attempt-id': { type: 'string' }, 'credential-aliases-json': { type: 'string' }, + hook: { type: 'string', multiple: true }, 'selected-hooks': { type: 'boolean' }, + headed: { type: 'boolean' }, + } }); + const args: LintArgs = { url: values.url, track: values.track ?? DEFAULT_TRACK, + level: values.level === undefined ? 1 : Number(values.level), json: values.json ?? false, + headed: values.headed ?? false, out: values.out, label: values.label, + parentAttemptId: values['parent-attempt-id'], + credentialAliases: values['credential-aliases-json'] === undefined + ? undefined : JSON.parse(values['credential-aliases-json']), + hooks: values.hook ?? (values['selected-hooks'] ? [] : undefined) }; + if (!args.url || !Number.isInteger(args.level) || args.level < 1) { + console.error('Usage: node dist/linter/lint.js --url --level [--json] [--headed]'); + process.exit(2); + } + return args; +} + +export function selectHooks(hooks: LintHook[], selectedIds?: string[]): LintHook[] { + if (selectedIds === undefined) return hooks; + const remaining = new Set(selectedIds); + const selected = hooks.filter(hook => remaining.delete(hook.id)); + const unknown: LintHook[] = [...remaining].sort().map(id => ({ + id, + element: `the selected application control ${id}`, + stage: 'scenario', + check: 'visible', + note: 'checked by the selected feature suite', + })); + return [...selected, ...unknown]; +} + +export function loadHooks(level: number, track: { contracts: string }, selectedIds?: string[]): LintHook[] { + const CONTRACTS_DIR = track.contracts; + const files = readdirSync(CONTRACTS_DIR).filter(f => /^\d+-[a-z-]+\.json$/.test(f)).sort(); + const hooks = []; + for (const f of files) { + const contract = JSON.parse(readFileSync(join(CONTRACTS_DIR, f), 'utf8')) as { + level: number; hooks: LintHook[]; + }; + if (contract.level <= level) hooks.push(...contract.hooks); + } + if (hooks.length === 0 && selectedIds === undefined) { + console.error(`No contracts found for level ${level} in ${CONTRACTS_DIR}`); + process.exit(2); + } + return selectHooks(hooks, selectedIds); +} + +const tid = stableElementSelector; +const uniq = Date.now().toString(36).slice(-5); + +// Browser, protocol, selector, and walk-script faults measure nothing about the app; +// a navigation timeout is unmeasured (grader/README.md outcome rules). +const errorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error ?? 'unknown error'); + +function faultStatus(error: unknown): 'FAIL' | 'HARNESS' | 'UNMEASURED' { + const message = errorMessage(error); + if (/^page\.goto: Timeout/.test(message)) return 'UNMEASURED'; + return harnessBrowserFailure(error) || error instanceof TypeError || error instanceof ReferenceError + || /Protocol error|while parsing .*selector|is not a valid selector/.test(message) ? 'HARNESS' : 'FAIL'; +} + +export async function checkHook(page: Page, hook: LintHook, results: LintResult[]): Promise { + const loc = page.locator(tid(hook.id)).first(); + try { + if (hook.revealedBy && !(await loc.count())) { + await page.locator(tid(hook.revealedBy)).first().click({ timeout: CHECK_TIMEOUT }); + } + await loc.waitFor({ + state: hook.check === 'visible' ? 'visible' : 'attached', + timeout: CHECK_TIMEOUT, + }); + results.push({ id: hook.id, status: 'PASS' }); + return true; + } catch (error) { + const status = faultStatus(error); + results.push({ + id: hook.id, + status, + detail: status !== 'FAIL' ? errorMessage(error).split(/\r?\n/, 1)[0] + : `no element matching ${tid(hook.id)} became ${hook.check} during contract stage ${JSON.stringify(hook.stage)}` + + (hook.revealedBy ? ` (after clicking ${tid(hook.revealedBy)})` : '') + + ` — expected: ${hook.element}`, + }); + return false; + } +} + +export function completeUnvisitedHooks(hooks: LintHook[], results: LintResult[]): LintResult[] { + const visited = new Set(results.map(result => result.id)); + for (const hook of hooks) { + if (visited.has(hook.id)) continue; + results.push(hook.stage === 'scenario' + ? { id: hook.id, status: 'SCENARIO', detail: hook.note } + : { id: hook.id, status: 'BLOCKED', + detail: `the core flow did not visit contract stage ${JSON.stringify(hook.stage)}` }); + } + return results; +} + +export function completeAbortedHooks(hooks: LintHook[], results: LintResult[], error: unknown): LintResult[] { + const visited = new Set(results.map(result => result.id)); + const detail = errorMessage(error) + .split(/\r?\n/).map(line => line.trim()).filter(Boolean).slice(0, 6).join(' ').slice(0, 800); + results.push({ id: 'core-flow', status: faultStatus(error), detail: `core flow aborted: ${detail}` }); + for (const hook of hooks) { + if (visited.has(hook.id)) continue; + if (hook.stage === 'scenario') { + results.push({ id: hook.id, status: 'SCENARIO', detail: hook.note }); + } else { + results.push({ id: hook.id, status: 'BLOCKED', detail: 'core flow aborted' }); + } + } + return results; +} + +async function run() { + const args = parseArgs(process.argv); + const track = loadTrack(args.track); + const hooks = loadHooks(args.level, track, args.hooks); + const byStage = (stage: string): LintHook[] => hooks.filter(h => h.stage === stage); + const results: LintResult[] = []; + const blocked = (stage: string): void => { + for (const h of hooks.filter(x => x.stage === stage)) { + results.push({ id: h.id, status: 'BLOCKED', detail: 'earlier core flow step failed' }); + } + }; + + if (hooks.length) { + const browser = await chromium.launch({ headless: !args.headed, ...attemptBrowserLaunchOptions() }); + const page = await browser.newContext().then(c => c.newPage()); + page.setDefaultTimeout(CHECK_TIMEOUT); + + try { + // The core flow is the one part of linting that is entirely + // application-specific, so each track brings its own. + const { walk } = await import(pathToFileURL(track.walk).href) as { + walk(context: LintWalkContext): Promise; + }; + await walk({ page, args, hooks, byStage, blocked, checkHook, results, uniq, tid, CHECK_TIMEOUT }); + // Every lintable hook must record explicit evidence. + completeUnvisitedHooks(hooks, results); + } catch (err: unknown) { + console.error(`Core flow aborted: ${err instanceof Error ? err.message : String(err)}`); + completeAbortedHooks(hooks, results, err); + } finally { + await browser.close(); + } + } + + const failures = results.filter(r => r.status !== 'PASS' && r.status !== 'SCENARIO'); + const report = { + label: args.label ?? null, + url: args.url, + level: args.level, + selectedHooks: args.hooks === undefined ? null : [...new Set(args.hooks)].sort(), + pass: failures.length === 0, + harness: results.some(r => r.status === 'HARNESS'), + unmeasured: results.some(r => r.status === 'UNMEASURED'), + counts: { + lintable: results.filter(r => r.status !== 'SCENARIO').length, + pass: results.filter(r => r.status === 'PASS').length, + fail: results.filter(r => r.status === 'FAIL').length, + blocked: results.filter(r => r.status === 'BLOCKED').length, + scenario: results.filter(r => r.status === 'SCENARIO').length, + }, + results, + }; + if (args.out) { + const id = `${args.parentAttemptId ?? args.label ?? 'lint'}-contract-lint`; + writeArtifact(args.out, { + kind: 'contract_lint', id, + attempt: { id, parentId: args.parentAttemptId ?? null }, + identities: emptyArtifactIdentities(), + payload: report, + }); + if (!args.json) console.log(`\nLint report written to ${args.out}`); + } + if (args.json) { + console.log(JSON.stringify(report, null, 2)); + } else { + for (const r of results) { + console.log(`${r.status.padEnd(9)} ${r.id}${r.detail ? ` — ${r.detail}` : ''}`); + } + console.log(failures.length === 0 + ? report.counts.pass > 0 + ? `\nAPPLICATION CONTRACT PASS (${report.counts.pass} interfaces)` + : report.counts.scenario > 0 + ? `\nAPPLICATION CONTRACT DEFERRED (${report.counts.scenario} interfaces checked during feature grading)` + : '\nNO STANDALONE INTERFACES SELECTED' + : `\nAPPLICATION CONTRACT FAIL (${failures.length} interfaces missing or blocked)`); + } + process.exit(failures.length === 0 ? 0 : 1); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) run(); diff --git a/tools/stack-bench/package-lock.json b/tools/stack-bench/package-lock.json new file mode 100644 index 00000000000..430532137f5 --- /dev/null +++ b/tools/stack-bench/package-lock.json @@ -0,0 +1,136 @@ +{ + "name": "@spacetimedb/stack-bench", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@spacetimedb/stack-bench", + "dependencies": { + "eventsource-parser": "3.1.1", + "playwright": "1.62.1", + "semver": "7.7.4", + "zod": "4.5.4" + }, + "devDependencies": { + "@types/node": "22.15.30", + "@types/semver": "7.8.0", + "typescript": "5.6.3" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@types/node": { + "version": "22.15.30", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.30.tgz", + "integrity": "sha512-6Q7lr06bEHdlfplU6YRbgG1SFBdlsfNC4/lX+SkhiTs0cpJkOElmWls8PxDFv4yY/xKb8Y6SO0OmSX4wgqTZbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-1mAINjtQCXXeLkJ9ehXkwOcBpqtLxiVtKhpUf83DdRNdQKV0iXZpaHYqRr7nj+wvxuJzoAmAwXI+sCNMv1CzLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.5.4.tgz", + "integrity": "sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/tools/stack-bench/package.json b/tools/stack-bench/package.json new file mode 100644 index 00000000000..3030d3f5c7f --- /dev/null +++ b/tools/stack-bench/package.json @@ -0,0 +1,85 @@ +{ + "name": "@spacetimedb/stack-bench", + "private": true, + "type": "module", + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "npm run clean && tsc -p tsconfig.build.json && node dist/scripts/copy-dashboard-assets.js", + "clean": "node --eval \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"", + "lint": "eslint appliance commands container dashboard grader linter scripts src tests", + "typecheck": "tsc -p tsconfig.json --noEmit", + "bootstrap:browsers": "playwright install chromium", + "prebench": "npm run build --silent", + "bench": "node dist/commands/bench.js", + "prepreflight": "npm run build", + "preflight": "node dist/commands/preflight.js", + "prerecover": "npm run build", + "recover": "node dist/commands/recovery.js recover", + "prerelease:bundle": "npm run build --silent", + "release:bundle": "node dist/src/releases/release-bundle.js", + "release:source": "npm run build --silent && node dist/src/releases/release-source.js --json", + "preverify:release": "npm run build --silent", + "verify:release": "node dist/src/releases/release-manifest.js verify", + "precheck:scenarios": "npm run build --silent", + "check:scenarios": "node dist/commands/check-scenarios.js --track chat && node dist/commands/check-scenarios.js --track ecommerce --recipe sequential-l1.json && node dist/commands/check-scenarios.js --track ecommerce --recipe sequential-l2.json && node dist/commands/check-scenarios.js --track ecommerce --recipe sequential-l3.json && node dist/commands/check-scenarios.js --track ecommerce --recipe progression-catalog.json", + "precheck:mutations": "npm run build", + "check:mutations": "node dist/commands/check-mutations.js", + "precheck:definition-snapshots": "npm run build --silent", + "check:definition-snapshots": "node dist/commands/definition-snapshots.js", + "precheck:composition": "npm run build --silent", + "check:composition": "node dist/commands/check-composition.js", + "precheck:prompts": "npm run build --silent", + "check:prompts": "node --test dist/tests/dependency-neutral-prompt.contract.js", + "precheck:calibration": "npm run build --silent", + "check:calibration": "node dist/commands/check-calibration.js", + "precheck:references": "npm run build --silent", + "check:references": "node dist/src/references/reference-fixtures.js", + "pregraph": "npm run build", + "graph": "node dist/commands/progression-graph.js tracks/ecommerce/progression/ecommerce.json", + "pack": "npm run build --silent && node dist/commands/composition-cli.js pack", + "recipe": "npm run build --silent && node dist/commands/composition-cli.js recipe", + "precampaign": "npm run build --silent", + "campaign": "node dist/commands/campaign-cli.js", + "predashboard": "npm run build --silent", + "dashboard": "node dist/dashboard/dashboard-server.js", + "prerepair": "npm run build", + "repair": "node dist/commands/repair-cli.js", + "pretest": "npm run build --silent", + "test": "node --test --test-concurrency=4 dist/tests/*.test.js", + "pretest:dashboard": "npm run build --silent", + "test:dashboard": "node --test dist/tests/dashboard/*.test.js", + "pretest:all": "npm run build --silent", + "test:all": "node --test --test-concurrency=4 dist/tests/*.test.js dist/tests/dashboard/*.test.js dist/tests/*.contract.js", + "pretest:contracts": "npm run build --silent", + "test:contracts": "node --test --test-concurrency=4 dist/tests/*.contract.js", + "pretest:mutation-definitions": "npm run build --silent", + "test:mutation-definitions": "node --test --test-concurrency=4 dist/tests/*.mutation.js", + "pretest:integration": "npm run build --silent", + "test:integration": "node --test --test-concurrency=1 dist/tests/*.integration.js", + "pretest:container": "npm run build --silent", + "test:container": "node dist/commands/container-smoke.js", + "pretest:references": "npm run build --silent", + "test:references": "node dist/src/references/reference-build.js", + "prequalify:reference": "npm run build --silent", + "qualify:reference": "node dist/src/references/reference-live.js", + "pretest:faults": "npm run build --silent", + "test:faults": "node dist/commands/fault-injection.js", + "pretest:loop": "npm run build", + "test:loop": "node dist/commands/test-loop.js", + "pretest:null": "npm run build --silent", + "test:null": "node dist/commands/null-control.js" + }, + "dependencies": { + "eventsource-parser": "3.1.1", + "playwright": "1.62.1", + "semver": "7.7.4", + "zod": "4.5.4" + }, + "devDependencies": { + "@types/node": "22.15.30", + "@types/semver": "7.8.0", + "typescript": "5.6.3" + } +} diff --git a/tools/stack-bench/reference-apps/README.md b/tools/stack-bench/reference-apps/README.md new file mode 100644 index 00000000000..47d58ddf62f --- /dev/null +++ b/tools/stack-bench/reference-apps/README.md @@ -0,0 +1,159 @@ +# Reference applications + +Reference applications validate the grader. They are simple, auditable fixtures, +not product examples or recommended application designs. + +`registry.json` is the source of truth for fixture identity, source, and supported +recipes. Qualification evidence proves whether an exact source is usable. + +One cumulative source tree can serve several recipes when each registry entry +binds the same source hash. Qualification evidence remains separate for each +recipe and calibration. + +## Qualification requirements + +A fixture must satisfy all of these conditions before it qualifies a run: + +1. Dependencies install from committed lockfiles in the benchmark build image. +2. The app starts in Docker with run-specific ports and database or module names. +3. Every required scored and supporting check passes for the exact recipe. +4. The source contains no secrets, generated bindings, build output, + transcripts, grader output, or mutation backups. +5. Each mutation has an exact source anchor and produces the intended conclusive + failure at an observation step (`expect`, `dbExpect`, or `waitUntilAbsent`), + without unrelated failures. Failing at a click, sign-in, or restart is + `CAUGHT_OFF_ASSERTION`, not a kill. +6. The registry records the qualified source hash. + +Compile success or an old full score does not promote a fixture. + +## Compile fixtures + +Run the model-free Docker compile check from `tools/stack-bench`: + +```bash +npm run test:references +``` + +Compile one changed fixture with: + +```bash +npm run test:references -- --fixture +``` + +The command copies source into a temporary workspace. It does not edit the +registered fixture. Compile success is not live grading evidence. +Local `node_modules` and `dist` directories can remain in the checkout. Reference +inspection excludes them from source hashes, and preparation does not copy them. +Dependencies and build output are recreated in the temporary workspace. + +## Live qualification + +Use a configured Linux appliance controller with immutable image identities. +Read the selected calibration before launch. Set the recipe, depth, and repetition +count explicitly; the command defaults are not the calibration policy. For example, +the dependency L3 reference command has this shape: + +```bash +node dist/src/references/reference-live.js --backend \ + --track ecommerce --level 3 --recipe ecommerce.progression-catalog \ + --feature-catalog progression/ecommerce.json \ + --repetitions --out +``` + +The qualifier binds the exact recipe, fixture, source, engine, image, stack, +runner, and check identities. It also verifies lease and resource cleanup. + +`referenceRepetitions` and `mutationRepetitions` come from the selected +[calibration](../tracks/ecommerce/composition/calibrations/). Registered evidence +must match these counts exactly. Extra repeated runs are useful stability +diagnostics, but cannot be substituted for an artifact with a different declared +repetition count. Two clean passes do not prove the absence of intermittent failures. + +For mutation evidence, combine `--mutations` with either `--mutation-id ` +or `--full-mutations`. The qualifier first +checks the clean baseline, then applies each selected defect through the same +isolated Docker lifecycle. + +During development, select only affected defects with `--mutation-id `. +Targeted output is diagnostic evidence. Use `--full-mutations` only when the +complete defect set is required. + +Targeted evidence can qualify a defined slice of an unchanged check population. +Each calibration evidence entry then declares `slice.checks` and a hash-pinned +`slice.snapshot` path. Qualifiers save the recipe hash inputs, calibration and +mutation inputs beside their output as `.inputs.json`. Preserve these +files with the original artifacts. Reconstructed older inputs must reproduce +the identities in the original evidence; a list of unchanged check IDs is not proof. + +The compiler verifies scenario setup, shared inputs, pack budgets, references, +runner, repetition policy and applicable defect definitions. Every required +check must have exactly one reference, mutation and null coverage entry per +required stack/repetition. It rejects missing or overlapping coverage. A targeted +mutation gate may also supply its verified clean baseline for reference coverage. +It retains the artifact's original identities and diagnostic label. + +This reuse path supports independently reset dependency scenarios with the same +qualification policy and population. Sequential inherited-stage evidence is not +supported. Changed executable hashes require a reviewed `qualificationReuse` +decision with retained supporting evidence. An unchanged commit label alone is +not sufficient. If any required input or coverage is missing, keep the candidate +unqualified and run only the missing scope; do not substitute a successful summary. + +Adding a stack does not invalidate a receipt for an unchanged existing stack or +the stack-neutral empty app. The measured stack must remain in both policies; +check selection and repetition counts must stay unchanged. The added stack still +needs its own complete reference and mutation evidence. Executable changes still +require the equivalence review above. + +For full mutation qualification, use the same scope, add +`--mutations --full-mutations`, set `--repetitions` to `mutationRepetitions`, and +choose a new output path. The runner can emit a companion clean-reference artifact +when the baseline repetition count also matches `referenceRepetitions`. +`--mutation-workers` runs independent defect controls with separate leases; it +does not change the selected checks or their pass rules. +The qualification status command generates commands for up to eight workers by +default. Use `qualification status ... --mutation-workers <1-8>` to select fewer. + +The matching dependency L3 empty-app control is: + +```bash +node dist/commands/null-control.js --track ecommerce --level 3 \ + --recipe ecommerce.progression-catalog --out +``` + +Add repeated `--selected-check ` options to run only the affected +null controls. The keys must belong to the calibration's selected checks. + +A scored check must fail conclusively on the empty app. Zero awarded points alone +are insufficient if the result is a harness failure or inconclusive. Check the +artifact's failure reasons, not only its process exit code. Reference, mutation, +and null runs make no model calls, but still consume local compute. Obtain +authorization before starting these long-running gates. + +Inspect the exact selected definition with: + +```bash +node dist/commands/qualification-cli.js status --track ecommerce --level 3 \ + --recipe ecommerce.progression-catalog +``` + +Static target coverage, a build pass, and historical reports cannot replace +matching live evidence. Preserve failed artifacts and write corrections to new +paths. Never edit old results to match a changed source or calibration. + +Do not edit a registered reference during qualification. A changed source hash +requires new evidence. + +The draft purchase-session check (`101a`) also tests an altered session credential. +It proves a valid purchase first, checks refusal and unchanged stored orders and +stock, then proves valid access again. Its order reader requires neither carts +nor warehouses. This extends the existing check without adding points. +The probe supports one bearer token or one session cookie. With bearer and cookie +credentials together, a second valid purchase must succeed using the bearer alone +before its tampered value is sent without cookies. This control is included in +purchase accounting; the browser session is unchanged. A failed isolated control, +multiple cookie-only credentials, other mixed credentials and cookie/CSRF ambiguity +leave the check inconclusive. They do not earn security credit or prove an application defect. These limits must remain +visible in result populations. The probe does not establish expiry, revocation, +or complete authentication security. diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/index.html b/tools/stack-bench/reference-apps/ecommerce/convex/client/index.html new file mode 100644 index 00000000000..8b77d8835cc --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/index.html @@ -0,0 +1,12 @@ + + + + + + Storefront + + +
+ + + diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/package-lock.json b/tools/stack-bench/reference-apps/ecommerce/convex/client/package-lock.json new file mode 100644 index 00000000000..931bd8e6d14 --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/package-lock.json @@ -0,0 +1,1469 @@ +{ + "name": "client", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "client", + "version": "1.0.0", + "dependencies": { + "convex": "1.46.0", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^6.1.0", + "typescript": "^5.5.3", + "vite": "^8.2.2" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.0.tgz", + "integrity": "sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.0.tgz", + "integrity": "sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.0.tgz", + "integrity": "sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.0.tgz", + "integrity": "sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.0.tgz", + "integrity": "sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.0.tgz", + "integrity": "sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.0.tgz", + "integrity": "sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.0.tgz", + "integrity": "sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.0.tgz", + "integrity": "sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.0.tgz", + "integrity": "sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.0.tgz", + "integrity": "sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.0.tgz", + "integrity": "sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.0.tgz", + "integrity": "sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.0.tgz", + "integrity": "sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.0.tgz", + "integrity": "sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.0.tgz", + "integrity": "sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.0.tgz", + "integrity": "sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.0.tgz", + "integrity": "sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.0.tgz", + "integrity": "sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.0.tgz", + "integrity": "sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.0.tgz", + "integrity": "sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.0.tgz", + "integrity": "sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.0.tgz", + "integrity": "sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.0.tgz", + "integrity": "sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.0.tgz", + "integrity": "sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.0.tgz", + "integrity": "sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.150.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.150.0.tgz", + "integrity": "sha512-rDS5/31E9HfPl/CIzGrn0DOlvBbXFseQ5URJ9sYMfstbKLD/c6Gm9vmRzRGDdAXyOIL4zmO37lc9RIwYqVruZw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.9.tgz", + "integrity": "sha512-tNISae1QEf/vkb3xkRcjV5SEdzPE97We5IVaa2Z8jSszQPZ8U60B/YCYpw4QI7VidYsBtKavczXf+DyDs9WGxw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.9.tgz", + "integrity": "sha512-YC8YsI30o606GTZi0VyzYlsDKFP8W61i/QzayHDkLbNEz/IShqAmTa+hsJRj13xTHA0H+6fk4b2UmGn+Q/cMlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.9.tgz", + "integrity": "sha512-IwhlH3qK5urrY8hZiEgGkHKEFN901p/p2bjxCxJlr4GyNnF7wYpUvK+Y43uaRYuC4hpfjzbR3SJC3arX1jGvmw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.9.tgz", + "integrity": "sha512-XxpJfVzFh+jilRxIXUqcfYAYcunIc/XEzIizsOL1fcJee5Sf7H3mH8WlLmfHfluz5amqR88QQo9izKtmMlavAw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.9.tgz", + "integrity": "sha512-kSfvhmgeWyfkbT3p/1s5vSgboogoah2zkm9fX2zjg2hHxSV7T4KhMWRUUaRk4OXNqoD3QAUeRqLcs1aZOK4U1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.9.tgz", + "integrity": "sha512-1RVzG17pxqbTfYLC352JlLt6kKLG+6Hr30n8DlIJqsnV5luUDd2Qdx9Ayw1Cabfyb1K9k0jXEZ7evxkRoT+uiw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.9.tgz", + "integrity": "sha512-BXqPvZ2drqVD+/Z8UpKwcs4Mp7grM+eGFku4CAEKrEtcbAsUpzREphK1sogCRZGreVPiMkiiBtw0n3TPteuqvw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.9.tgz", + "integrity": "sha512-11vWvo8YDwLzukt27J3aYDWU+gg2P7J+ZOmiJ0hkF5BXZDW7pVya7r40MXDy6ya0i9KamoENSVKIugvJNgFXIA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.9.tgz", + "integrity": "sha512-a1tijMkdwsIARtc0F39ApURROkf3NwqinI6TOiSSWCTR7dT96dffNvMUtDHnq64wKNTIZOIlzKrFvvFUznJiyw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.9.tgz", + "integrity": "sha512-x6SQNdAvv4c3hWqTMaWuawzMX9myaCs/yEmlGsxJzkdClnHW7FbrjQuSiRDhuSYzEYoEMhsaJy9qHG/XNemJPQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.9.tgz", + "integrity": "sha512-9s0AZ8BFK5/n7B/TBoa2yJE3gI3KURrbXcPBlsAsvjU4VeJKgE90y1YtNxyEUIcHPQkg6/yfF3qihUrcM/Kf0Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.9.tgz", + "integrity": "sha512-P7VWAmV+WdJluH7ovnRGoiv2i8To7GAZ+kGzfGup635cyL7SyYl3lSUaA3Gp5THf0n/Co5EyEqb2zbqq+nMOHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.9.tgz", + "integrity": "sha512-1qixtsE4BK8h+yS3BfmZ09UhA7O/N4IACva6YBr7EBvCJraByTuRcgOTaiA62Tm0vey3UcKXLOaoGHtYmNGEVg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.9.tgz", + "integrity": "sha512-ok8IQjcEPs1AKZfuEUznVBrJw+gK4soq+bx8b1X2XoMqVClarc1q5JDmVtWXY1xfr6ZuHTAsPXHTgTrqKTZeww==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.9.tgz", + "integrity": "sha512-Ip2mXoU0hM0boq3Rf+ekuT653OROSo6aSYcPT1VHE4q52KvyxgFkQgrgb/IEsxOuvQ2fZZbs8khJAyCEPM24/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.31", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.31.tgz", + "integrity": "sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.1.1.tgz", + "integrity": "sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "oxc-transform-react": "^0.145.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "oxc-transform-react": { + "optional": true + } + } + }, + "node_modules/convex": { + "version": "1.46.0", + "resolved": "https://registry.npmjs.org/convex/-/convex-1.46.0.tgz", + "integrity": "sha512-dhBwDQTZDOhi5SUGAX17EWNehCdNq6/UFrSL0eLZuGfMkdsQbCVWVRZCzbZ5CF+/pgvoGxtApbEoTSipqL4JTw==", + "license": "Apache-2.0", + "dependencies": { + "esbuild": "0.27.0", + "prettier": "^3.0.0", + "ws": "8.21.0" + }, + "bin": { + "convex": "bin/main.js" + }, + "engines": { + "node": ">=20.0.0", + "npm": ">=7.0.0" + }, + "peerDependencies": { + "@auth0/auth0-react": "^2.0.1", + "@clerk/clerk-react": "^4.12.8 || ^5.0.0", + "@clerk/react": "^6.4.3", + "react": "^18.0.0 || ^19.0.0-0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@auth0/auth0-react": { + "optional": true + }, + "@clerk/clerk-react": { + "optional": true + }, + "@clerk/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/esbuild": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.0.tgz", + "integrity": "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.0", + "@esbuild/android-arm": "0.27.0", + "@esbuild/android-arm64": "0.27.0", + "@esbuild/android-x64": "0.27.0", + "@esbuild/darwin-arm64": "0.27.0", + "@esbuild/darwin-x64": "0.27.0", + "@esbuild/freebsd-arm64": "0.27.0", + "@esbuild/freebsd-x64": "0.27.0", + "@esbuild/linux-arm": "0.27.0", + "@esbuild/linux-arm64": "0.27.0", + "@esbuild/linux-ia32": "0.27.0", + "@esbuild/linux-loong64": "0.27.0", + "@esbuild/linux-mips64el": "0.27.0", + "@esbuild/linux-ppc64": "0.27.0", + "@esbuild/linux-riscv64": "0.27.0", + "@esbuild/linux-s390x": "0.27.0", + "@esbuild/linux-x64": "0.27.0", + "@esbuild/netbsd-arm64": "0.27.0", + "@esbuild/netbsd-x64": "0.27.0", + "@esbuild/openbsd-arm64": "0.27.0", + "@esbuild/openbsd-x64": "0.27.0", + "@esbuild/openharmony-arm64": "0.27.0", + "@esbuild/sunos-x64": "0.27.0", + "@esbuild/win32-arm64": "0.27.0", + "@esbuild/win32-ia32": "0.27.0", + "@esbuild/win32-x64": "0.27.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prettier": { + "version": "3.9.8", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.8.tgz", + "integrity": "sha512-WRFq3Wn3WId7LLROfMLdH7xaFr2jR62wU8nLO6rQUOLOxNZUviyJQs1M0iIhLexSFy+L+w0ch66wtoO2jRjG0A==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/rolldown": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.9.tgz", + "integrity": "sha512-hx/Pv0N1haXRb11qkfnK5MXB/iqr7i0yjWQqmO9uHqZpBgQSqzc8UsSnEpalsh+j1I8qQ2CkXAkJC8Br3dKSlg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.150.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.9", + "@rolldown/binding-android-arm64": "1.2.9", + "@rolldown/binding-darwin-arm64": "1.2.9", + "@rolldown/binding-darwin-x64": "1.2.9", + "@rolldown/binding-freebsd-x64": "1.2.9", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.9", + "@rolldown/binding-linux-arm64-gnu": "1.2.9", + "@rolldown/binding-linux-arm64-musl": "1.2.9", + "@rolldown/binding-linux-ppc64-gnu": "1.2.9", + "@rolldown/binding-linux-s390x-gnu": "1.2.9", + "@rolldown/binding-linux-x64-gnu": "1.2.9", + "@rolldown/binding-linux-x64-musl": "1.2.9", + "@rolldown/binding-openharmony-arm64": "1.2.9", + "@rolldown/binding-win32-arm64-msvc": "1.2.9", + "@rolldown/binding-win32-x64-msvc": "1.2.9" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/vite": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.0.tgz", + "integrity": "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.7", + "postcss": "^8.5.28", + "rolldown": "~1.2.6", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.7.1", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/package.json b/tools/stack-bench/reference-apps/ecommerce/convex/client/package.json new file mode 100644 index 00000000000..b2cb2fbc428 --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/package.json @@ -0,0 +1,23 @@ +{ + "name": "client", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1", + "convex": "1.46.0" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^6.1.0", + "typescript": "^5.5.3", + "vite": "^8.2.2" + } +} diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/src/App.tsx b/tools/stack-bench/reference-apps/ecommerce/convex/client/src/App.tsx new file mode 100644 index 00000000000..39ad76e47db --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/src/App.tsx @@ -0,0 +1,1235 @@ +import React, { useEffect, useMemo, useRef, useState, useCallback } from "react"; +import { mutate, authenticate, validateSession, subscribeState, signOut, TOKEN_KEY } from "./request"; +import { ProgressionPanel } from "./ProgressionPanel"; + +const CATALOG_PAGE_SIZE = 10; + +interface ItemT { + id: string; + name: string; + price: number; + description?: string; + category: string; + stock: number; + purchaseCount: number; + variants?: string[]; +} + +interface ReviewT { + id: string; + itemId: string; + userId: string; + username: string; + rating: number; + comment: string; + createdAt: string; +} + +interface ItemDetailT { + id: string; + name: string; + price: number; + description: string; + stock: number; + reviews: ReviewT[]; + average: number; +} + +interface CartLineT { + itemId: string; + name: string; + price: number; + stock: number; + quantity: number; +} + +interface CartT { + items: CartLineT[]; + total: number; + promotionCode?: string; + discount?: number; +} + +interface OrderLineT { + itemId: string; + name: string; + price: number; + quantity: number; + returned?: boolean; + warehouseNames?: string[]; +} + +interface OrderT { + id: string; + items: OrderLineT[]; + total: number; + status: "pending" | "shipped" | "delivered" | "cancelled" | "refunded"; + discount?: number; + refundTotal?: number; + createdAt: string; +} + +interface UserT { + id: string; + username: string; + isAdmin: boolean; + isStaff: boolean; + roles?: string[]; +} + +interface AdminLocationT { + id: string; + itemId: string; + itemName: string; + warehouseId: string; + warehouseName: string; + quantity: number; +} + +interface CategoryTotalT { + category: string; + units: number; + revenue: number; +} + +interface AdminOverviewT { + items: Array<{ id: string; name: string; price: number; stock: number; category: string }>; + warehouses: Array<{ id: string; name: string; total: number }>; + locations: AdminLocationT[]; + revenue: number; + categories: CategoryTotalT[]; + lowStock: Array<{ id: string; name: string; stock: number }>; + queueDepth: number; +} + +interface FulfilmentQueueT { + orders: OrderT[]; + depth: number; +} + +function useTransientError(): [string, (msg: string) => void] { + const [message, setMessage] = useState(""); + const timer = useRef | null>(null); + const show = useCallback((msg: string) => { + setMessage(msg); + if (timer.current) clearTimeout(timer.current); + timer.current = setTimeout(() => setMessage(""), 5000); + }, []); + return [message, show]; +} + + +export default function App() { + const [token, setToken] = useState(() => localStorage.getItem(TOKEN_KEY)); + const [currentUser, setCurrentUser] = useState(null); + const [initializing, setInitializing] = useState(true); + const [items, setItems] = useState([]); + const [searchQuery, setSearchQuery] = useState(""); + const [categoryFilter, setCategoryFilter] = useState(""); + const [minimumPrice, setMinimumPrice] = useState(""); + const [maximumPrice, setMaximumPrice] = useState(""); + const [inStockOnly, setInStockOnly] = useState(false); + const [searchPage, setSearchPage] = useState(0); + const [cart, setCart] = useState({ items: [], total: 0 }); + // Keep floating panels mutually exclusive so navigation remains reachable. + const [activeView, setActiveView] = useState<"cart" | "orders" | "admin" | "fulfilment" | null>(null); + const cartOpen = activeView === "cart"; + const ordersOpen = activeView === "orders"; + const [orders, setOrders] = useState([]); + const adminOpen = activeView === "admin"; + const [adminOverview, setAdminOverview] = useState(null); + const fulfilmentOpen = activeView === "fulfilment"; + const [fulfilmentQueue, setFulfilmentQueue] = useState({ orders: [], depth: 0 }); + const [recommended, setRecommended] = useState([]); + const [selectedItemId, setSelectedItemId] = useState(null); + const [itemDetails, setItemDetails] = useState([]); + const itemDetail = itemDetails.find(item => item.id === selectedItemId) || null; + + const [buyError, showBuyError] = useTransientError(); + const [orderError, showOrderError] = useTransientError(); + + + const saveSession = (tok: string, user: UserT) => { + setToken(tok); + setCurrentUser(user); + }; + + const clearSession = () => { + localStorage.removeItem(TOKEN_KEY); + setToken(null); + setCurrentUser(null); + setCart({ items: [], total: 0 }); + setOrders([]); + setAdminOverview(null); + setFulfilmentQueue({ orders: [], depth: 0 }); + setActiveView(null); + }; + + // Validate a restored session once. Native subscriptions own application data. + useEffect(() => { + if (!token) return; + validateSession().catch(clearSession); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + useEffect(() => subscribeState((state) => { + setInitializing(false); + setItems(state.items); + setCurrentUser(state.user); + setCart(state.cart); + setOrders(state.orders); + setAdminOverview(state.admin); + setFulfilmentQueue(state.fulfilment || { orders: [], depth: 0 }); + setRecommended(state.recommended); + setItemDetails(state.details); + }), [token]); + + // Escape closes whichever overlay is open. + useEffect(() => { + const onKeyDown = (e: KeyboardEvent) => { + if (e.key !== "Escape") return; + if (selectedItemId) setSelectedItemId(null); + else if (activeView) setActiveView(null); + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [selectedItemId, activeView]); + + const handleSignUp = async (username: string, password: string) => { + const data = await authenticate(username, password, "signUp"); + saveSession(data.token, data.user); + }; + + const handleSignIn = async (username: string, password: string) => { + const data = await authenticate(username, password, "signIn"); + saveSession(data.token, data.user); + }; + + const handleSignOut = async () => { + await signOut(); + clearSession(); + }; + + const handleBuyNow = async (itemId: string) => { + try { + await mutate("api:buy_now", { itemId }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleAddToCart = async (itemId: string) => { + try { + await mutate("api:add_to_cart", { itemId, quantity: 1 }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleQuantityChange = async (itemId: string, quantity: number) => { + try { + await mutate("api:update_cart_quantity", { itemId, quantity }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleCheckout = async () => { + try { + await mutate("api:checkout"); + } catch (err: any) { + showBuyError(err.message); + } + }; + + + const openOrders = () => setActiveView("orders"); + const openAdmin = () => setActiveView("admin"); + const openFulfilment = () => setActiveView("fulfilment"); + + const [reviewError, setReviewError] = useState(""); + const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => { + setReviewError(""); + try { + await mutate("api:submit_review", { itemId, rating, comment }); + } catch (err: any) { + setReviewError(err.message); + } + }; + + const handleRestock = async (itemId: string, warehouseId: string, quantity: number) => { + try { + await mutate("api:admin_restock", { itemId, warehouseId, quantity }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleTransfer = async (itemId: string, fromWarehouseId: string, toWarehouseId: string, quantity: number) => { + try { + await mutate("api:admin_transfer_stock", { itemId, fromWarehouseId, toWarehouseId, quantity }); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handlePriceChange = async (itemId: string, price: number) => { + try { + await mutate("api:admin_change_price", { itemId, price }); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handleShipOrder = async (orderId: string) => { + try { + await mutate("api:ship_order", { orderId }); + } catch (err: any) { + showOrderError(err.message); + throw err; + } + }; + + const handleCancelOrder = async (orderId: string) => { + try { + await mutate("api:cancel_order", { orderId }); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handleReturnItem = async (orderId: string, itemId: string) => { + try { + await mutate("api:return_order_item", { orderId, itemId }); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const filteredItems = useMemo(() => { + const q = searchQuery.trim().toLowerCase(); + const min = minimumPrice === "" ? -Infinity : Number(minimumPrice); + const max = maximumPrice === "" ? Infinity : Number(maximumPrice); + const filtering = Boolean(q || categoryFilter || minimumPrice || maximumPrice || inStockOnly); + return items.filter((it) => (!q || it.name.toLowerCase().includes(q)) + && (!categoryFilter || it.category === categoryFilter) + && it.price >= min && it.price <= max && (!inStockOnly || it.stock > 0)) + .sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || a.name.localeCompare(b.name)); + }, [items, searchQuery, categoryFilter, minimumPrice, maximumPrice, inStockOnly]); + const searchResults = filteredItems.slice(searchPage * CATALOG_PAGE_SIZE, + searchPage * CATALOG_PAGE_SIZE + CATALOG_PAGE_SIZE); + + const cartCount = cart.items.reduce((s, l) => s + l.quantity, 0); + const selectedItem = items.find((it) => it.id === selectedItemId) || null; + const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff; + + return ( +
+ {initializing && ( +
+
+
Connecting to Storefront...
+
+ )} + +
+

+ Storefront +

+ + { setSearchQuery(e.target.value); setSearchPage(0); }} + onKeyDown={(e) => { + if (e.key === "Escape") setSearchQuery(""); + }} + /> +
+
+ + {currentUser && ( + + )} + {currentUser?.isAdmin && ( + + )} + {(currentUser?.isStaff || currentUser?.isAdmin) && ( + + )} + {currentUser ? ( + <> + + {currentUser.username} + + + + ) : ( + + )} +
+
+ +
+
+ {buyError && ( +
+ {buyError} +
+ )} + +
+
+ + { setMinimumPrice(e.target.value); setSearchPage(0); }} /> + { setMaximumPrice(e.target.value); setSearchPage(0); }} /> + +
+

Catalog

+
+
+ {searchResults.map((item) => ( + setSelectedItemId(item.id)} + onBuy={() => handleBuyNow(item.id)} + onAddToCart={() => handleAddToCart(item.id)} + /> + ))} +
+
+
+ + +
+
+ +
+

Recommended for you

+
+ {recommended.length === 0 ? ( +
Nothing recommended yet
+ ) : ( + recommended.map((item, index) => ( +
+ {index + 1} + setSelectedItemId(item.id)} onBuy={() => handleBuyNow(item.id)} + onAddToCart={() => handleAddToCart(item.id)} /> + +
+ )) + )} +
+
+ + {!activeView && (currentUser?.isStaff || currentUser?.isAdmin) && + } +
+ + {selectedItem && ( + { + setSelectedItemId(null); + setReviewError(""); + }} + onBuy={() => handleBuyNow(selectedItem.id)} + onAddToCart={() => handleAddToCart(selectedItem.id)} + onSubmitReview={(rating, comment) => handleReviewSubmit(selectedItem.id, rating, comment)} + /> + )} +
+ +
setActiveView(null)} /> +
+
+

Cart

+ +
+
+ {cart.items.length === 0 ? ( +
+ Your cart is empty +
+ ) : ( + <> + {cart.items.map((line) => ( + handleQuantityChange(line.itemId, qty)} + onRemove={() => handleQuantityChange(line.itemId, 0)} + /> + ))} +
+ Total + ${cart.total.toFixed(2)} +
+ + + )} +
+
+ +
setActiveView(null)} /> +
+
+

Order history

+ +
+ {orderError && ( +
+ {orderError} +
+ )} +
+ {orders.length === 0 ? ( +
You haven't placed any orders yet
+ ) : ( + orders.map((order) => ( +
+
+ {new Date(order.createdAt).toLocaleString()} + {order.status !== "cancelled" && order.items.length > 0 && order.items.every(line => line.returned) ? "returned" : order.status} +
+
{order.items.map(l => {l.name} ×{l.quantity}{l.returned ? " (returned)" : ""} )}
+
+ ${order.total.toFixed(2)} +
+
{Number(order.discount || 0).toFixed(2)}
+
{Number(order.refundTotal || 0).toFixed(2)}
+
+ {order.status === "pending" && ( + + )} + {["shipped", "delivered"].includes(order.status) && + order.items + .filter((l) => !l.returned) + .map((l) => ( + + ))} +
+
+ )) + )} +
+
+ + {fulfilmentOpen && (currentUser?.isStaff || currentUser?.isAdmin) && ( + setActiveView(null)} + onShip={handleShipOrder} orderError={orderError}> + + + )} + + {adminOpen && currentUser?.isAdmin && ( + setActiveView(null)} + onRestock={handleRestock} + onTransfer={handleTransfer} + onPriceChange={handlePriceChange} + orderError={orderError} + > + + + )} +
+ ); +} + +function ItemCard({ + item, + isCustomer, + onOpen, + onBuy, + onAddToCart, + testId = "item-card", +}: { + item: ItemT; + isCustomer: boolean; + onOpen: () => void; + onBuy: () => void; + onAddToCart: () => void; + testId?: string | null; +}) { + const outOfStock = item.stock === 0; + const [submitState, setSubmitState] = useState('idle'); + const requestAlert = async (event: React.MouseEvent) => { + event.stopPropagation(); + setSubmitState('pending'); + try { + await mutate("progression:stockAlert", { itemId: item.id }); + setSubmitState('succeeded'); + } catch { setSubmitState('failed'); } + }; + return ( +
+
+ {item.name} +
+
+ + ${item.price.toFixed(2)} + +
+
+ 0 && item.stock <= 5 ? " low" : ""}`} data-role="item-stock"> + {item.stock} + + {outOfStock && ( + + Out of stock + + )} +
+ {isCustomer && ( +
e.stopPropagation()}> + + +
+ )} + {(item.variants || []).map(variant => + {variant})} + {isCustomer && outOfStock && } +
+ ); +} + +function ItemDetailPanel({ + item, + detail, + isCustomer, + reviewError, + onClose, + onBuy, + onAddToCart, + onSubmitReview, +}: { + item: ItemT; + detail: ItemDetailT | null; + isCustomer: boolean; + reviewError: string; + onClose: () => void; + onBuy: () => void; + onAddToCart: () => void; + onSubmitReview: (rating: number, comment: string) => void; +}) { + const [rating, setRating] = useState(5); + const [comment, setComment] = useState(""); + const outOfStock = item.stock === 0; + + const submit = (e: React.FormEvent) => { + e.preventDefault(); + onSubmitReview(rating, comment); + setComment(""); + }; + + return ( +
+
+

{item.name}

+ +
+
+ + ${item.price.toFixed(2)} + + 0 && item.stock <= 5 ? "item-stock low" : "item-stock"}> + {item.stock} in stock + +
+ {outOfStock && ( + + Out of stock + + )} +

{detail?.description || "Loading description..."}

+ + {isCustomer && ( +
+ + +
+ )} + +

+ Reviews — average {(detail?.average ?? 0).toFixed(1)} +

+ {!detail || detail.reviews.length === 0 ? ( +
No reviews yet
+ ) : ( + detail.reviews.map((r) => ( +
+
+ {r.username} + {"★".repeat(r.rating)} +
+
{r.comment}
+
+ )) + )} + + {isCustomer && ( +
+ + setComment(e.target.value)} + /> + +
+ )} + {reviewError && ( +
+ {reviewError} +
+ )} +
+ ); +} + +function CartLineRow({ + line, + onQuantityChange, + onRemove, +}: { + line: CartLineT; + onQuantityChange: (qty: number) => void; + onRemove: () => void; +}) { + const [value, setValue] = useState(String(line.quantity)); + + useEffect(() => { + setValue(String(line.quantity)); + }, [line.quantity]); + + const commit = () => { + const qty = Number(value); + if (Number.isInteger(qty) && qty >= 1 && qty !== line.quantity) { + onQuantityChange(qty); + } else { + setValue(String(line.quantity)); + } + }; + + return ( +
+ {line.name} + setValue(e.target.value)} + onBlur={commit} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + commit(); + } + }} + /> + ${(line.price * line.quantity).toFixed(2)} + +
+ ); +} + +function AuthBox({ + onSignUp, + onSignIn, +}: { + onSignUp: (username: string, password: string) => Promise; + onSignIn: (username: string, password: string) => Promise; +}) { + const [signUpUsername, setSignUpUsername] = useState(""); + const [signUpPassword, setSignUpPassword] = useState(""); + const [signUpError, setSignUpError] = useState(""); + + const [showSignIn, setShowSignIn] = useState(false); + const [signInUsername, setSignInUsername] = useState(""); + const [signInPassword, setSignInPassword] = useState(""); + const [signInError, setSignInError] = useState(""); + + const submitSignUp = async (e: React.FormEvent) => { + e.preventDefault(); + setSignUpError(""); + try { + await onSignUp(signUpUsername.trim(), signUpPassword); + } catch (err: any) { + setSignUpError(err.message); + } + }; + + const submitSignIn = async (e: React.FormEvent) => { + e.preventDefault(); + setSignInError(""); + try { + await onSignIn(signInUsername.trim(), signInPassword); + } catch (err: any) { + setSignInError(err.message); + } + }; + + return ( +
+
+ setSignUpUsername(e.target.value)} + /> + setSignUpPassword(e.target.value)} + /> + + {signUpError && ( +
+ {signUpError} +
+ )} +
+ + {showSignIn && ( +
+ setSignInUsername(e.target.value)} + /> + setSignInPassword(e.target.value)} + /> + + {signInError && ( +
+ {signInError} +
+ )} +
+ )} +
+ ); +} + +function AdminPanel({ + overview, + onClose, + onRestock, + onTransfer, + onPriceChange, + orderError, + children, +}: { + overview: AdminOverviewT | null; + onClose: () => void; + onRestock: (itemId: string, warehouseId: string, quantity: number) => void; + onTransfer: (itemId: string, fromWarehouseId: string, toWarehouseId: string, quantity: number) => void; + onPriceChange: (itemId: string, price: number) => void; + orderError: string; + children?: React.ReactNode; +}) { + const [restockValues, setRestockValues] = useState>({}); + const [globalRestock, setGlobalRestock] = useState({ item: "", warehouse: "", quantity: "" }); + const [priceValues, setPriceValues] = useState>({}); + const [transferValues, setTransferValues] = useState< + Record + >({}); + + if (!overview) { + return ( +
+
+

Admin

+ +
+
Loading admin data...
+
+ ); + } + + const warehouses = overview.warehouses; + + const transferFor = (itemId: string) => + transferValues[itemId] || { from: warehouses[0]?.id || "", to: warehouses[1]?.id || warehouses[0]?.id || "", qty: "" }; + + return ( +
+
+

Admin

+ +
+ + {orderError && ( +
+ {orderError} +
+ )} + +
+ Total revenue: ${overview.revenue.toFixed(2)} +
+ +
+ setGlobalRestock(value => ({ ...value, item: event.target.value }))} /> + setGlobalRestock(value => ({ ...value, warehouse: event.target.value }))} /> + setGlobalRestock(value => ({ ...value, quantity: event.target.value }))} /> + +
+ +
+
+

Items

+ {overview.items.map((it) => { + const transfer = transferFor(it.id); + return ( +
+ {it.name} + {it.stock} +
+ setPriceValues((prev) => ({ ...prev, [it.id]: e.target.value }))} + /> + +
+
+ + + + setTransferValues((prev) => ({ ...prev, [it.id]: { ...transferFor(it.id), qty: e.target.value } })) + } + /> + +
+
+ ); + })} +
+
+

Warehouses

+
+ {overview.warehouses.map((w) => ( + + {w.name} — {w.total} + + ))} +
+

Stock by warehouse

+ {overview.locations.map((loc) => ( +
+ + {loc.itemName} @ {loc.warehouseName} + + {loc.quantity} +
{ + e.preventDefault(); + const qty = Number(restockValues[loc.id]); + if (Number.isInteger(qty) && qty >= 1) { + onRestock(loc.itemId, loc.warehouseId, qty); + setRestockValues((prev) => ({ ...prev, [loc.id]: "" })); + } + }}> + setRestockValues((prev) => ({ ...prev, [loc.id]: e.target.value }))} + /> + +
+
+ ))} +
+
+ +
+
+

Low stock

+
+ {overview.lowStock.length === 0 ? ( +
Nothing is running low
+ ) : ( + overview.lowStock.map((it) => ( +
+ {it.name} + {it.stock} +
+ )) + )} +
+
+
+

Category totals

+ {overview.categories.map((c) => ( +
+ {c.category} + {c.units} + ${c.revenue.toFixed(2)} +
+ ))} +
+
+ {children} +
+ ); +} + +function FulfilmentPanel({ + queue, + onClose, + onShip, + orderError, + children, +}: { + queue: FulfilmentQueueT; + onClose: () => void; + onShip: (orderId: string) => Promise; + orderError: string; + children?: React.ReactNode; +}) { + const [submitState, setSubmitState] = useState('idle'); + const ship = async (orderId: string) => { + setSubmitState('pending'); + try { await onShip(orderId); setSubmitState('succeeded'); } + catch { setSubmitState('failed'); } + }; + return ( +
+
+

Fulfilment queue

+ +
+ + {orderError && ( +
+ {orderError} +
+ )} + +
+ Orders waiting: {queue.depth} +
+ + {queue.orders.length === 0 ? ( +
Nothing waiting to ship
+ ) : ( + queue.orders.map((order) => ( +
+
+ {new Date(order.createdAt).toLocaleString()} +
+
{order.items.map((l) => `${l.name} ×${l.quantity}`).join(", ")}
+
+ {order.items.map((l, idx) => ( + + {(l.warehouseNames || []).join(", ") || "Unknown"} + + ))} +
+ +
+ )) + )} + {children} +
+ ); +} diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/src/ProgressionPanel.tsx b/tools/stack-bench/reference-apps/ecommerce/convex/client/src/ProgressionPanel.tsx new file mode 100644 index 00000000000..4edfcc65b27 --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/src/ProgressionPanel.tsx @@ -0,0 +1,262 @@ +import { mutate, subscribeProgression } from "./request"; +import { useEffect, useState } from "react"; + +type User = { username: string; isAdmin: boolean; isStaff: boolean; roles?: string[] }; +type Item = { id: string; name: string }; +type Order = { id: string; items: Array<{ name: string }>; total: number }; + + +function nameFor(items: Item[], id: unknown) { + return items.find(item => item.id === String(id))?.name || String(id || "Unknown item"); +} + +export function ProgressionPanel({ token, user, items, orders, onSignIn, staffOnly = false }: { + token: string | null; + user: User | null; + items: Item[]; + orders: Order[]; + onSignIn: (username: string, password: string) => Promise; + staffOnly?: boolean; +}) { + const [state, setState] = useState({ tickets: [], promotions: [], notifications: [], + scheduledRestocks: [], ledger: [], staffUsers: [] }); + const [error, setError] = useState(""); + const [staffName, setStaffName] = useState(""); + const [staffPassword, setStaffPassword] = useState(""); + const [profileOpen, setProfileOpen] = useState(false); + const [supportOpen, setSupportOpen] = useState(false); + const [notificationsOpen, setNotificationsOpen] = useState(false); + const [profileName, setProfileName] = useState(""); + const [profileAddress, setProfileAddress] = useState(""); + const [supportEmail, setSupportEmail] = useState(""); + const [supportSubject, setSupportSubject] = useState(""); + const [supportMessage, setSupportMessage] = useState(""); + const [supportReference, setSupportReference] = useState(""); + const [preference, setPreference] = useState({ order: false, stock: false }); + + useEffect(() => { + setPreference({ order: !!state.preference?.order, stock: !!state.preference?.stock }); + }, [user?.username, state.preference?.order, state.preference?.stock]); + + useEffect(() => { + setProfileName(state.profile?.name || ""); + setProfileAddress(state.profile?.address || ""); + }, [user?.username, state.profile?.name, state.profile?.address]); + + useEffect(() => subscribeProgression(next => { + setState({ ...next, loadedToken: token }); + }), [token]); + + + const act = async (name: string, args: Record = {}) => { + setError(""); + try { + return await mutate(name, args); + } catch (err: any) { + setError(err.message); + return null; + } + }; + + if (staffOnly) { + if (!(user?.isStaff || user?.isAdmin)) return null; + return ; + } + + const submitSupport = async () => { + const result = await act("progression:submitSupport", { + email: supportEmail, subject: supportSubject, message: supportMessage, + }); + if (result) setSupportReference(result.ticket.reference); + }; + + const saveProfile = () => act("progression:saveProfile", { name: profileName, address: profileAddress }); + + return
+ {!user ?
+

Staff sign in

+ setStaffName(event.target.value)} placeholder="Username" /> + setStaffPassword(event.target.value)} placeholder="Password" /> + +
: {user.username}} + + + + {profileOpen && user &&
+

Profile

+ setProfileName(event.target.value)} placeholder="Name" /> + setProfileAddress(event.target.value)} placeholder="Address" /> + +

{state.profile?.address || ""}

+
} + + {supportOpen &&
+

Support

+ setSupportEmail(event.target.value)} placeholder="Email" /> + setSupportSubject(event.target.value)} placeholder="Subject" /> +