From 0e2050f448e741695ac56ed4de45224f4a67ffd6 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 15:38:28 -0600 Subject: [PATCH 1/5] fix(bench): release 0.1.1 against runtime 0.90.1 --- .github/workflows/ci.yml | 25 +++++++ .github/workflows/publish.yml | 70 +++++++++++++++++++ bench/package.json | 9 ++- bench/pnpm-lock.yaml | 75 +++++++++++++++++++-- bench/scripts/verify-packed-consumer.mjs | 86 ++++++++++++++++++++++++ bench/tsconfig.public.json | 16 +++++ 6 files changed, 273 insertions(+), 8 deletions(-) create mode 100644 bench/scripts/verify-packed-consumer.mjs create mode 100644 bench/tsconfig.public.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c0dcc1f..c1b16531 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,3 +39,28 @@ jobs: - name: Docs (regenerate API reference + freshness gate) run: pnpm run docs:check + + agent-bench: + runs-on: ubuntu-latest + defaults: + run: + working-directory: bench + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: bench/pnpm-lock.yaml + + - name: Install published dependencies + run: pnpm install --frozen-lockfile + + - name: Typecheck public entrypoint against installed packages + run: pnpm run typecheck:public + + - name: Verify packed package in a clean consumer + run: pnpm run verify:package diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 29890325..ecd096ff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,10 +4,12 @@ on: push: tags: - 'v*' + - 'agent-bench-v*' workflow_dispatch: jobs: verify: + if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -90,3 +92,71 @@ jobs: else npm publish --provenance --access public fi + + verify-agent-bench: + if: startsWith(github.ref, 'refs/tags/agent-bench-v') + runs-on: ubuntu-latest + defaults: + run: + working-directory: bench + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: bench/pnpm-lock.yaml + + - name: Install deps + run: pnpm install --frozen-lockfile + + - name: Typecheck public entrypoint against installed packages + run: pnpm run typecheck:public + + - name: Verify packed package in a clean consumer + run: pnpm run verify:package + + - name: Verify tag/version lock + run: | + NPM_VERSION=$(node -p "require('./package.json').version") + TAG_VERSION="${GITHUB_REF#refs/tags/agent-bench-v}" + if [ "$TAG_VERSION" != "$NPM_VERSION" ]; then + echo "::error::Tag/version mismatch: tag=$TAG_VERSION package=$NPM_VERSION." + exit 1 + fi + echo "Version locked: $NPM_VERSION" + + publish-agent-bench: + needs: verify-agent-bench + if: startsWith(github.ref, 'refs/tags/agent-bench-v') + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + defaults: + run: + working-directory: bench + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + + - run: pnpm install --frozen-lockfile + + - name: Publish to npm (OIDC trusted publishing) + run: | + npm install -g npm@11 + NAME=$(node -p "require('./package.json').name") + VERSION=$(node -p "require('./package.json').version") + if npm view "$NAME@$VERSION" version >/dev/null 2>&1; then + echo "$NAME@$VERSION already on registry; skipping publish" + else + npm publish --provenance --access public + fi diff --git a/bench/package.json b/bench/package.json index 8d4aad86..5aff63ce 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.1.0", + "version": "0.1.1", "type": "module", "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.", "main": "src/index.ts", @@ -15,14 +15,17 @@ "gate-cli": "tsx src/gate-cli.mts", "run-benchmarks": "tsx src/run-benchmarks-cli.mts", "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl", - "terminal-compare": "tsx src/terminal-compare.ts" + "terminal-compare": "tsx src/terminal-compare.ts", + "typecheck:public": "tsc -p tsconfig.public.json", + "verify:package": "node scripts/verify-packed-consumer.mjs" }, "dependencies": { "@tangle-network/agent-eval": "^0.106.3", - "@tangle-network/agent-runtime": "^0.89.0", + "@tangle-network/agent-runtime": "^0.90.1", "@tangle-network/sandbox": "^0.9.7" }, "devDependencies": { + "@types/node": "^25.0.0", "tsx": "^4.19.0", "typescript": "^6.0.3" }, diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml index 186f3f70..7c703e47 100644 --- a/bench/pnpm-lock.yaml +++ b/bench/pnpm-lock.yaml @@ -12,12 +12,15 @@ importers: specifier: ^0.106.3 version: 0.106.3(typescript@6.0.3) '@tangle-network/agent-runtime': - specifier: ^0.89.0 - version: 0.89.0(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))) + specifier: ^0.90.1 + version: 0.90.1(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) '@tangle-network/sandbox': specifier: ^0.9.7 version: 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) devDependencies: + '@types/node': + specifier: ^25.0.0 + version: 25.9.5 tsx: specifier: ^4.19.0 version: 4.22.4 @@ -256,6 +259,11 @@ packages: engines: {node: '>=20'} hasBin: true + '@tangle-network/agent-eval@0.108.1': + resolution: {integrity: sha512-5T6XKtqcxB8OrR1c8V4EGZk+gQizxmVVQKLCkxTlnMh2lApV+e0EIbEUpKZzD9HmCQSgRY8a2tW3hCTFBuZAyg==} + engines: {node: '>=20'} + hasBin: true + '@tangle-network/agent-interface@0.10.1': resolution: {integrity: sha512-yehY/0EgKvu8lG6jIVoZCtMPLkj8VEWwasuAtuph2RaB9MKE5wuxRF647O6jw8KufNZ3aQ2UVVWpZ19dGCbs6w==} @@ -265,8 +273,13 @@ packages: '@tangle-network/agent-interface@0.14.0': resolution: {integrity: sha512-9CyGhIpl90E7v4MTm3b1ti3Bp7BfPigk2Nafgi21Lg0U+QxlNB656F2JmVpUuSbOo9aGZPtg5nXu5EBTlV5a1g==} - '@tangle-network/agent-runtime@0.89.0': - resolution: {integrity: sha512-tjQ/uJORuLOAE/E99w0NxWb5O0uP/nwescMRyIqxsqLXzry7WhNh9DK/BHKTEfb6XMvcIaT92J/3T1ZJTtXZ9Q==} + '@tangle-network/agent-knowledge@1.11.1': + resolution: {integrity: sha512-2vvNHHsb4TQFnY+SrL7rYaMhwTxTK8R/lD9IAteonmgwcE5amhpPQZqftw+UDMl1d6OBe4QX+Qv1pNmWQDXzqQ==} + engines: {node: '>=20'} + hasBin: true + + '@tangle-network/agent-runtime@0.90.1': + resolution: {integrity: sha512-doX0KtA5QAJOYifCAAd8EmjOoMWwlpsVfvXDRdaZJIakAzrznstSllK9NeaENpZBj19yb6XTN3jPKuZwbcl6rQ==} engines: {node: '>=20'} hasBin: true peerDependencies: @@ -311,6 +324,9 @@ packages: engines: {node: '>=18'} hasBin: true + '@types/node@25.9.5': + resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} + abitype@1.2.3: resolution: {integrity: sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==} peerDependencies: @@ -372,6 +388,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + viem@2.52.0: resolution: {integrity: sha512-py2QPYe9e1f4DmPJCsXF7zHmyZ0PkJrBxdQZ5dvNXvzy3UzWkUn7dNfC0TMeNm6Qv1tKw3b6qXXExpx6L0oMbw==} peerDependencies: @@ -563,6 +582,24 @@ snapshots: - typescript - utf-8-validate + '@tangle-network/agent-eval@0.108.1(typescript@6.0.3)': + dependencies: + '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) + '@ax-llm/ax': 19.0.45(zod@4.4.3) + '@hono/node-server': 2.0.4(hono@4.12.23) + '@tangle-network/agent-interface': 0.10.1 + '@tangle-network/tcloud': 0.4.14(typescript@6.0.3)(zod@4.4.3) + hono: 4.12.23 + zod: 4.4.3 + transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai + - bufferutil + - openai + - typescript + - utf-8-validate + '@tangle-network/agent-interface@0.10.1': dependencies: zod: 4.4.3 @@ -575,12 +612,34 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-runtime@0.89.0(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))': + '@tangle-network/agent-knowledge@1.11.1(typescript@6.0.3)': + dependencies: + '@tangle-network/agent-eval': 0.108.1(typescript@6.0.3) + zod: 4.4.3 + transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai + - bufferutil + - openai + - typescript + - utf-8-validate + + '@tangle-network/agent-runtime@0.90.1(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': dependencies: '@tangle-network/agent-eval': 0.106.3(typescript@6.0.3) + '@tangle-network/agent-knowledge': 1.11.1(typescript@6.0.3) optionalDependencies: '@tangle-network/agent-interface': 0.14.0 '@tangle-network/sandbox': 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) + transitivePeerDependencies: + - '@mastra/core' + - '@modelcontextprotocol/sdk' + - ai + - bufferutil + - openai + - typescript + - utf-8-validate '@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3))': dependencies: @@ -609,6 +668,10 @@ snapshots: - utf-8-validate - zod + '@types/node@25.9.5': + dependencies: + undici-types: 7.24.6 + abitype@1.2.3(typescript@6.0.3)(zod@4.4.3): optionalDependencies: typescript: 6.0.3 @@ -685,6 +748,8 @@ snapshots: typescript@6.0.3: {} + undici-types@7.24.6: {} + viem@2.52.0(typescript@6.0.3)(zod@4.4.3): dependencies: '@noble/curves': 1.9.1 diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs new file mode 100644 index 00000000..f4578703 --- /dev/null +++ b/bench/scripts/verify-packed-consumer.mjs @@ -0,0 +1,86 @@ +import { execFile } from 'node:child_process' +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const scratch = await mkdtemp(path.join(tmpdir(), 'agent-bench-consumer-')) + +async function run(command, args, cwd) { + try { + return await execFileAsync(command, args, { cwd, maxBuffer: 10 * 1024 * 1024 }) + } catch (error) { + if (error?.stdout) process.stderr.write(error.stdout) + if (error?.stderr) process.stderr.write(error.stderr) + throw error + } +} + +try { + const packDir = path.join(scratch, 'pack') + const consumerDir = path.join(scratch, 'consumer') + await mkdir(packDir) + await mkdir(consumerDir) + + const packed = await run('npm', ['pack', '--json', '--pack-destination', packDir], benchDir) + const [{ filename }] = JSON.parse(packed.stdout) + const tarball = path.join(packDir, filename) + const manifest = JSON.parse(await readFile(path.join(benchDir, 'package.json'), 'utf8')) + + await writeFile( + path.join(consumerDir, 'package.json'), + `${JSON.stringify( + { + name: 'agent-bench-package-verifier', + private: true, + type: 'module', + dependencies: { '@tangle-network/agent-bench': `file:${tarball}` }, + devDependencies: { + '@types/node': manifest.devDependencies['@types/node'], + typescript: manifest.devDependencies.typescript, + }, + }, + null, + 2, + )}\n`, + ) + await writeFile( + path.join(consumerDir, 'index.ts'), + "import { resolveAdapter, type BenchmarkAdapter } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nvoid adapter\n", + ) + await writeFile( + path.join(consumerDir, 'tsconfig.json'), + `${JSON.stringify( + { + compilerOptions: { + module: 'esnext', + target: 'es2023', + lib: ['es2023'], + moduleResolution: 'bundler', + allowImportingTsExtensions: true, + strict: true, + noEmit: true, + skipLibCheck: true, + types: ['node'], + }, + files: ['index.ts'], + }, + null, + 2, + )}\n`, + ) + + await run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--package-lock=false'], consumerDir) + await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir) + const runtimeManifest = JSON.parse( + await readFile(path.join(consumerDir, 'node_modules/@tangle-network/agent-runtime/package.json'), 'utf8'), + ) + console.log( + `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}`, + ) +} finally { + await rm(scratch, { recursive: true, force: true }) +} diff --git a/bench/tsconfig.public.json b/bench/tsconfig.public.json new file mode 100644 index 00000000..e688a3f6 --- /dev/null +++ b/bench/tsconfig.public.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "module": "esnext", + "target": "es2023", + "lib": ["es2023"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"], + "esModuleInterop": true, + "resolveJsonModule": true + }, + "files": ["src/index.ts"] +} From 58a6680b67df573ff7fd887b38c8c0a39d5f2a62 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 15:43:02 -0600 Subject: [PATCH 2/5] test(bench): probe runBenchmarks in packed consumer --- bench/scripts/verify-packed-consumer.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index f4578703..24db5b4d 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -49,7 +49,7 @@ try { ) await writeFile( path.join(consumerDir, 'index.ts'), - "import { resolveAdapter, type BenchmarkAdapter } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nvoid adapter\n", + "import { resolveAdapter, runBenchmarks, type BenchmarkAdapter } from '@tangle-network/agent-bench'\n\nconst adapter: BenchmarkAdapter = resolveAdapter('swe-bench')\nvoid adapter\nvoid runBenchmarks\n", ) await writeFile( path.join(consumerDir, 'tsconfig.json'), From 43cc5ec8872c149c368692ccfd405334c24086da Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 15:54:44 -0600 Subject: [PATCH 3/5] test(bench): run package tests before publish --- .github/workflows/ci.yml | 3 +++ .github/workflows/publish.yml | 5 ++++ bench/package.json | 3 ++- bench/pnpm-lock.yaml | 33 ++++-------------------- bench/scripts/verify-packed-consumer.mjs | 21 ++++++++++----- 5 files changed, 30 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1b16531..b90e790e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,5 +62,8 @@ jobs: - name: Typecheck public entrypoint against installed packages run: pnpm run typecheck:public + - name: Test deterministic package paths + run: pnpm test + - name: Verify packed package in a clean consumer run: pnpm run verify:package diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ecd096ff..868ca30e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -116,6 +116,9 @@ jobs: - name: Typecheck public entrypoint against installed packages run: pnpm run typecheck:public + - name: Test deterministic package paths + run: pnpm test + - name: Verify packed package in a clean consumer run: pnpm run verify:package @@ -147,6 +150,8 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + cache: pnpm + cache-dependency-path: bench/pnpm-lock.yaml - run: pnpm install --frozen-lockfile diff --git a/bench/package.json b/bench/package.json index 5aff63ce..3ed6a062 100644 --- a/bench/package.json +++ b/bench/package.json @@ -16,11 +16,12 @@ "run-benchmarks": "tsx src/run-benchmarks-cli.mts", "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl", "terminal-compare": "tsx src/terminal-compare.ts", + "test": "TSX_TSCONFIG_PATH=tsconfig.public.json node --test --import tsx src/benchmarks/appworld.test.mts src/benchmarks/external-adapters.test.mts src/benchmarks/rag-benchmarks.test.mts src/corpus.test.mts src/gate.test.mts src/refine-loop.test.mts src/run-benchmarks.test.mts src/selector.test.mts", "typecheck:public": "tsc -p tsconfig.public.json", "verify:package": "node scripts/verify-packed-consumer.mjs" }, "dependencies": { - "@tangle-network/agent-eval": "^0.106.3", + "@tangle-network/agent-eval": "^0.108.1", "@tangle-network/agent-runtime": "^0.90.1", "@tangle-network/sandbox": "^0.9.7" }, diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml index 7c703e47..78f1db81 100644 --- a/bench/pnpm-lock.yaml +++ b/bench/pnpm-lock.yaml @@ -9,11 +9,11 @@ importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.106.3 - version: 0.106.3(typescript@6.0.3) + specifier: ^0.108.1 + version: 0.108.1(typescript@6.0.3) '@tangle-network/agent-runtime': specifier: ^0.90.1 - version: 0.90.1(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) + version: 0.90.1(@tangle-network/agent-eval@0.108.1(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) '@tangle-network/sandbox': specifier: ^0.9.7 version: 0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) @@ -254,11 +254,6 @@ packages: '@tangle-network/agent-core@0.3.4': resolution: {integrity: sha512-Hvz3ABRouNtBmRvGqPxifAO2yuILneJMylWH5jW/jeS2F03RvqkGYuXyGXWWLqosYbb3hVAvSEe4Ykm2FMGEDQ==} - '@tangle-network/agent-eval@0.106.3': - resolution: {integrity: sha512-5mLpTNoR4YcAbHueCrYcdoBRo+I9zmjXipzWaYfJ2FGj4byA6L4wrTPdTneOLjvxOGlrq+H/CRmHB738fP8R/Q==} - engines: {node: '>=20'} - hasBin: true - '@tangle-network/agent-eval@0.108.1': resolution: {integrity: sha512-5T6XKtqcxB8OrR1c8V4EGZk+gQizxmVVQKLCkxTlnMh2lApV+e0EIbEUpKZzD9HmCQSgRY8a2tW3hCTFBuZAyg==} engines: {node: '>=20'} @@ -564,24 +559,6 @@ snapshots: '@tangle-network/agent-interface': 0.14.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.106.3(typescript@6.0.3)': - dependencies: - '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) - '@ax-llm/ax': 19.0.45(zod@4.4.3) - '@hono/node-server': 2.0.4(hono@4.12.23) - '@tangle-network/agent-interface': 0.10.1 - '@tangle-network/tcloud': 0.4.14(typescript@6.0.3)(zod@4.4.3) - hono: 4.12.23 - zod: 4.4.3 - transitivePeerDependencies: - - '@mastra/core' - - '@modelcontextprotocol/sdk' - - ai - - bufferutil - - openai - - typescript - - utf-8-validate - '@tangle-network/agent-eval@0.108.1(typescript@6.0.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) @@ -625,9 +602,9 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-runtime@0.90.1(@tangle-network/agent-eval@0.106.3(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': + '@tangle-network/agent-runtime@0.90.1(@tangle-network/agent-eval@0.108.1(typescript@6.0.3))(@tangle-network/agent-interface@0.14.0)(@tangle-network/sandbox@0.9.7(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': dependencies: - '@tangle-network/agent-eval': 0.106.3(typescript@6.0.3) + '@tangle-network/agent-eval': 0.108.1(typescript@6.0.3) '@tangle-network/agent-knowledge': 1.11.1(typescript@6.0.3) optionalDependencies: '@tangle-network/agent-interface': 0.14.0 diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index 24db5b4d..68207f7d 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -11,7 +11,7 @@ const scratch = await mkdtemp(path.join(tmpdir(), 'agent-bench-consumer-')) async function run(command, args, cwd) { try { - return await execFileAsync(command, args, { cwd, maxBuffer: 10 * 1024 * 1024 }) + return await execFileAsync(command, args, { cwd, maxBuffer: 10 * 1024 * 1024, timeout: 120_000 }) } catch (error) { if (error?.stdout) process.stderr.write(error.stdout) if (error?.stderr) process.stderr.write(error.stderr) @@ -29,6 +29,10 @@ try { const [{ filename }] = JSON.parse(packed.stdout) const tarball = path.join(packDir, filename) const manifest = JSON.parse(await readFile(path.join(benchDir, 'package.json'), 'utf8')) + const devDependencies = manifest.devDependencies + if (!devDependencies?.['@types/node'] || !devDependencies.typescript) { + throw new Error('package verification requires @types/node and typescript devDependencies') + } await writeFile( path.join(consumerDir, 'package.json'), @@ -39,8 +43,8 @@ try { type: 'module', dependencies: { '@tangle-network/agent-bench': `file:${tarball}` }, devDependencies: { - '@types/node': manifest.devDependencies['@types/node'], - typescript: manifest.devDependencies.typescript, + '@types/node': devDependencies['@types/node'], + typescript: devDependencies.typescript, }, }, null, @@ -75,9 +79,14 @@ try { await run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--package-lock=false'], consumerDir) await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir) - const runtimeManifest = JSON.parse( - await readFile(path.join(consumerDir, 'node_modules/@tangle-network/agent-runtime/package.json'), 'utf8'), - ) + let runtimeManifest + try { + runtimeManifest = JSON.parse( + await readFile(path.join(consumerDir, 'node_modules/@tangle-network/agent-runtime/package.json'), 'utf8'), + ) + } catch (error) { + throw new Error('packed consumer did not install @tangle-network/agent-runtime', { cause: error }) + } console.log( `packed consumer verified: ${manifest.name}@${manifest.version} with @tangle-network/agent-runtime@${runtimeManifest.version}`, ) From 53f5e8c4d8f512111d962ed2bbb0049110189b69 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 15:56:48 -0600 Subject: [PATCH 4/5] chore(bench): harden release verification --- .github/workflows/publish.yml | 4 ++++ bench/scripts/verify-packed-consumer.mjs | 27 ++++++++++++------------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 868ca30e..416bccf8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -94,6 +94,8 @@ jobs: fi verify-agent-bench: + # Manual dispatch remains root-only. Re-run a failed bench tag workflow so + # tag/version locking stays intact and the root package is never co-published. if: startsWith(github.ref, 'refs/tags/agent-bench-v') runs-on: ubuntu-latest defaults: @@ -157,6 +159,8 @@ jobs: - name: Publish to npm (OIDC trusted publishing) run: | + # Requires the npmjs Trusted Publisher on @tangle-network/agent-bench: + # org tangle-network, repo agent-runtime, workflow publish.yml. npm install -g npm@11 NAME=$(node -p "require('./package.json').name") VERSION=$(node -p "require('./package.json').version") diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index 68207f7d..0a8fc878 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -13,9 +13,11 @@ async function run(command, args, cwd) { try { return await execFileAsync(command, args, { cwd, maxBuffer: 10 * 1024 * 1024, timeout: 120_000 }) } catch (error) { - if (error?.stdout) process.stderr.write(error.stdout) + if (error?.stdout) process.stdout.write(error.stdout) if (error?.stderr) process.stderr.write(error.stderr) - throw error + const invocation = [command, ...args].join(' ') + const message = error instanceof Error ? error.message : String(error) + throw new Error(`${invocation} failed: ${message}`, { cause: error }) } } @@ -30,9 +32,16 @@ try { const tarball = path.join(packDir, filename) const manifest = JSON.parse(await readFile(path.join(benchDir, 'package.json'), 'utf8')) const devDependencies = manifest.devDependencies - if (!devDependencies?.['@types/node'] || !devDependencies.typescript) { + if ( + typeof devDependencies?.['@types/node'] !== 'string' || + typeof devDependencies.typescript !== 'string' || + !devDependencies['@types/node'] || + !devDependencies.typescript + ) { throw new Error('package verification requires @types/node and typescript devDependencies') } + const publicTsconfig = JSON.parse(await readFile(path.join(benchDir, 'tsconfig.public.json'), 'utf8')) + if (!publicTsconfig.compilerOptions) throw new Error('tsconfig.public.json must define compilerOptions') await writeFile( path.join(consumerDir, 'package.json'), @@ -59,17 +68,7 @@ try { path.join(consumerDir, 'tsconfig.json'), `${JSON.stringify( { - compilerOptions: { - module: 'esnext', - target: 'es2023', - lib: ['es2023'], - moduleResolution: 'bundler', - allowImportingTsExtensions: true, - strict: true, - noEmit: true, - skipLibCheck: true, - types: ['node'], - }, + compilerOptions: publicTsconfig.compilerOptions, files: ['index.ts'], }, null, From 1acfa58db0b613df5e22d4363f15ec09955fb488 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 16:15:02 -0600 Subject: [PATCH 5/5] test(bench): run every discovered package test --- bench/package.json | 2 +- bench/scripts/run-package-tests.mjs | 54 ++++++++++++++++++++++++ bench/scripts/verify-packed-consumer.mjs | 10 ++++- bench/src/swe-bench-env.test.ts | 49 ++++++++++----------- bench/src/tb-container-executor.test.mts | 2 +- 5 files changed, 89 insertions(+), 28 deletions(-) create mode 100644 bench/scripts/run-package-tests.mjs diff --git a/bench/package.json b/bench/package.json index 3ed6a062..ff3b5359 100644 --- a/bench/package.json +++ b/bench/package.json @@ -16,7 +16,7 @@ "run-benchmarks": "tsx src/run-benchmarks-cli.mts", "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl", "terminal-compare": "tsx src/terminal-compare.ts", - "test": "TSX_TSCONFIG_PATH=tsconfig.public.json node --test --import tsx src/benchmarks/appworld.test.mts src/benchmarks/external-adapters.test.mts src/benchmarks/rag-benchmarks.test.mts src/corpus.test.mts src/gate.test.mts src/refine-loop.test.mts src/run-benchmarks.test.mts src/selector.test.mts", + "test": "node scripts/run-package-tests.mjs", "typecheck:public": "tsc -p tsconfig.public.json", "verify:package": "node scripts/verify-packed-consumer.mjs" }, diff --git a/bench/scripts/run-package-tests.mjs b/bench/scripts/run-package-tests.mjs new file mode 100644 index 00000000..ced391d7 --- /dev/null +++ b/bench/scripts/run-package-tests.mjs @@ -0,0 +1,54 @@ +import { execFile } from 'node:child_process' +import { access, readdir } from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') +const sourceDir = path.join(benchDir, 'src') + +async function collectTests(dir) { + const files = [] + for (const entry of await readdir(dir, { withFileTypes: true })) { + const absolute = path.join(dir, entry.name) + if (entry.isDirectory()) files.push(...(await collectTests(absolute))) + else if (entry.isFile() && /\.test\.(?:mts|ts)$/.test(entry.name)) files.push(absolute) + } + return files.sort() +} + +async function run(command, args, env = process.env) { + try { + await execFileAsync(command, args, { + cwd: benchDir, + env, + maxBuffer: 10 * 1024 * 1024, + timeout: 120_000, + }) + } catch (error) { + if (error?.stdout) process.stdout.write(error.stdout) + if (error?.stderr) process.stderr.write(error.stderr) + const invocation = [command, ...args].join(' ') + const message = error instanceof Error ? error.message : String(error) + throw new Error(`${invocation} failed: ${message}`, { cause: error }) + } +} + +const python = path.join(benchDir, '.venv', 'bin', 'python') +try { + await access(python) +} catch { + await run('python3', ['-m', 'venv', '.venv']) +} + +const tests = await collectTests(sourceDir) +const relativeTests = tests.map((file) => path.relative(benchDir, file)) +if (relativeTests.length === 0) throw new Error('no package tests found under src/') + +await run(process.execPath, ['--test', '--import', 'tsx', ...relativeTests], { + ...process.env, + TSX_TSCONFIG_PATH: 'tsconfig.public.json', +}) + +console.log(`package tests passed: ${tests.length}/${tests.length} files`) diff --git a/bench/scripts/verify-packed-consumer.mjs b/bench/scripts/verify-packed-consumer.mjs index 0a8fc878..1542aafe 100644 --- a/bench/scripts/verify-packed-consumer.mjs +++ b/bench/scripts/verify-packed-consumer.mjs @@ -35,10 +35,12 @@ try { if ( typeof devDependencies?.['@types/node'] !== 'string' || typeof devDependencies.typescript !== 'string' || + typeof devDependencies.tsx !== 'string' || !devDependencies['@types/node'] || - !devDependencies.typescript + !devDependencies.typescript || + !devDependencies.tsx ) { - throw new Error('package verification requires @types/node and typescript devDependencies') + throw new Error('package verification requires @types/node, typescript, and tsx devDependencies') } const publicTsconfig = JSON.parse(await readFile(path.join(benchDir, 'tsconfig.public.json'), 'utf8')) if (!publicTsconfig.compilerOptions) throw new Error('tsconfig.public.json must define compilerOptions') @@ -54,6 +56,7 @@ try { devDependencies: { '@types/node': devDependencies['@types/node'], typescript: devDependencies.typescript, + tsx: devDependencies.tsx, }, }, null, @@ -76,8 +79,11 @@ try { )}\n`, ) + // Intentionally resolve the declared ranges like a brand-new registry consumer. + // The preceding frozen install + public typecheck cover the exact bench lockfile. await run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--package-lock=false'], consumerDir) await run('npm', ['exec', '--', 'tsc', '-p', 'tsconfig.json'], consumerDir) + await run('npm', ['exec', '--', 'tsx', 'index.ts'], consumerDir) let runtimeManifest try { runtimeManifest = JSON.parse( diff --git a/bench/src/swe-bench-env.test.ts b/bench/src/swe-bench-env.test.ts index 30dbdc02..a2a93d78 100644 --- a/bench/src/swe-bench-env.test.ts +++ b/bench/src/swe-bench-env.test.ts @@ -1,27 +1,28 @@ +import assert from 'node:assert/strict' import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { afterAll, describe, expect, it } from 'vitest' +import { after, describe, it } from 'node:test' import { isInsideJail, isTestPath, jailPath } from './swe-bench-env' describe('isTestPath', () => { it('flags test directories and test-named python files', () => { - expect(isTestPath('tests/test_models.py')).toBe(true) - expect(isTestPath('pkg/test/helpers.py')).toBe(true) - expect(isTestPath('pkg/tests/helpers.py')).toBe(true) - expect(isTestPath('test_models.py')).toBe(true) - expect(isTestPath('models_test.py')).toBe(true) - expect(isTestPath('conftest.py')).toBe(true) - expect(isTestPath('pkg/conftest.py')).toBe(true) + assert.equal(isTestPath('tests/test_models.py'), true) + assert.equal(isTestPath('pkg/test/helpers.py'), true) + assert.equal(isTestPath('pkg/tests/helpers.py'), true) + assert.equal(isTestPath('test_models.py'), true) + assert.equal(isTestPath('models_test.py'), true) + assert.equal(isTestPath('conftest.py'), true) + assert.equal(isTestPath('pkg/conftest.py'), true) }) it('does not flag ordinary source files', () => { - expect(isTestPath('src/foo.py')).toBe(false) - expect(isTestPath('pkg/models.py')).toBe(false) + assert.equal(isTestPath('src/foo.py'), false) + assert.equal(isTestPath('pkg/models.py'), false) // `testing.py` is not a test file by the test_/_test/conftest rules. - expect(isTestPath('pkg/testing.py')).toBe(false) + assert.equal(isTestPath('pkg/testing.py'), false) // A `latest/` segment must not trip the `tests?/` directory rule. - expect(isTestPath('latest/foo.py')).toBe(false) + assert.equal(isTestPath('latest/foo.py'), false) }) }) @@ -29,15 +30,15 @@ describe('jailPath', () => { const root = '/work/repo' it('rejects `..` traversal and absolute paths', () => { - expect(jailPath(root, '../x')).toBeNull() - expect(jailPath(root, 'a/../../etc/passwd')).toBeNull() - expect(jailPath(root, '/etc/passwd')).toBeNull() + assert.equal(jailPath(root, '../x'), null) + assert.equal(jailPath(root, 'a/../../etc/passwd'), null) + assert.equal(jailPath(root, '/etc/passwd'), null) }) it('accepts in-repo relative paths and strips a leading `./`', () => { - expect(jailPath(root, 'src/a.py')).toBe('src/a.py') - expect(jailPath(root, './a.py')).toBe('a.py') - expect(jailPath(root, 'a.py')).toBe('a.py') + assert.equal(jailPath(root, 'src/a.py'), 'src/a.py') + assert.equal(jailPath(root, './a.py'), 'a.py') + assert.equal(jailPath(root, 'a.py'), 'a.py') }) }) @@ -46,13 +47,13 @@ describe('isInsideJail (realpath containment)', () => { // assert containment. Offline — operates on a throwaway temp dir, no git clone, no network. const dir = mkdtempSync(join(tmpdir(), 'swe-jail-')) const jailRoot = realpathSync(dir) - afterAll(() => rmSync(dir, { recursive: true, force: true })) + after(() => rmSync(dir, { recursive: true, force: true })) it('admits a real file inside the jail', () => { const inside = join(dir, 'a.py') writeFileSync(inside, 'x = 1\n') - expect(isInsideJail(jailRoot, realpathSync(inside))).toBe(true) - expect(isInsideJail(jailRoot, jailRoot)).toBe(true) + assert.equal(isInsideJail(jailRoot, realpathSync(inside)), true) + assert.equal(isInsideJail(jailRoot, jailRoot), true) }) it('rejects reading through a symlink that escapes the jail', () => { @@ -61,11 +62,11 @@ describe('isInsideJail (realpath containment)', () => { symlinkSync('/etc', link) // `resolveInJail` does `realpathSync(join(ws.dir, relPath))` then this containment check. const real = realpathSync(join(dir, 'escape/passwd')) - expect(real).toBe('/etc/passwd') - expect(isInsideJail(jailRoot, real)).toBe(false) + assert.equal(real, '/etc/passwd') + assert.equal(isInsideJail(jailRoot, real), false) }) it('rejects a sibling dir that shares the jail-root prefix', () => { - expect(isInsideJail('/tmp/swe-x', '/tmp/swe-x-evil/secret')).toBe(false) + assert.equal(isInsideJail('/tmp/swe-x', '/tmp/swe-x-evil/secret'), false) }) }) diff --git a/bench/src/tb-container-executor.test.mts b/bench/src/tb-container-executor.test.mts index 180f07d2..8afc3569 100644 --- a/bench/src/tb-container-executor.test.mts +++ b/bench/src/tb-container-executor.test.mts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict' import { chmod, mkdtemp, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' -import type { AgentSpec, ExecutorContext } from '../../src/runtime/index' +import type { AgentSpec, ExecutorContext } from '@tangle-network/agent-runtime/loops' import { buildTbDockerExecArgs, createTbContainerExecutor } from './tb-container-executor.mts' const spec: AgentSpec = { profile: { name: 'tb-test-worker' }, harness: null }