fix(bench): release 0.1.1 against runtime 0.90.1

.github/workflows/ci.yml

@@ -39,3 +39,31 @@ jobs:
 
       - name: Docs (regenerate API reference + freshness gate)
         run: pnpm run docs:check
+
+  agent-bench:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: bench
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+          cache-dependency-path: bench/pnpm-lock.yaml
+
+      - name: Install published dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Typecheck public entrypoint against installed packages
+        run: pnpm run typecheck:public
+
+      - name: Test deterministic package paths
+        run: pnpm test
+
+      - name: Verify packed package in a clean consumer
+        run: pnpm run verify:package

.github/workflows/publish.yml

@@ -4,10 +4,12 @@ on:
   push:
     tags:
       - 'v*'
+      - 'agent-bench-v*'
   workflow_dispatch:
 
 jobs:
   verify:
+    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -90,3 +92,80 @@ jobs:
           else
             npm publish --provenance --access public
           fi
+
+  verify-agent-bench:
+    # Manual dispatch remains root-only. Re-run a failed bench tag workflow so
+    # tag/version locking stays intact and the root package is never co-published.
+    if: startsWith(github.ref, 'refs/tags/agent-bench-v')
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: bench
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+          cache-dependency-path: bench/pnpm-lock.yaml
+
+      - name: Install deps
+        run: pnpm install --frozen-lockfile
+
+      - name: Typecheck public entrypoint against installed packages
+        run: pnpm run typecheck:public
+
+      - name: Test deterministic package paths
+        run: pnpm test
+
+      - name: Verify packed package in a clean consumer
+        run: pnpm run verify:package
+
+      - name: Verify tag/version lock
+        run: |
+          NPM_VERSION=$(node -p "require('./package.json').version")
+          TAG_VERSION="${GITHUB_REF#refs/tags/agent-bench-v}"
+          if [ "$TAG_VERSION" != "$NPM_VERSION" ]; then
+            echo "::error::Tag/version mismatch: tag=$TAG_VERSION package=$NPM_VERSION."
+            exit 1
+          fi
+          echo "Version locked: $NPM_VERSION"
+
+  publish-agent-bench:
+    needs: verify-agent-bench
+    if: startsWith(github.ref, 'refs/tags/agent-bench-v')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    defaults:
+      run:
+        working-directory: bench
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+          cache-dependency-path: bench/pnpm-lock.yaml
+
+      - run: pnpm install --frozen-lockfile
+
+      - name: Publish to npm (OIDC trusted publishing)
+        run: |
+          # Requires the npmjs Trusted Publisher on @tangle-network/agent-bench:
+          # org tangle-network, repo agent-runtime, workflow publish.yml.
+          npm install -g npm@11
+          NAME=$(node -p "require('./package.json').name")
+          VERSION=$(node -p "require('./package.json').version")
+          if npm view "$NAME@$VERSION" version >/dev/null 2>&1; then
+            echo "$NAME@$VERSION already on registry; skipping publish"
+          else
+            npm publish --provenance --access public
+          fi

bench/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "type": "module",
   "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
   "main": "src/index.ts",
@@ -15,14 +15,18 @@
     "gate-cli": "tsx src/gate-cli.mts",
     "run-benchmarks": "tsx src/run-benchmarks-cli.mts",
     "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl",
-    "terminal-compare": "tsx src/terminal-compare.ts"
+    "terminal-compare": "tsx src/terminal-compare.ts",
+    "test": "node scripts/run-package-tests.mjs",
+    "typecheck:public": "tsc -p tsconfig.public.json",
+    "verify:package": "node scripts/verify-packed-consumer.mjs"
   },
   "dependencies": {
-    "@tangle-network/agent-eval": "^0.106.3",
-    "@tangle-network/agent-runtime": "^0.89.0",
+    "@tangle-network/agent-eval": "^0.108.1",
+    "@tangle-network/agent-runtime": "^0.90.1",
     "@tangle-network/sandbox": "^0.9.7"
   },
   "devDependencies": {
+    "@types/node": "^25.0.0",
     "tsx": "^4.19.0",
     "typescript": "^6.0.3"
   },

bench/scripts/run-package-tests.mjs

@@ -0,0 +1,54 @@
+import { execFile } from 'node:child_process'
+import { access, readdir } from 'node:fs/promises'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { promisify } from 'node:util'
+
+const execFileAsync = promisify(execFile)
+const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
+const sourceDir = path.join(benchDir, 'src')
+
+async function collectTests(dir) {
+  const files = []
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const absolute = path.join(dir, entry.name)
+    if (entry.isDirectory()) files.push(...(await collectTests(absolute)))
+    else if (entry.isFile() && /\.test\.(?:mts|ts)$/.test(entry.name)) files.push(absolute)
+  }
+  return files.sort()
+}
+
+async function run(command, args, env = process.env) {
+  try {
+    await execFileAsync(command, args, {
+      cwd: benchDir,
+      env,
+      maxBuffer: 10 * 1024 * 1024,
+      timeout: 120_000,
+    })
+  } catch (error) {
+    if (error?.stdout) process.stdout.write(error.stdout)
+    if (error?.stderr) process.stderr.write(error.stderr)
+    const invocation = [command, ...args].join(' ')
+    const message = error instanceof Error ? error.message : String(error)
+    throw new Error(`${invocation} failed: ${message}`, { cause: error })
+  }
+}
+
+const python = path.join(benchDir, '.venv', 'bin', 'python')
+try {
+  await access(python)
+} catch {
+  await run('python3', ['-m', 'venv', '.venv'])
+}
+
+const tests = await collectTests(sourceDir)
+const relativeTests = tests.map((file) => path.relative(benchDir, file))
+if (relativeTests.length === 0) throw new Error('no package tests found under src/')
+
+await run(process.execPath, ['--test', '--import', 'tsx', ...relativeTests], {
+  ...process.env,
+  TSX_TSCONFIG_PATH: 'tsconfig.public.json',
+})
+
+console.log(`package tests passed: ${tests.length}/${tests.length} files`)