From 94601394f1c237d694bbbe1e4d224c423dd7708f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 16:30:34 -0600 Subject: [PATCH 01/10] feat(campaign)!: make code surfaces content-addressed --- .claude/skills/agent-eval/SKILL.md | 2 +- .../2026-07-09T22-27-32Z/findings.jsonl | 1 + .../2026-07-09T22-27-32Z/manifest.json | 32 ++ .../2026-07-09T22-27-32Z/summary.md | 34 ++ .evolve/skill-runs.jsonl | 3 + CHANGELOG.md | 9 + clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- docs/design/loop-taxonomy.md | 6 +- docs/improvement-glossary.md | 2 +- package.json | 2 +- scripts/verify-package-exports.mjs | 12 +- src/campaign/analyst-surface.test.ts | 6 +- src/campaign/index.ts | 7 + src/campaign/presets/run-optimization.ts | 19 +- src/campaign/proposers/policy-edit.ts | 4 +- src/campaign/provenance.ts | 18 +- src/campaign/surface-identity.ts | 65 ++++ src/campaign/types.ts | 37 ++- src/campaign/worktree/index.ts | 312 ++++++++++++++++-- tests/campaign/policy-edit-proposer.test.ts | 37 +++ tests/campaign/presets.test.ts | 44 ++- tests/campaign/skill-opt.test.ts | 2 +- tests/campaign/worktree.test.ts | 235 ++++++++++++- 25 files changed, 787 insertions(+), 108 deletions(-) create mode 100644 .evolve/critical-audit/2026-07-09T22-27-32Z/findings.jsonl create mode 100644 .evolve/critical-audit/2026-07-09T22-27-32Z/manifest.json create mode 100644 .evolve/critical-audit/2026-07-09T22-27-32Z/summary.md create mode 100644 src/campaign/surface-identity.ts diff --git a/.claude/skills/agent-eval/SKILL.md b/.claude/skills/agent-eval/SKILL.md index 5e8278c6..a07d6340 100644 --- a/.claude/skills/agent-eval/SKILL.md +++ b/.claude/skills/agent-eval/SKILL.md @@ -976,7 +976,7 @@ Do you have ≥5 well-defined transformations to test (e.g. directive enum)? **Recipe by tier:** - **Prompt-tier (string surface)** — start with `gepaDriver()`. Escalate to a custom `Mutator` + `evolutionaryDriver` only when you have a known directive enum that beats LLM-discovery. -- **Code-tier (`CodeSurface`)** — use agent-runtime's `improvementDriver` with a `reflectiveGenerator` or `agenticGenerator`. The substrate's drivers don't reason about code diffs. +- **Code-tier (`CodeSurface`)** — use agent-runtime's `improvementDriver` with a `reflectiveGenerator` or `agenticGenerator`, then finalize through `gitWorktreeAdapter` and call `verifyCodeSurface` before execution. The substrate's drivers don't reason about code diffs. - **Knowledge-tier** — out of scope here; agent-knowledge owns this. ### Surface menu — what string to pick diff --git a/.evolve/critical-audit/2026-07-09T22-27-32Z/findings.jsonl b/.evolve/critical-audit/2026-07-09T22-27-32Z/findings.jsonl new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T22-27-32Z/findings.jsonl @@ -0,0 +1 @@ + diff --git a/.evolve/critical-audit/2026-07-09T22-27-32Z/manifest.json b/.evolve/critical-audit/2026-07-09T22-27-32Z/manifest.json new file mode 100644 index 00000000..f4572e75 --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T22-27-32Z/manifest.json @@ -0,0 +1,32 @@ +{ + "scope": "uncommitted diff for immutable CodeSurface identity", + "base": "39e8002748f43aa9b440805e7c73000d8aba1a19", + "head": "39e8002748f43aa9b440805e7c73000d8aba1a19", + "project_type": "typescript-node", + "flags": ["diff-only"], + "files": [ + ".claude/skills/agent-eval/SKILL.md", + "CHANGELOG.md", + "docs/design/loop-taxonomy.md", + "docs/improvement-glossary.md", + "scripts/verify-package-exports.mjs", + "src/campaign/analyst-surface.test.ts", + "src/campaign/index.ts", + "src/campaign/presets/run-optimization.ts", + "src/campaign/proposers/policy-edit.ts", + "src/campaign/provenance.ts", + "src/campaign/surface-identity.ts", + "src/campaign/types.ts", + "src/campaign/worktree/index.ts", + "tests/campaign/policy-edit-proposer.test.ts", + "tests/campaign/presets.test.ts", + "tests/campaign/skill-opt.test.ts", + "tests/campaign/worktree.test.ts" + ], + "findings_count_by_severity": { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0 + } +} diff --git a/.evolve/critical-audit/2026-07-09T22-27-32Z/summary.md b/.evolve/critical-audit/2026-07-09T22-27-32Z/summary.md new file mode 100644 index 00000000..a871bfed --- /dev/null +++ b/.evolve/critical-audit/2026-07-09T22-27-32Z/summary.md @@ -0,0 +1,34 @@ +# Immutable CodeSurface critical audit + +## Findings + +No remaining correctness, security, architecture, or public-contract findings in the scoped diff. + +The pre-audit hardening pass found one real repository-redirection path through inherited `GIT_DIR`, `GIT_WORK_TREE`, and `GIT_CONFIG_*` variables. +The implementation now removes repository-routing Git variables and disables filesystem-monitor and untracked-cache shortcuts during verification; the real-Git adversarial test proves a wrong locator still fails while those variables point at the expected repository. + +## Open assumptions and residual risk + +- `verifyCodeSurface` is a point-in-time identity check, not a filesystem lease. + Sealed evaluation must consume the returned patch bytes in an isolated checkout, verify the resulting tree, and avoid executing the mutable proposal worktree. +- Candidate symlink and path-containment policy belongs to that materializer. + This package proves Git content identity; it does not declare an arbitrary repository safe to execute. +- Binary-patch bytes are capped at 256 MiB and generated with pinned diff options. + Larger candidates fail instead of weakening verification. + +## Review scores + +- Correctness and security: 8/10. +- Architecture and maintainability: 8/10. +- Standards and real-system coverage: 8/10. +- Overall: 8/10. + +## Checks + +- `pnpm typecheck` +- `pnpm lint` (four pre-existing warnings, zero errors) +- `pnpm test -- --reporter=dot` +- `pnpm build` +- `pnpm verify:package` + +APPROVE — content identity is path-independent, byte-verifiable, fail-closed, and covered by real Git repositories; the sealed materializer remains the required consumer-side boundary. diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index e2586c67..20cc752d 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -17,3 +17,6 @@ {"skill":"critical-audit","ts":"2026-06-21T12:57:05Z","project":"agent-eval","target":"PR #264 review comment 4762012574","verdict":"APPROVE_AFTER_FIXES","score":"8/10","findings":"1 medium, 9 low; 8 fixed, 1 not_actionable, 1 deferred","files":[".gitignore","CHANGELOG.md","src/agent-profile.ts","src/agent-profile.test.ts","src/contract/define-agent-eval.ts","src/contract/index.ts","tests/contract-define-agent-eval.test.ts"]} {"skill":"critical-audit","ts":"2026-07-09T21:54:37Z","project":"agent-eval","target":"clustered paired binary inference for repository-disjoint coding benchmarks","verdict":"APPROVE_AFTER_FIXES","score":"8/10","findings":"3 medium found and resolved: estimand mismatch, one-cluster interval, insufficient bootstrap draws","files":["src/clustered-paired-binary.ts","src/statistics.ts","src/index.ts","tests/clustered-paired-binary.test.ts",".claude/skills/agent-eval/SKILL.md"]} {"skill":"critical-audit","ts":"2026-07-09T22:35:41Z","project":"agent-eval","target":"clustered paired binary inference at abe1d89a11fe1b40b75d0e4d96ccf68dfa2403b1","verdict":"APPROVE","score":"8/10","findings":"1 low accepted risk: independent-package parity vector remains before paper use","files":["src/clustered-paired-binary.ts","src/statistics.ts","src/index.ts","tests/clustered-paired-binary.test.ts",".claude/skills/agent-eval/SKILL.md"]} +{"skill":"agent-eval","ts":"2026-07-09T22:27:32Z","project":"agent-eval","target":"immutable CodeSurface identity","outcome":"implemented content-addressed Git identity and fail-loud verification","branch":"feat/immutable-code-surface","model_runs":0} +{"skill":"harden","ts":"2026-07-09T22:27:32Z","project":"agent-eval","target":"CodeSurface Git trust boundary","outcome":"fixed inherited Git environment repository-redirection bypass; added real-Git adversarial coverage","branch":"feat/immutable-code-surface","model_runs":0} +{"skill":"critical-audit","ts":"2026-07-09T22:27:32Z","project":"agent-eval","target":"uncommitted immutable CodeSurface diff","verdict":"APPROVE","score":"8/10","findings":"0 remaining; point-in-time checkout verification documented as residual boundary","files":["src/campaign/surface-identity.ts","src/campaign/types.ts","src/campaign/worktree/index.ts","tests/campaign/worktree.test.ts"]} diff --git a/CHANGELOG.md b/CHANGELOG.md index 90e50842..c8ca3425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.112.0] — 2026-07-09 — immutable code candidates + +### Changed + +- `CodeSurface` is now a finalized, content-addressed code candidate. `gitWorktreeAdapter.finalize()` records exact base/candidate commits, the final tree, and the SHA-256 + byte length of the raw binary Git patch; `surfaceHash` and `surfaceContentHash` no longer use filesystem paths. +- `resolveWorktreePath()` now verifies the candidate before returning its checkout. Dirty or ignored files, moved refs, missing objects, wrong trees, and patch-byte mismatches fail instead of being evaluated under stale identity. Use `verifyCodeSurface()` when a verification receipt is needed directly. + +This intentionally rejects path-only `CodeSurface` objects; callers must finalize code candidates through a worktree adapter. + ## [0.111.0] — 2026-07-09 — repository-clustered paired inference ### Added diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 687c6007..59d4b596 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.111.0" +version = "0.112.0" description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 5d63aab6..03ed89b8 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -58,7 +58,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.111.0" + __version__ = "0.112.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 5af1ec79..a3a87c32 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.10" [[package]] name = "agent-eval-rpc" -version = "0.111.0" +version = "0.112.0" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/docs/design/loop-taxonomy.md b/docs/design/loop-taxonomy.md index da0f7d37..2374ee45 100644 --- a/docs/design/loop-taxonomy.md +++ b/docs/design/loop-taxonomy.md @@ -162,7 +162,11 @@ imports agent-eval and implements it. `MutableSurface` is the thing the proposer changes. It has tiers, least → most invasive. `MutableSurface = string | CodeSurface` spans all of them: `string` -for tiers 1–2, `CodeSurface{ worktreeRef }` for tier 4. +for tiers 1–2, and a finalized `CodeSurface` for tier 4. A code surface's +worktree path is only its locator; exact base/candidate commits, final tree, +and binary-patch digest are its portable identity. Call `verifyCodeSurface` +before executing the checkout so a moved ref or post-finalization mutation +fails before measurement. | Tier | Surface | Generator that changes it | Blast radius | |---|---|---|---| diff --git a/docs/improvement-glossary.md b/docs/improvement-glossary.md index fa0c4b38..af6898f4 100644 --- a/docs/improvement-glossary.md +++ b/docs/improvement-glossary.md @@ -30,7 +30,7 @@ Grounded to `agent-eval/src/campaign/types.ts` unless noted. | Term | Plain sentence | If you know DSPy/GEPA | |---|---|---| -| **surface** | The one thing being changed this run — a prompt string, a JSON config string, or a code/worktree ref (`MutableSurface = string \| CodeSurface`, `types.ts:158`). | The optimized artifact: a signature/predictor's instruction text, or the module config. | +| **surface** | The one thing being changed this run — a prompt string, a JSON config string, or a finalized code candidate (`MutableSurface = string \| CodeSurface`, `types.ts`). | The optimized artifact: a signature/predictor's instruction text, or the module config. | | **proposer** | The strategy that, given the current surface + what failed, proposes the next batch of candidate surfaces to measure — it does **not** run the agent or score anything (`SurfaceProposer.propose`, `types.ts:286`). | The optimizer / teleprompter (MIPRO, BootstrapFewShot, GEPA's reflective proposer). | | **candidate** | One proposed surface plus its human `label` and `rationale`, ready to be measured (`ProposedCandidate`, `types.ts:166`). | One trial instruction/program the optimizer wants to evaluate. | | **generation** | One round of *propose → measure → rank → promote*; the loop runs up to `maxGenerations` of them (`GenerationRecord`, `types.ts:549`). | One GEPA iteration / optimization step. | diff --git a/package.json b/package.json index bb4041db..8fb018d1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.111.0", + "version": "0.112.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index c0a74b91..b047c974 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -80,8 +80,16 @@ try { throw new Error('missing belief-state export analyzeBeliefPolicy') } const campaign = await import('@tangle-network/agent-eval/campaign') - if (!('compositeProposer' in campaign)) { - throw new Error('missing campaign export compositeProposer') + for (const name of [ + 'compositeProposer', + 'gitWorktreeAdapter', + 'verifyCodeSurface', + 'resolveWorktreePath', + 'assertCodeSurfaceIdentity', + 'surfaceHash', + 'surfaceContentHash', + ]) { + if (!(name in campaign)) throw new Error('missing campaign export ' + name) } `, ], diff --git a/src/campaign/analyst-surface.test.ts b/src/campaign/analyst-surface.test.ts index 41329889..cac00fa9 100644 --- a/src/campaign/analyst-surface.test.ts +++ b/src/campaign/analyst-surface.test.ts @@ -61,9 +61,9 @@ describe('buildAnalystSurfaceDispatch — runs the analyst with the surface as a it('fails loud if handed a code-tier surface (analyst prompt is prompt-tier)', async () => { const dispatch = buildAnalystSurfaceDispatch({ analystOptions: { ai: {} as never } }) - await expect(dispatch({ kind: 'code', worktreeRef: 'wt/abc' }, scenario, ctx)).rejects.toThrow( - /prompt-tier/, - ) + await expect( + dispatch({ kind: 'code', worktreeRef: 'wt/abc' } as never, scenario, ctx), + ).rejects.toThrow(/prompt-tier/) }) }) diff --git a/src/campaign/index.ts b/src/campaign/index.ts index 0ba1bd16..30c1af87 100644 --- a/src/campaign/index.ts +++ b/src/campaign/index.ts @@ -286,6 +286,11 @@ export { type SkillPatchRejection, } from './skill-patch' export { type CampaignStorage, fsCampaignStorage, inMemoryCampaignStorage } from './storage' +// ── Code-surface content identity ──────────────────────────────────── +export { + assertCodeSurfaceIdentity, + codeSurfaceIdentityMaterial, +} from './surface-identity' export { isTransientTransportFailure, type TransientFailureOptions } from './transient-failure' export type { CampaignAggregates, @@ -331,9 +336,11 @@ export type { export { isProposedCandidate, labelTrustRank } from './types' // ── Worktree adapter (VCS-pluggable; code-tier surfaces) ───────────── export { + type CodeSurfaceVerification, type GitWorktreeAdapterOptions, gitWorktreeAdapter, resolveWorktreePath, + verifyCodeSurface, type Worktree, type WorktreeAdapter, WorktreeAdapterError, diff --git a/src/campaign/presets/run-optimization.ts b/src/campaign/presets/run-optimization.ts index 96562b8c..e5e96ce9 100644 --- a/src/campaign/presets/run-optimization.ts +++ b/src/campaign/presets/run-optimization.ts @@ -15,10 +15,10 @@ * re-score + release gate + optional PR. */ -import { createHash } from 'node:crypto' import { type Objective, paretoFrontier } from '../../pareto' import { type RunCampaignOptions, runCampaign } from '../run-campaign' import { campaignBreakdown, campaignMeanComposite } from '../score-utils' +import { surfaceHash } from '../surface-identity' import { type CampaignResult, type GenerationRecord, @@ -355,19 +355,4 @@ function computeParetoFrontier(scored: ParetoParent[]): ParetoParent[] { return paretoFrontier(scored, objectives).frontier } -/** - * Short (16-char) sha256 fingerprint of a `MutableSurface`: hashes text content for prompt surfaces, or the worktree + base ref pair for code surfaces. - */ -export function surfaceHash(surface: MutableSurface): string { - // Prompt/tool surfaces (string) hash by content; code surfaces hash by the - // worktree + base ref pair (the content lives in git, not in the string). - const material = - typeof surface === 'string' - ? surface - : JSON.stringify({ - kind: surface.kind, - worktreeRef: surface.worktreeRef, - baseRef: surface.baseRef ?? null, - }) - return createHash('sha256').update(material).digest('hex').slice(0, 16) -} +export { surfaceHash } from '../surface-identity' diff --git a/src/campaign/proposers/policy-edit.ts b/src/campaign/proposers/policy-edit.ts index ef5b7338..e4f2e140 100644 --- a/src/campaign/proposers/policy-edit.ts +++ b/src/campaign/proposers/policy-edit.ts @@ -16,6 +16,7 @@ import { policyEditsFromFindings, } from '../../analyst/policy-edit' import type { AnalystFinding } from '../../analyst/types' +import { assertCodeSurfaceIdentity } from '../surface-identity' import type { MutableSurface, ProposeContext, ProposedCandidate, SurfaceProposer } from '../types' export interface PolicyEditProposerOptions { @@ -108,7 +109,8 @@ function coerceCandidateSurface(surface: unknown): MutableSurface { if (surface && typeof surface === 'object') { const obj = surface as Record if (obj.kind === 'code' && typeof obj.worktreeRef === 'string') { - return surface as MutableSurface + assertCodeSurfaceIdentity(surface) + return surface } return JSON.stringify(surface, null, 2) } diff --git a/src/campaign/provenance.ts b/src/campaign/provenance.ts index c47ce729..ced927b9 100644 --- a/src/campaign/provenance.ts +++ b/src/campaign/provenance.ts @@ -37,24 +37,10 @@ import type { import { summarizeBackendIntegrity } from '../integrity/backend-integrity' import type { RunRecord } from '../run-record' import type { CampaignStorage } from './storage' +import { surfaceContentHash } from './surface-identity' import type { CampaignResult, GateDecision, GateResult, MutableSurface, Scenario } from './types' -/** Stable sha256 (full hex) of a surface's effective text. Code surfaces hash - * their worktree+base identity since the content lives in git. Distinct from - * `surfaceHash` (16-char content fingerprint used as a loop identity key); - * this is the byte-identical-verifiable content hash the provenance record + - * `RunRecord.promptHash` carry. */ -export function surfaceContentHash(surface: MutableSurface): string { - const material = - typeof surface === 'string' - ? surface - : JSON.stringify({ - kind: surface.kind, - worktreeRef: surface.worktreeRef, - baseRef: surface.baseRef ?? null, - }) - return `sha256:${createHash('sha256').update(material).digest('hex')}` -} +export { surfaceContentHash } from './surface-identity' export interface LoopProvenanceCandidate { /** Generation index this candidate was proposed in. */ diff --git a/src/campaign/surface-identity.ts b/src/campaign/surface-identity.ts new file mode 100644 index 00000000..b79b23f6 --- /dev/null +++ b/src/campaign/surface-identity.ts @@ -0,0 +1,65 @@ +import { createHash } from 'node:crypto' +import type { CodeSurface, MutableSurface } from './types' + +const GIT_OBJECT_ID = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/ +const SHA256 = /^sha256:[a-f0-9]{64}$/ + +/** Fail when a code surface does not carry a complete immutable identity. */ +export function assertCodeSurfaceIdentity(surface: unknown): asserts surface is CodeSurface { + if (!surface || typeof surface !== 'object') { + throw new TypeError('CodeSurface must be an object') + } + const candidate = surface as Partial + if (candidate.kind !== 'code') throw new TypeError('CodeSurface.kind must be "code"') + if (typeof candidate.worktreeRef !== 'string' || candidate.worktreeRef.trim().length === 0) { + throw new TypeError('CodeSurface.worktreeRef must be a non-empty locator') + } + if (typeof candidate.baseRef !== 'string' || candidate.baseRef.trim().length === 0) { + throw new TypeError('CodeSurface.baseRef must be a non-empty ref label') + } + for (const [field, value] of [ + ['baseCommit', candidate.baseCommit], + ['baseTree', candidate.baseTree], + ['candidateCommit', candidate.candidateCommit], + ['candidateTree', candidate.candidateTree], + ] as const) { + if (typeof value !== 'string' || !GIT_OBJECT_ID.test(value)) { + throw new TypeError(`CodeSurface.${field} must be a full Git object id`) + } + } + const patch = candidate.patch + if (!patch || typeof patch !== 'object' || patch.format !== 'git-diff-binary') { + throw new TypeError('CodeSurface.patch.format must be "git-diff-binary"') + } + if (typeof patch.sha256 !== 'string' || !SHA256.test(patch.sha256)) { + throw new TypeError('CodeSurface.patch.sha256 must be a sha256 digest') + } + if (!Number.isSafeInteger(patch.byteLength) || patch.byteLength < 0) { + throw new TypeError('CodeSurface.patch.byteLength must be a non-negative safe integer') + } +} + +/** Canonical, location-independent identity of a finalized code candidate. + * Commit metadata is excluded: two commits with the same base, final tree, + * and patch bytes are the same executable candidate. */ +export function codeSurfaceIdentityMaterial(surface: CodeSurface): string { + assertCodeSurfaceIdentity(surface) + return JSON.stringify({ + schema: 'tangle.code-surface.v1', + baseCommit: surface.baseCommit, + baseTree: surface.baseTree, + candidateTree: surface.candidateTree, + patch: surface.patch, + }) +} + +/** Full SHA-256 content identity for a prompt or finalized code surface. */ +export function surfaceContentHash(surface: MutableSurface): `sha256:${string}` { + const material = typeof surface === 'string' ? surface : codeSurfaceIdentityMaterial(surface) + return `sha256:${createHash('sha256').update(material).digest('hex')}` +} + +/** Short loop key derived from the same content identity as provenance. */ +export function surfaceHash(surface: MutableSurface): string { + return surfaceContentHash(surface).slice('sha256:'.length, 'sha256:'.length + 16) +} diff --git a/src/campaign/types.ts b/src/campaign/types.ts index 0523e5ce..789e53f0 100644 --- a/src/campaign/types.ts +++ b/src/campaign/types.ts @@ -132,20 +132,35 @@ export interface JudgeScore { // ── Optimization (population + generations + mutator) ───────────────── -/** A tier-4 code surface — a candidate change to the agent's +/** A tier-4 code surface — a finalized candidate change to the agent's * IMPLEMENTATION, not its prompt. Produced by autoresearch (reads codebase + - * trace findings → opens a worktree). Measured by checking out `worktreeRef` - * and running the worker against the changed code. See the improvement-tier - * table in `docs/design/loop-taxonomy.md`. */ + * trace findings → opens a worktree). `worktreeRef` locates the candidate; + * the exact commits, tree, and binary-patch digest identify it. See the + * improvement-tier table in `docs/design/loop-taxonomy.md`. */ export interface CodeSurface { - kind: 'code' - /** Worktree path or git ref holding the candidate code change. The - * consumer's `dispatchWithSurface` checks this out before running. */ - worktreeRef: string - /** Base ref the change is measured against. Default: the repo's main. */ - baseRef?: string + readonly kind: 'code' + /** Worktree path or git ref holding the candidate code change. This is a + * mutable locator and is deliberately excluded from content hashes. */ + readonly worktreeRef: string + /** Human-readable ref the worktree was forked from. Not identity-bearing. */ + readonly baseRef: string + /** Exact commit the candidate was forked from. */ + readonly baseCommit: string + /** Exact tree object for `baseCommit`. */ + readonly baseTree: string + /** Exact finalized candidate commit. */ + readonly candidateCommit: string + /** Exact tree object for `candidateCommit`. */ + readonly candidateTree: string + /** Identity of the exact patch artifact. The deployable candidate bundle + * carries the same descriptor plus its base64-encoded content. */ + readonly patch: { + readonly format: 'git-diff-binary' + readonly sha256: `sha256:${string}` + readonly byteLength: number + } /** Human summary of what changed — rendered into the auto-PR body. */ - summary?: string + readonly summary?: string } /** The mutable surface a proposer changes. Tiers (see diff --git a/src/campaign/worktree/index.ts b/src/campaign/worktree/index.ts index 27b86884..cb8c2f42 100644 --- a/src/campaign/worktree/index.ts +++ b/src/campaign/worktree/index.ts @@ -2,8 +2,8 @@ * VCS-pluggable worktree adapter. One improvement = one worktree, PR-like * (multiple commits allowed). A code-tier proposer's `propose()` creates a * worktree, an agent commits the change into it, and `finalize()` returns a - * `CodeSurface{ worktreeRef }` the measurement checks out to run the worker - * against the changed code. On promotion the worktree becomes the PR branch. + * content-addressed `CodeSurface` the measurement verifies before running. + * On promotion the worktree becomes the PR branch. * * The interface is VCS-agnostic so a future `jj` ([jj-vcs](https://github.com/jj-vcs/jj)) * adapter can slot in without touching proposer code. Only the git adapter @@ -11,25 +11,47 @@ */ import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' import { existsSync } from 'node:fs' import { basename, isAbsolute, join } from 'node:path' +import { assertCodeSurfaceIdentity, surfaceContentHash } from '../surface-identity' import type { CodeSurface } from '../types' +const MAX_GIT_OUTPUT_BYTES = 256 * 1024 * 1024 + +type GitOutput = string | Uint8Array +type GitRunner = (args: string[], cwd: string) => GitOutput + +const GIT_REPOSITORY_ENV = new Set([ + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_COMMON_DIR', + 'GIT_DIR', + 'GIT_INDEX_FILE', + 'GIT_NAMESPACE', + 'GIT_OBJECT_DIRECTORY', + 'GIT_PREFIX', + 'GIT_QUARANTINE_PATH', + 'GIT_WORK_TREE', +]) + export interface Worktree { /** Absolute path to the checked-out worktree directory. */ - path: string + readonly path: string /** The branch the worktree is on (becomes the PR branch on promotion). */ - branch: string + readonly branch: string /** The ref the worktree was forked from. */ - baseRef: string + readonly baseRef: string + /** Exact commit `baseRef` resolved to before the worktree was created. */ + readonly baseCommit: string + /** Exact tree object for `baseCommit`. */ + readonly baseTree: string } export interface WorktreeAdapter { /** Create an isolated worktree on a fresh branch off `baseRef`. */ create(opts: { baseRef: string; label: string }): Promise - /** Commit any pending changes in the worktree, then return a CodeSurface - * pointing at it. The agent has already written its change into - * `worktree.path` by the time this is called. */ + /** Commit pending changes, freeze the exact Git objects + binary patch, and + * verify the worktree still matches that identity. */ finalize(worktree: Worktree, summary: string): Promise /** Remove the worktree (and its branch) — called for losing candidates. */ discard(worktree: Worktree): Promise @@ -53,13 +75,20 @@ export interface GitWorktreeAdapterOptions { worktreeDir?: string /** Branch-name prefix. Default: `improve`. */ branchPrefix?: string - /** Test seam — defaults to a real `git` runner. */ - git?: (args: string[], cwd: string) => string + /** Test seam — defaults to a real `git` runner. The return value must contain + * stdout verbatim; trimming a binary diff changes candidate identity. */ + git?: GitRunner } -function defaultGit(args: string[], cwd: string): string { +function defaultGit(args: string[], cwd: string): Uint8Array { try { - return execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() + const env = { ...process.env } + for (const key of Object.keys(env)) { + if (GIT_REPOSITORY_ENV.has(key) || key === 'GIT_CONFIG' || key.startsWith('GIT_CONFIG_')) { + delete env[key] + } + } + return execFileSync('git', args, { cwd, env, maxBuffer: MAX_GIT_OUTPUT_BYTES }) } catch (err) { const stderr = err && typeof err === 'object' && 'stderr' in err @@ -69,6 +98,202 @@ function defaultGit(args: string[], cwd: string): string { } } +function gitBytes(git: GitRunner, args: string[], cwd: string): Buffer { + try { + const output = git(args, cwd) + return typeof output === 'string' ? Buffer.from(output, 'utf8') : Buffer.from(output) + } catch (err) { + if (err instanceof WorktreeAdapterError) throw err + throw new WorktreeAdapterError(`git ${args.join(' ')} failed: ${String(err)}`, err) + } +} + +function gitText(git: GitRunner, args: string[], cwd: string): string { + return gitBytes(git, args, cwd).toString('utf8').trim() +} + +function sha256(bytes: Uint8Array): `sha256:${string}` { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +const PATCH_ARGS = [ + 'diff', + '--binary', + '--full-index', + '--no-color', + '--no-ext-diff', + '--no-textconv', + '--no-renames', + '--diff-algorithm=myers', + '--no-indent-heuristic', + '--src-prefix=a/', + '--dst-prefix=b/', +] as const + +function patchBytes( + git: GitRunner, + cwd: string, + baseCommit: string, + candidateCommit: string, +): Buffer { + return gitBytes(git, [...PATCH_ARGS, baseCommit, candidateCommit, '--'], cwd) +} + +function resolveCommit(git: GitRunner, cwd: string, ref: string): string { + return gitText(git, ['rev-parse', '--verify', `${ref}^{commit}`], cwd) +} + +function unresolvedWorktreePath(surface: CodeSurface, worktreeDir?: string): string { + if (isAbsolute(surface.worktreeRef)) return surface.worktreeRef + if (worktreeDir) return join(worktreeDir, basename(surface.worktreeRef)) + return surface.worktreeRef +} + +export interface CodeSurfaceVerification { + /** Verified worktree path. */ + path: string + /** Git's canonical root for the verified checkout. */ + repoRoot: string + /** Recomputed full content identity. */ + contentHash: `sha256:${string}` + /** Exact verified binary-patch bytes. Candidate-bundle builders encode this + * directly instead of reproducing Git diff options. */ + patchBytes: Uint8Array +} + +function verifyCodeSurfaceWithGit( + surface: CodeSurface, + path: string, + git: GitRunner, +): CodeSurfaceVerification { + assertCodeSurfaceIdentity(surface) + if (!existsSync(path)) { + throw new WorktreeAdapterError(`CodeSurface worktree does not exist: ${path}`) + } + + const indexFlags = gitText(git, ['ls-files', '-v'], path) + .split('\n') + .filter((line) => line.length > 0 && (line.startsWith('S ') || /^[a-z]/.test(line))) + if (indexFlags.length > 0) { + throw new WorktreeAdapterError( + `CodeSurface worktree uses hidden index entries that cannot be verified: ${path}\n${indexFlags.join('\n')}`, + ) + } + + const status = gitText( + git, + [ + '-c', + 'core.fsmonitor=false', + '-c', + 'core.untrackedCache=false', + 'status', + '--porcelain=v1', + '--untracked-files=all', + '--ignored=matching', + '--ignore-submodules=none', + ], + path, + ) + if (status.length > 0) { + throw new WorktreeAdapterError( + `CodeSurface worktree changed after finalization: ${path}\n${status}`, + ) + } + + const candidateCommit = resolveCommit(git, path, 'HEAD') + if (candidateCommit !== surface.candidateCommit) { + throw new WorktreeAdapterError( + `CodeSurface candidate commit mismatch: expected ${surface.candidateCommit}, got ${candidateCommit}`, + ) + } + + const baseCommit = resolveCommit(git, path, surface.baseCommit) + if (baseCommit !== surface.baseCommit) { + throw new WorktreeAdapterError( + `CodeSurface base commit mismatch: expected ${surface.baseCommit}, got ${baseCommit}`, + ) + } + + const baseTree = gitText(git, ['rev-parse', '--verify', `${surface.baseCommit}^{tree}`], path) + if (baseTree !== surface.baseTree) { + throw new WorktreeAdapterError( + `CodeSurface base tree mismatch: expected ${surface.baseTree}, got ${baseTree}`, + ) + } + + const candidateObject = resolveCommit(git, path, surface.candidateCommit) + if (candidateObject !== surface.candidateCommit) { + throw new WorktreeAdapterError( + `CodeSurface candidate object mismatch: expected ${surface.candidateCommit}, got ${candidateObject}`, + ) + } + + const candidateTree = gitText( + git, + ['rev-parse', '--verify', `${surface.candidateCommit}^{tree}`], + path, + ) + if (candidateTree !== surface.candidateTree) { + throw new WorktreeAdapterError( + `CodeSurface tree mismatch: expected ${surface.candidateTree}, got ${candidateTree}`, + ) + } + + // Rebuild the index from the frozen commit before comparing it to the + // filesystem. This clears cached stat data, so same-size/same-mtime edits + // cannot hide behind Git's normal status shortcut. + gitText(git, ['read-tree', '--reset', surface.candidateCommit], path) + try { + gitText( + git, + [ + '-c', + 'core.filemode=true', + '-c', + 'core.symlinks=true', + '-c', + 'core.fsmonitor=false', + '-c', + 'core.ignorestat=false', + 'diff-files', + '--no-ext-diff', + '--no-textconv', + '--quiet', + '--', + ], + path, + ) + } catch (err) { + throw new WorktreeAdapterError( + `CodeSurface tracked bytes changed after finalization: ${path}`, + err, + ) + } + + const mergeBase = gitText(git, ['merge-base', surface.baseCommit, surface.candidateCommit], path) + if (mergeBase !== surface.baseCommit) { + throw new WorktreeAdapterError( + `CodeSurface candidate ${surface.candidateCommit} does not descend from base ${surface.baseCommit}`, + ) + } + + const patch = patchBytes(git, path, surface.baseCommit, surface.candidateCommit) + const actualPatchHash = sha256(patch) + if (actualPatchHash !== surface.patch.sha256 || patch.byteLength !== surface.patch.byteLength) { + throw new WorktreeAdapterError( + `CodeSurface patch mismatch: expected ${surface.patch.sha256}/${surface.patch.byteLength} bytes, got ${actualPatchHash}/${patch.byteLength} bytes`, + ) + } + + return { + path, + repoRoot: gitText(git, ['rev-parse', '--show-toplevel'], path), + contentHash: surfaceContentHash(surface), + patchBytes: new Uint8Array(patch), + } +} + /** Slugify a label into a branch-safe segment. */ function slug(label: string): string { return ( @@ -93,40 +318,75 @@ export function gitWorktreeAdapter(opts: GitWorktreeAdapterOptions): WorktreeAda const id = `${slug(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}` const branch = `${branchPrefix}/${id}` const path = join(worktreeDir, id) - git(['worktree', 'add', '-b', branch, path, baseRef], opts.repoRoot) - return { path, branch, baseRef } + const baseCommit = resolveCommit(git, opts.repoRoot, baseRef) + const baseTree = gitText( + git, + ['rev-parse', '--verify', `${baseCommit}^{tree}`], + opts.repoRoot, + ) + gitText(git, ['worktree', 'add', '-b', branch, path, baseCommit], opts.repoRoot) + return { path, branch, baseRef, baseCommit, baseTree } }, async finalize(worktree, summary) { // Stage + commit any pending changes the agent left in the worktree. // A no-op commit is refused by git, so only commit when the tree is dirty. - const status = git(['status', '--porcelain'], worktree.path) + const status = gitText( + git, + ['status', '--porcelain=v1', '--untracked-files=all'], + worktree.path, + ) if (status.length > 0) { - git(['add', '-A'], worktree.path) - git(['commit', '-m', summary], worktree.path) + gitText(git, ['add', '-A'], worktree.path) + gitText(git, ['commit', '-m', summary], worktree.path) } - return { + const candidateCommit = resolveCommit(git, worktree.path, 'HEAD') + const candidateTree = gitText( + git, + ['rev-parse', '--verify', `${candidateCommit}^{tree}`], + worktree.path, + ) + const patch = patchBytes(git, worktree.path, worktree.baseCommit, candidateCommit) + const surface: CodeSurface = { kind: 'code', worktreeRef: worktree.path, baseRef: worktree.baseRef, + baseCommit: worktree.baseCommit, + baseTree: worktree.baseTree, + candidateCommit, + candidateTree, + patch: { + format: 'git-diff-binary', + sha256: sha256(patch), + byteLength: patch.byteLength, + }, summary, } + verifyCodeSurfaceWithGit(surface, worktree.path, git) + return surface }, async discard(worktree) { // Remove the worktree, then delete its branch. Force-remove because the // worktree may hold uncommitted experiment state we're discarding. - git(['worktree', 'remove', '--force', worktree.path], opts.repoRoot) - git(['branch', '-D', worktree.branch], opts.repoRoot) + gitText(git, ['worktree', 'remove', '--force', worktree.path], opts.repoRoot) + gitText(git, ['branch', '-D', worktree.branch], opts.repoRoot) }, } } -/** Resolve a `CodeSurface`'s worktreeRef to a directory the measurement can - * run the worker in. A path ref is returned as-is; anything else is treated - * as a ref under the adapter's worktree dir. */ +/** Verify a finalized code surface against its current checkout. This rejects + * dirty/ignored files, moved refs, missing Git objects, and byte mismatches. */ +export function verifyCodeSurface( + surface: CodeSurface, + worktreeDir?: string, +): CodeSurfaceVerification { + const path = unresolvedWorktreePath(surface, worktreeDir) + return verifyCodeSurfaceWithGit(surface, path, defaultGit) +} + +/** Resolve a code candidate for evaluation only after verifying its immutable + * identity against the checkout at `worktreeRef`. */ export function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string { - if (isAbsolute(surface.worktreeRef) && existsSync(surface.worktreeRef)) return surface.worktreeRef - if (worktreeDir) return join(worktreeDir, basename(surface.worktreeRef)) - return surface.worktreeRef + return verifyCodeSurface(surface, worktreeDir).path } diff --git a/tests/campaign/policy-edit-proposer.test.ts b/tests/campaign/policy-edit-proposer.test.ts index 65c4e848..8089819c 100644 --- a/tests/campaign/policy-edit-proposer.test.ts +++ b/tests/campaign/policy-edit-proposer.test.ts @@ -4,6 +4,19 @@ import { makeFinding } from '../../src/analyst/types' import { policyEditProposer } from '../../src/campaign/proposers/policy-edit' import type { CodeSurface, ProposeContext } from '../../src/campaign/types' +const CODE_IDENTITY = { + baseRef: 'main', + baseCommit: 'a'.repeat(40), + baseTree: 'b'.repeat(40), + candidateCommit: 'c'.repeat(40), + candidateTree: 'd'.repeat(40), + patch: { + format: 'git-diff-binary' as const, + sha256: `sha256:${'e'.repeat(64)}` as const, + byteLength: 123, + }, +} + function ctx( findings: unknown[], currentSurface: ProposeContext['currentSurface'] = 'Base prompt.', @@ -175,6 +188,7 @@ describe('policyEditProposer', () => { const codeSurface: CodeSurface = { kind: 'code', worktreeRef: '/tmp/policy-edit-candidate', + ...CODE_IDENTITY, summary: 'old summary', } const codeEdit = makePolicyEdit({ @@ -197,10 +211,33 @@ describe('policyEditProposer', () => { expect(out[0]!.surface).toEqual({ kind: 'code', worktreeRef: '/tmp/policy-edit-candidate', + ...CODE_IDENTITY, summary: 'updated summary', }) }) + it('rejects a path-only CodeSurface instead of preserving mutable identity', async () => { + const codeEdit = makePolicyEdit({ + axis: 'agent_profile', + target: { surface: 'runtime-config', path: 'summary' }, + change: { kind: 'json', mode: 'set', path: 'summary', value: 'updated' }, + claim: 'Candidate code surface must already be finalized.', + expectedGain: { metric: 'holdout.composite', direction: 'increase', amount: 0.04 }, + confidence: 0.85, + risk: 'low', + source: { + findingIds: ['f_code_surface'], + analystIds: ['trace-analyst'], + evidenceRefs: [{ kind: 'artifact', uri: 'file:///tmp/policy-edit-candidate' }], + }, + }) + const proposer = policyEditProposer() + + await expect( + proposer.propose(ctx([codeEdit], { kind: 'code', worktreeRef: '/tmp/path-only' } as never)), + ).rejects.toThrow(/baseRef/) + }) + it('fails loud when judge-derived findings try to steer proposals', async () => { const judgeFinding = makeFinding({ analyst_id: 'judge', diff --git a/tests/campaign/presets.test.ts b/tests/campaign/presets.test.ts index fd080065..928cfbbd 100644 --- a/tests/campaign/presets.test.ts +++ b/tests/campaign/presets.test.ts @@ -28,6 +28,24 @@ import { surfaceContentHash, surfaceHash, } from '../../src/campaign/index' + +function fakeCodeSurface(worktreeRef: string, identityDigit: string): CodeSurface { + return { + kind: 'code', + worktreeRef, + baseRef: 'main', + baseCommit: 'a'.repeat(40), + baseTree: 'b'.repeat(40), + candidateCommit: identityDigit.repeat(40), + candidateTree: identityDigit.repeat(40), + patch: { + format: 'git-diff-binary', + sha256: `sha256:${identityDigit.repeat(64)}`, + byteLength: 1, + }, + } +} + import type { RunRecord } from '../../src/run-record' interface FakeScenario extends Scenario { @@ -1160,17 +1178,14 @@ describe('MutableSurface widening', () => { expect(surfaceHash('hello')).not.toBe(surfaceHash('world')) }) - it('surfaceHash distinguishes code surfaces by worktree + base ref', () => { - const a: CodeSurface = { kind: 'code', worktreeRef: '/wt/a', baseRef: 'main' } - const b: CodeSurface = { kind: 'code', worktreeRef: '/wt/b', baseRef: 'main' } - const aAgain: CodeSurface = { - kind: 'code', - worktreeRef: '/wt/a', - baseRef: 'main', - summary: 'ignored in hash', - } + it('surfaceHash identifies code content independently of its worktree path', () => { + const a = fakeCodeSurface('/wt/a', '1') + const sameBytesElsewhere = fakeCodeSurface('/wt/b', '1') + const differentBytes = fakeCodeSurface('/wt/c', '2') + const aAgain: CodeSurface = { ...fakeCodeSurface('/wt/a', '1'), summary: 'ignored in hash' } expect(surfaceHash(a)).toBe(surfaceHash(aAgain)) // summary not part of identity - expect(surfaceHash(a)).not.toBe(surfaceHash(b)) + expect(surfaceHash(a)).toBe(surfaceHash(sameBytesElsewhere)) + expect(surfaceHash(a)).not.toBe(surfaceHash(differentBytes)) expect(surfaceHash(a)).toMatch(/^[a-f0-9]{16}$/) }) @@ -1188,9 +1203,10 @@ describe('MutableSurface widening', () => { }) { return new Array(populationSize).fill(0).map( (_, i): MutableSurface => ({ - kind: 'code', - worktreeRef: `/wt/gen${generation}-cand${i}`, - baseRef: 'main', + ...fakeCodeSurface( + `/wt/gen${generation}-cand${i}`, + (generation * populationSize + i + 1).toString(16), + ), summary: `candidate ${i}`, }), ) @@ -1203,7 +1219,7 @@ describe('MutableSurface widening', () => { const result = await runOptimization({ scenarios: SCENARIOS, - baselineSurface: { kind: 'code', worktreeRef: '/wt/main', baseRef: 'main' }, + baselineSurface: fakeCodeSurface('/wt/main', 'f'), dispatchWithSurface, proposer: codeProposer, populationSize: 2, diff --git a/tests/campaign/skill-opt.test.ts b/tests/campaign/skill-opt.test.ts index a27a6afb..02d90933 100644 --- a/tests/campaign/skill-opt.test.ts +++ b/tests/campaign/skill-opt.test.ts @@ -217,7 +217,7 @@ describe('skillOptProposer', () => { }) await expect( proposer.propose({ - currentSurface: { kind: 'code', worktreeRef: '/wt/a' }, + currentSurface: { kind: 'code', worktreeRef: '/wt/a' } as never, history: [], findings: [], populationSize: 1, diff --git a/tests/campaign/worktree.test.ts b/tests/campaign/worktree.test.ts index 4a724dba..90cc64fa 100644 --- a/tests/campaign/worktree.test.ts +++ b/tests/campaign/worktree.test.ts @@ -1,9 +1,17 @@ import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { gitWorktreeAdapter, resolveWorktreePath } from '../../src/campaign/worktree' +import { surfaceHash } from '../../src/campaign/presets/run-optimization' +import { surfaceContentHash } from '../../src/campaign/provenance' +import { + gitWorktreeAdapter, + resolveWorktreePath, + verifyCodeSurface, + WorktreeAdapterError, +} from '../../src/campaign/worktree' function git(args: string[], cwd: string): string { return execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() @@ -33,6 +41,8 @@ describe('gitWorktreeAdapter — real git worktrees', () => { expect(wt.branch).toMatch(/^improve\/tighten-the-rubric-/) expect(wt.baseRef).toBe('main') + expect(wt.baseCommit).toBe(git(['rev-parse', 'main'], repoRoot)) + expect(wt.baseTree).toBe(git(['rev-parse', 'main^{tree}'], repoRoot)) // The worktree is a real checkout with the baseline content. expect(git(['rev-parse', '--abbrev-ref', 'HEAD'], wt.path)).toBe(wt.branch) const worktrees = git(['worktree', 'list'], repoRoot) @@ -50,6 +60,36 @@ describe('gitWorktreeAdapter — real git worktrees', () => { expect(surface.kind).toBe('code') expect(surface.worktreeRef).toBe(wt.path) expect(surface.baseRef).toBe('main') + expect(surface.baseCommit).toBe(git(['rev-parse', 'main'], repoRoot)) + expect(surface.baseTree).toBe(git(['rev-parse', 'main^{tree}'], repoRoot)) + expect(surface.candidateCommit).toBe(git(['rev-parse', 'HEAD'], wt.path)) + expect(surface.candidateTree).toBe(git(['rev-parse', 'HEAD^{tree}'], wt.path)) + const patch = execFileSync( + 'git', + [ + 'diff', + '--binary', + '--full-index', + '--no-color', + '--no-ext-diff', + '--no-textconv', + '--no-renames', + '--diff-algorithm=myers', + '--no-indent-heuristic', + '--src-prefix=a/', + '--dst-prefix=b/', + surface.baseCommit, + surface.candidateCommit, + '--', + ], + { cwd: wt.path }, + ) + expect(surface.patch).toEqual({ + format: 'git-diff-binary', + sha256: `sha256:${createHash('sha256').update(patch).digest('hex')}`, + byteLength: patch.byteLength, + }) + expect(Buffer.from(verifyCodeSurface(surface).patchBytes)).toEqual(patch) expect(surface.summary).toBe('improve the prompt') // The change is committed on the branch (one commit beyond base). const log = git(['log', '--oneline', 'main..HEAD'], wt.path) @@ -61,9 +101,186 @@ describe('gitWorktreeAdapter — real git worktrees', () => { it('finalize on a clean worktree makes no empty commit', async () => { const adapter = gitWorktreeAdapter({ repoRoot }) const wt = await adapter.create({ baseRef: 'main', label: 'noop' }) - await adapter.finalize(wt, 'no changes') + const surface = await adapter.finalize(wt, 'no changes') // No commits beyond base — git refuses empty commits and we don't force one. expect(git(['log', '--oneline', 'main..HEAD'], wt.path)).toBe('') + expect(surface.candidateCommit).toBe(surface.baseCommit) + expect(surface.patch.byteLength).toBe(0) + expect(surface.patch.sha256).toBe( + `sha256:${createHash('sha256').update(Buffer.alloc(0)).digest('hex')}`, + ) + expect(verifyCodeSurface(surface).path).toBe(wt.path) + }) + + it('hashes candidate bytes, not worktree paths or commit metadata', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const a = await adapter.create({ baseRef: 'main', label: 'same-a' }) + const b = await adapter.create({ baseRef: 'main', label: 'same-b' }) + writeFileSync(join(a.path, 'prompt.txt'), 'identical candidate bytes\n') + writeFileSync(join(b.path, 'prompt.txt'), 'identical candidate bytes\n') + + const surfaceA = await adapter.finalize(a, 'candidate message A') + const surfaceB = await adapter.finalize(b, 'candidate message B') + + expect(surfaceA.worktreeRef).not.toBe(surfaceB.worktreeRef) + expect(surfaceA.candidateCommit).not.toBe(surfaceB.candidateCommit) + expect(surfaceA.patch).toEqual(surfaceB.patch) + expect(surfaceA.candidateTree).toBe(surfaceB.candidateTree) + expect(surfaceContentHash(surfaceA)).toBe(surfaceContentHash(surfaceB)) + expect(surfaceHash(surfaceA)).toBe(surfaceHash(surfaceB)) + }) + + it('changes both content hashes when one candidate byte changes', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const a = await adapter.create({ baseRef: 'main', label: 'bytes-a' }) + const b = await adapter.create({ baseRef: 'main', label: 'bytes-b' }) + writeFileSync(join(a.path, 'prompt.txt'), 'candidate A\n') + writeFileSync(join(b.path, 'prompt.txt'), 'candidate B\n') + + const surfaceA = await adapter.finalize(a, 'candidate') + const surfaceB = await adapter.finalize(b, 'candidate') + + expect(surfaceA.patch.sha256).not.toBe(surfaceB.patch.sha256) + expect(surfaceContentHash(surfaceA)).not.toBe(surfaceContentHash(surfaceB)) + expect(surfaceHash(surfaceA)).not.toBe(surfaceHash(surfaceB)) + }) + + it('rejects tracked mutation after finalization before resolving the path', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'mutated' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'finalized bytes\n') + const surface = await adapter.finalize(wt, 'finalize') + + writeFileSync(join(wt.path, 'prompt.txt'), 'mutated after finalize\n') + + expect(() => verifyCodeSurface(surface)).toThrow(/changed after finalization/) + expect(() => resolveWorktreePath(surface)).toThrow(/changed after finalization/) + }) + + it('rejects assume-unchanged entries that can hide tracked mutations', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'hidden-mutation' }) + const surface = await adapter.finalize(wt, 'no-op') + git(['update-index', '--assume-unchanged', 'prompt.txt'], wt.path) + writeFileSync(join(wt.path, 'prompt.txt'), 'hidden mutation\n') + + expect(() => verifyCodeSurface(surface)).toThrow(/hidden index entries/) + }) + + it('rejects an ignored file instead of evaluating bytes outside Git identity', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'ignored-dirty' }) + writeFileSync(join(wt.path, '.gitignore'), '*.cache\n') + writeFileSync(join(wt.path, 'agent.cache'), 'mutable execution input\n') + + await expect(adapter.finalize(wt, 'add ignore rule')).rejects.toThrow( + /changed after finalization.*agent\.cache/s, + ) + }) + + it('rejects a surface pointed at the wrong finalized ref', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const a = await adapter.create({ baseRef: 'main', label: 'wrong-ref-a' }) + const b = await adapter.create({ baseRef: 'main', label: 'wrong-ref-b' }) + writeFileSync(join(a.path, 'prompt.txt'), 'candidate A\n') + writeFileSync(join(b.path, 'prompt.txt'), 'candidate B\n') + const surfaceA = await adapter.finalize(a, 'candidate A') + await adapter.finalize(b, 'candidate B') + + expect(() => verifyCodeSurface({ ...surfaceA, worktreeRef: b.path })).toThrow( + /candidate commit mismatch/, + ) + }) + + it('rejects a forged patch digest even when the checkout is unchanged', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'forged-digest' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate\n') + const surface = await adapter.finalize(wt, 'candidate') + + expect(() => + verifyCodeSurface({ + ...surface, + patch: { ...surface.patch, sha256: `sha256:${'0'.repeat(64)}` }, + }), + ).toThrow(/patch mismatch/) + }) + + it('rejects a surface locator pointed at another repository', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'right-repo' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate\n') + const surface = await adapter.finalize(wt, 'candidate') + + const otherRepo = mkdtempSync(join(tmpdir(), 'wt-wrong-repo-')) + try { + git(['init', '-q', '-b', 'main'], otherRepo) + git(['config', 'user.email', 'test@test.dev'], otherRepo) + git(['config', 'user.name', 'Test'], otherRepo) + const emptyHooks = join(otherRepo, '.empty-hooks') + mkdirSync(emptyHooks) + git(['config', 'core.hooksPath', emptyHooks], otherRepo) + writeFileSync(join(otherRepo, 'prompt.txt'), 'unrelated\n') + git(['add', '-A'], otherRepo) + git(['commit', '-q', '-m', 'unrelated'], otherRepo) + + expect(() => verifyCodeSurface({ ...surface, worktreeRef: otherRepo })).toThrow( + /candidate commit mismatch/, + ) + } finally { + rmSync(otherRepo, { recursive: true, force: true }) + } + }) + + it('does not let inherited Git environment redirect verification', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'env-right-repo' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate\n') + const surface = await adapter.finalize(wt, 'candidate') + + const otherRepo = mkdtempSync(join(tmpdir(), 'wt-env-wrong-repo-')) + const priorGitDir = process.env.GIT_DIR + const priorGitWorkTree = process.env.GIT_WORK_TREE + try { + git(['init', '-q', '-b', 'main'], otherRepo) + git(['config', 'user.email', 'test@test.dev'], otherRepo) + git(['config', 'user.name', 'Test'], otherRepo) + const emptyHooks = join(otherRepo, '.empty-hooks') + mkdirSync(emptyHooks) + git(['config', 'core.hooksPath', emptyHooks], otherRepo) + writeFileSync(join(otherRepo, 'prompt.txt'), 'unrelated\n') + git(['add', '-A'], otherRepo) + git(['commit', '-q', '-m', 'unrelated'], otherRepo) + + process.env.GIT_DIR = git(['rev-parse', '--absolute-git-dir'], wt.path) + process.env.GIT_WORK_TREE = wt.path + + expect(() => verifyCodeSurface({ ...surface, worktreeRef: otherRepo })).toThrow( + /candidate commit mismatch/, + ) + } finally { + if (priorGitDir === undefined) delete process.env.GIT_DIR + else process.env.GIT_DIR = priorGitDir + if (priorGitWorkTree === undefined) delete process.env.GIT_WORK_TREE + else process.env.GIT_WORK_TREE = priorGitWorkTree + rmSync(otherRepo, { recursive: true, force: true }) + } + }) + + it('keeps the exact fork commit when the human base ref advances', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'moving-base' }) + const originalBase = wt.baseCommit + + writeFileSync(join(repoRoot, 'later.txt'), 'main advanced\n') + git(['add', 'later.txt'], repoRoot) + git(['commit', '-q', '-m', 'advance main'], repoRoot) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate from original base\n') + + const surface = await adapter.finalize(wt, 'candidate') + expect(surface.baseCommit).toBe(originalBase) + expect(surface.baseCommit).not.toBe(git(['rev-parse', 'main'], repoRoot)) + expect(verifyCodeSurface(surface).path).toBe(wt.path) }) it('discard removes the worktree and its branch', async () => { @@ -95,18 +312,16 @@ describe('gitWorktreeAdapter — real git worktrees', () => { it('surfaces a typed error when git fails', async () => { const adapter = gitWorktreeAdapter({ repoRoot }) await expect(adapter.create({ baseRef: 'no-such-ref', label: 'x' })).rejects.toThrow( - /git worktree add .* failed/, + WorktreeAdapterError, ) }) }) describe('resolveWorktreePath', () => { - it('returns an existing absolute path as-is', () => { - const p = mkdtempSync(join(tmpdir(), 'wt-resolve-')) - try { - expect(resolveWorktreePath({ kind: 'code', worktreeRef: p })).toBe(p) - } finally { - rmSync(p, { recursive: true, force: true }) - } + it('returns a verified absolute worktree path', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'resolve' }) + const surface = await adapter.finalize(wt, 'no-op') + expect(resolveWorktreePath(surface)).toBe(wt.path) }) }) From c284c942fe37dc6b84598c2f6b7f9ac7c4941665 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 9 Jul 2026 17:38:24 -0600 Subject: [PATCH 02/10] chore(audit): remove uncommitted provenance record --- .../2026-07-09T22-27-32Z/findings.jsonl | 1 - .../2026-07-09T22-27-32Z/manifest.json | 32 ----------------- .../2026-07-09T22-27-32Z/summary.md | 34 ------------------- .evolve/skill-runs.jsonl | 1 - 4 files changed, 68 deletions(-) delete mode 100644 .evolve/critical-audit/2026-07-09T22-27-32Z/findings.jsonl delete mode 100644 .evolve/critical-audit/2026-07-09T22-27-32Z/manifest.json delete mode 100644 .evolve/critical-audit/2026-07-09T22-27-32Z/summary.md diff --git a/.evolve/critical-audit/2026-07-09T22-27-32Z/findings.jsonl b/.evolve/critical-audit/2026-07-09T22-27-32Z/findings.jsonl deleted file mode 100644 index 8b137891..00000000 --- a/.evolve/critical-audit/2026-07-09T22-27-32Z/findings.jsonl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/.evolve/critical-audit/2026-07-09T22-27-32Z/manifest.json b/.evolve/critical-audit/2026-07-09T22-27-32Z/manifest.json deleted file mode 100644 index f4572e75..00000000 --- a/.evolve/critical-audit/2026-07-09T22-27-32Z/manifest.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "scope": "uncommitted diff for immutable CodeSurface identity", - "base": "39e8002748f43aa9b440805e7c73000d8aba1a19", - "head": "39e8002748f43aa9b440805e7c73000d8aba1a19", - "project_type": "typescript-node", - "flags": ["diff-only"], - "files": [ - ".claude/skills/agent-eval/SKILL.md", - "CHANGELOG.md", - "docs/design/loop-taxonomy.md", - "docs/improvement-glossary.md", - "scripts/verify-package-exports.mjs", - "src/campaign/analyst-surface.test.ts", - "src/campaign/index.ts", - "src/campaign/presets/run-optimization.ts", - "src/campaign/proposers/policy-edit.ts", - "src/campaign/provenance.ts", - "src/campaign/surface-identity.ts", - "src/campaign/types.ts", - "src/campaign/worktree/index.ts", - "tests/campaign/policy-edit-proposer.test.ts", - "tests/campaign/presets.test.ts", - "tests/campaign/skill-opt.test.ts", - "tests/campaign/worktree.test.ts" - ], - "findings_count_by_severity": { - "critical": 0, - "high": 0, - "medium": 0, - "low": 0 - } -} diff --git a/.evolve/critical-audit/2026-07-09T22-27-32Z/summary.md b/.evolve/critical-audit/2026-07-09T22-27-32Z/summary.md deleted file mode 100644 index a871bfed..00000000 --- a/.evolve/critical-audit/2026-07-09T22-27-32Z/summary.md +++ /dev/null @@ -1,34 +0,0 @@ -# Immutable CodeSurface critical audit - -## Findings - -No remaining correctness, security, architecture, or public-contract findings in the scoped diff. - -The pre-audit hardening pass found one real repository-redirection path through inherited `GIT_DIR`, `GIT_WORK_TREE`, and `GIT_CONFIG_*` variables. -The implementation now removes repository-routing Git variables and disables filesystem-monitor and untracked-cache shortcuts during verification; the real-Git adversarial test proves a wrong locator still fails while those variables point at the expected repository. - -## Open assumptions and residual risk - -- `verifyCodeSurface` is a point-in-time identity check, not a filesystem lease. - Sealed evaluation must consume the returned patch bytes in an isolated checkout, verify the resulting tree, and avoid executing the mutable proposal worktree. -- Candidate symlink and path-containment policy belongs to that materializer. - This package proves Git content identity; it does not declare an arbitrary repository safe to execute. -- Binary-patch bytes are capped at 256 MiB and generated with pinned diff options. - Larger candidates fail instead of weakening verification. - -## Review scores - -- Correctness and security: 8/10. -- Architecture and maintainability: 8/10. -- Standards and real-system coverage: 8/10. -- Overall: 8/10. - -## Checks - -- `pnpm typecheck` -- `pnpm lint` (four pre-existing warnings, zero errors) -- `pnpm test -- --reporter=dot` -- `pnpm build` -- `pnpm verify:package` - -APPROVE — content identity is path-independent, byte-verifiable, fail-closed, and covered by real Git repositories; the sealed materializer remains the required consumer-side boundary. diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index 20cc752d..1c9e0c9b 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -19,4 +19,3 @@ {"skill":"critical-audit","ts":"2026-07-09T22:35:41Z","project":"agent-eval","target":"clustered paired binary inference at abe1d89a11fe1b40b75d0e4d96ccf68dfa2403b1","verdict":"APPROVE","score":"8/10","findings":"1 low accepted risk: independent-package parity vector remains before paper use","files":["src/clustered-paired-binary.ts","src/statistics.ts","src/index.ts","tests/clustered-paired-binary.test.ts",".claude/skills/agent-eval/SKILL.md"]} {"skill":"agent-eval","ts":"2026-07-09T22:27:32Z","project":"agent-eval","target":"immutable CodeSurface identity","outcome":"implemented content-addressed Git identity and fail-loud verification","branch":"feat/immutable-code-surface","model_runs":0} {"skill":"harden","ts":"2026-07-09T22:27:32Z","project":"agent-eval","target":"CodeSurface Git trust boundary","outcome":"fixed inherited Git environment repository-redirection bypass; added real-Git adversarial coverage","branch":"feat/immutable-code-surface","model_runs":0} -{"skill":"critical-audit","ts":"2026-07-09T22:27:32Z","project":"agent-eval","target":"uncommitted immutable CodeSurface diff","verdict":"APPROVE","score":"8/10","findings":"0 remaining; point-in-time checkout verification documented as residual boundary","files":["src/campaign/surface-identity.ts","src/campaign/types.ts","src/campaign/worktree/index.ts","tests/campaign/worktree.test.ts"]} From d70a91f2c7314ee1f76ec71f47705db6d8387c14 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 10 Jul 2026 01:24:17 -0600 Subject: [PATCH 03/10] fix(campaign): verify raw code surface bytes --- CHANGELOG.md | 2 +- docs/design/loop-taxonomy.md | 4 +- src/campaign/worktree/index.ts | 334 ++++++++++++++++++++++++++++---- tests/campaign/worktree.test.ts | 126 +++++++++++- 4 files changed, 426 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8ca3425..cd889009 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ### Changed - `CodeSurface` is now a finalized, content-addressed code candidate. `gitWorktreeAdapter.finalize()` records exact base/candidate commits, the final tree, and the SHA-256 + byte length of the raw binary Git patch; `surfaceHash` and `surfaceContentHash` no longer use filesystem paths. -- `resolveWorktreePath()` now verifies the candidate before returning its checkout. Dirty or ignored files, moved refs, missing objects, wrong trees, and patch-byte mismatches fail instead of being evaluated under stale identity. Use `verifyCodeSurface()` when a verification receipt is needed directly. +- `resolveWorktreePath()` now verifies the candidate before returning its checkout. Dirty or ignored files, moved refs, missing objects, wrong trees, raw byte or executable-mode mismatches, external symlinks, submodules, and patch-byte mismatches fail instead of being evaluated under stale identity. Raw file hashing bypasses Git clean/smudge filters so repository configuration cannot hide different executable bytes. Use `verifyCodeSurface()` when a verification receipt is needed directly. This intentionally rejects path-only `CodeSurface` objects; callers must finalize code candidates through a worktree adapter. diff --git a/docs/design/loop-taxonomy.md b/docs/design/loop-taxonomy.md index 2374ee45..ec6eb74e 100644 --- a/docs/design/loop-taxonomy.md +++ b/docs/design/loop-taxonomy.md @@ -166,7 +166,9 @@ for tiers 1–2, and a finalized `CodeSurface` for tier 4. A code surface's worktree path is only its locator; exact base/candidate commits, final tree, and binary-patch digest are its portable identity. Call `verifyCodeSurface` before executing the checkout so a moved ref or post-finalization mutation -fails before measurement. +fails before measurement. Verification hashes raw files and executable modes +without Git filters and rejects external symlinks or submodules whose bytes are +not represented by the candidate tree. | Tier | Surface | Generator that changes it | Blast radius | |---|---|---|---| diff --git a/src/campaign/worktree/index.ts b/src/campaign/worktree/index.ts index cb8c2f42..c7a65df6 100644 --- a/src/campaign/worktree/index.ts +++ b/src/campaign/worktree/index.ts @@ -12,12 +12,23 @@ import { execFileSync } from 'node:child_process' import { createHash } from 'node:crypto' -import { existsSync } from 'node:fs' -import { basename, isAbsolute, join } from 'node:path' +import { + closeSync, + existsSync, + constants as fsConstants, + fstatSync, + lstatSync, + openSync, + readlinkSync, + readSync, + realpathSync, +} from 'node:fs' +import { basename, isAbsolute, join, relative, resolve, sep } from 'node:path' import { assertCodeSurfaceIdentity, surfaceContentHash } from '../surface-identity' import type { CodeSurface } from '../types' const MAX_GIT_OUTPUT_BYTES = 256 * 1024 * 1024 +const FILE_HASH_CHUNK_BYTES = 1024 * 1024 type GitOutput = string | Uint8Array type GitRunner = (args: string[], cwd: string) => GitOutput @@ -88,6 +99,7 @@ function defaultGit(args: string[], cwd: string): Uint8Array { delete env[key] } } + env.GIT_NO_REPLACE_OBJECTS = '1' return execFileSync('git', args, { cwd, env, maxBuffer: MAX_GIT_OUTPUT_BYTES }) } catch (err) { const stderr = @@ -149,6 +161,250 @@ function unresolvedWorktreePath(surface: CodeSurface, worktreeDir?: string): str return surface.worktreeRef } +interface GitTreeEntry { + mode: string + objectId: string + path: string +} + +function displayGitPath(path: string): string { + return JSON.stringify(path) +} + +function parseGitTreeEntries(bytes: Uint8Array): GitTreeEntry[] { + const input = Buffer.from(bytes) + const entries: GitTreeEntry[] = [] + let start = 0 + while (start < input.length) { + const end = input.indexOf(0, start) + if (end < 0) throw new WorktreeAdapterError('Git tree output was not NUL-terminated') + const record = input.subarray(start, end) + start = end + 1 + if (record.length === 0) continue + + const tab = record.indexOf(0x09) + if (tab < 0) throw new WorktreeAdapterError('Git tree entry did not contain a path') + const metadata = record.subarray(0, tab).toString('ascii').split(' ') + if (metadata.length !== 3) throw new WorktreeAdapterError('Git tree entry was malformed') + const mode = metadata[0] + const type = metadata[1] + const objectId = metadata[2] + if (!mode || !type || !objectId) { + throw new WorktreeAdapterError('Git tree entry was malformed') + } + const pathBytes = record.subarray(tab + 1) + const path = pathBytes.toString('utf8') + if (!Buffer.from(path, 'utf8').equals(pathBytes)) { + throw new WorktreeAdapterError('CodeSurface paths must be valid UTF-8') + } + if (type !== 'blob' && type !== 'commit') { + throw new WorktreeAdapterError( + `CodeSurface contains unsupported Git object type ${type} at ${displayGitPath(path)}`, + ) + } + entries.push({ mode, objectId, path }) + } + return entries +} + +function parseGitPathList(bytes: Uint8Array): string[] { + const input = Buffer.from(bytes) + const paths: string[] = [] + let start = 0 + while (start < input.length) { + const end = input.indexOf(0, start) + if (end < 0) throw new WorktreeAdapterError('Git path output was not NUL-terminated') + const pathBytes = input.subarray(start, end) + start = end + 1 + if (pathBytes.length === 0) continue + const path = pathBytes.toString('utf8') + if (!Buffer.from(path, 'utf8').equals(pathBytes)) { + throw new WorktreeAdapterError('CodeSurface paths must be valid UTF-8') + } + paths.push(path) + } + return paths +} + +function isWithinRoot(root: string, candidate: string): boolean { + const rel = relative(root, candidate) + return rel === '' || (!isAbsolute(rel) && rel !== '..' && !rel.startsWith(`..${sep}`)) +} + +function assertSafeRelativePath(root: string, path: string): string { + if (path.length === 0 || isAbsolute(path)) { + throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`) + } + const absolutePath = resolve(root, path) + if (!isWithinRoot(root, absolutePath) || absolutePath === root) { + throw new WorktreeAdapterError(`CodeSurface path escapes its worktree: ${displayGitPath(path)}`) + } + + const segments = path.split(/[\\/]/u) + let parent = root + for (const segment of segments.slice(0, -1)) { + if (segment.length === 0 || segment === '.' || segment === '..') { + throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`) + } + parent = join(parent, segment) + const stat = lstatSync(parent) + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new WorktreeAdapterError( + `CodeSurface path traverses a non-directory or symbolic link: ${displayGitPath(path)}`, + ) + } + } + return absolutePath +} + +function gitObjectHashAlgorithm(objectId: string): 'sha1' | 'sha256' { + if (/^[a-f0-9]{40}$/.test(objectId)) return 'sha1' + if (/^[a-f0-9]{64}$/.test(objectId)) return 'sha256' + throw new WorktreeAdapterError(`Unsupported Git object id: ${objectId}`) +} + +function hashGitBlobBytes(bytes: Uint8Array, objectId: string): string { + const hash = createHash(gitObjectHashAlgorithm(objectId)) + hash.update(`blob ${bytes.byteLength}\0`) + hash.update(bytes) + return hash.digest('hex') +} + +function hashGitBlobFile(path: string, objectId: string): { hash: string; executable: boolean } { + const before = lstatSync(path) + if (!before.isFile() || before.isSymbolicLink()) { + throw new WorktreeAdapterError(`CodeSurface expected a regular file at ${displayGitPath(path)}`) + } + + const noFollow = process.platform === 'win32' ? 0 : fsConstants.O_NOFOLLOW + const fd = openSync(path, fsConstants.O_RDONLY | noFollow) + try { + const opened = fstatSync(fd) + if ( + !opened.isFile() || + opened.dev !== before.dev || + opened.ino !== before.ino || + opened.size !== before.size + ) { + throw new WorktreeAdapterError( + `CodeSurface file changed while it was being verified: ${displayGitPath(path)}`, + ) + } + + const hash = createHash(gitObjectHashAlgorithm(objectId)) + hash.update(`blob ${opened.size}\0`) + const chunk = Buffer.allocUnsafe(FILE_HASH_CHUNK_BYTES) + let total = 0 + while (true) { + const count = readSync(fd, chunk, 0, chunk.length, null) + if (count === 0) break + total += count + hash.update(chunk.subarray(0, count)) + } + if (total !== opened.size) { + throw new WorktreeAdapterError( + `CodeSurface file changed while it was being verified: ${displayGitPath(path)}`, + ) + } + return { hash: hash.digest('hex'), executable: (opened.mode & 0o111) !== 0 } + } finally { + closeSync(fd) + } +} + +function assertSymlinkTargetIsBound( + root: string, + linkPath: string, + trackedPaths: ReadonlySet, +): void { + const targetBytes = readlinkSync(linkPath, { encoding: 'buffer' }) + const target = targetBytes.toString('utf8') + if (!Buffer.from(target, 'utf8').equals(targetBytes)) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link has an unsafe target at ${displayGitPath(linkPath)}`, + ) + } + if (isAbsolute(target)) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`, + ) + } + let resolvedTarget: string + try { + resolvedTarget = realpathSync(linkPath) + } catch (err) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link target is missing or cyclic at ${displayGitPath(linkPath)}`, + err, + ) + } + if (!isWithinRoot(root, resolvedTarget)) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`, + ) + } + + const targetRelative = relative(root, resolvedTarget).split(sep).join('/') + const targetIsTracked = + trackedPaths.has(targetRelative) || + [...trackedPaths].some((trackedPath) => trackedPath.startsWith(`${targetRelative}/`)) + if (!targetIsTracked) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link resolves to untracked content at ${displayGitPath(linkPath)}`, + ) + } +} + +function assertRawTreeMatchesWorktree(git: GitRunner, root: string, candidateCommit: string): void { + const entries = parseGitTreeEntries( + gitBytes(git, ['ls-tree', '-r', '-z', '--full-tree', candidateCommit], root), + ) + const trackedPaths = new Set(entries.map((entry) => entry.path)) + + for (const entry of entries) { + const absolutePath = assertSafeRelativePath(root, entry.path) + if (entry.mode === '160000' || entry.mode === '040000') { + throw new WorktreeAdapterError( + `CodeSurface contains a Git submodule whose executable bytes are not bound: ${displayGitPath(entry.path)}`, + ) + } + if (entry.mode === '120000') { + const stat = lstatSync(absolutePath) + if (!stat.isSymbolicLink()) { + throw new WorktreeAdapterError( + `CodeSurface expected a symbolic link at ${displayGitPath(entry.path)}`, + ) + } + const targetBytes = readlinkSync(absolutePath, { encoding: 'buffer' }) + const actualObjectId = hashGitBlobBytes(targetBytes, entry.objectId) + if (actualObjectId !== entry.objectId) { + throw new WorktreeAdapterError( + `CodeSurface raw symbolic-link bytes differ from the candidate tree at ${displayGitPath(entry.path)}`, + ) + } + assertSymlinkTargetIsBound(root, absolutePath, trackedPaths) + continue + } + if (entry.mode !== '100644' && entry.mode !== '100755') { + throw new WorktreeAdapterError( + `CodeSurface contains unsupported Git mode ${entry.mode} at ${displayGitPath(entry.path)}`, + ) + } + + const actual = hashGitBlobFile(absolutePath, entry.objectId) + if (actual.hash !== entry.objectId) { + throw new WorktreeAdapterError( + `CodeSurface raw tracked bytes differ from the candidate tree because the worktree changed after finalization at ${displayGitPath(entry.path)}`, + ) + } + if (process.platform !== 'win32' && actual.executable !== (entry.mode === '100755')) { + throw new WorktreeAdapterError( + `CodeSurface executable mode differs from the candidate tree at ${displayGitPath(entry.path)}`, + ) + } + } +} + export interface CodeSurfaceVerification { /** Verified worktree path. */ path: string @@ -170,6 +426,23 @@ function verifyCodeSurfaceWithGit( if (!existsSync(path)) { throw new WorktreeAdapterError(`CodeSurface worktree does not exist: ${path}`) } + const lexicalRoot = resolve(path) + const canonicalRoot = realpathSync(path) + if ( + lstatSync(lexicalRoot).isSymbolicLink() || + (process.platform !== 'win32' && lexicalRoot !== canonicalRoot) + ) { + throw new WorktreeAdapterError( + `CodeSurface worktree locator must not contain a symbolic link: ${path}`, + ) + } + const repoRoot = gitText(git, ['rev-parse', '--show-toplevel'], path) + const canonicalRepoRoot = realpathSync(repoRoot) + if (canonicalRepoRoot !== canonicalRoot) { + throw new WorktreeAdapterError( + `CodeSurface worktree locator is not the repository root: expected ${repoRoot}, got ${path}`, + ) + } const indexFlags = gitText(git, ['ls-files', '-v'], path) .split('\n') @@ -180,24 +453,15 @@ function verifyCodeSurfaceWithGit( ) } - const status = gitText( - git, - [ - '-c', - 'core.fsmonitor=false', - '-c', - 'core.untrackedCache=false', - 'status', - '--porcelain=v1', - '--untracked-files=all', - '--ignored=matching', - '--ignore-submodules=none', - ], - path, - ) - if (status.length > 0) { + const extraPaths = [ + ...parseGitPathList(gitBytes(git, ['ls-files', '--others', '--exclude-standard', '-z'], path)), + ...parseGitPathList( + gitBytes(git, ['ls-files', '--others', '--ignored', '--exclude-standard', '-z'], path), + ), + ] + if (extraPaths.length > 0) { throw new WorktreeAdapterError( - `CodeSurface worktree changed after finalization: ${path}\n${status}`, + `CodeSurface worktree changed after finalization: ${path}\n${extraPaths.join('\n')}`, ) } @@ -240,37 +504,34 @@ function verifyCodeSurfaceWithGit( ) } - // Rebuild the index from the frozen commit before comparing it to the - // filesystem. This clears cached stat data, so same-size/same-mtime edits - // cannot hide behind Git's normal status shortcut. - gitText(git, ['read-tree', '--reset', surface.candidateCommit], path) try { gitText( git, [ - '-c', - 'core.filemode=true', - '-c', - 'core.symlinks=true', - '-c', - 'core.fsmonitor=false', - '-c', - 'core.ignorestat=false', - 'diff-files', + 'diff-index', + '--cached', + '--quiet', '--no-ext-diff', '--no-textconv', - '--quiet', + surface.candidateCommit, '--', ], path, ) } catch (err) { throw new WorktreeAdapterError( - `CodeSurface tracked bytes changed after finalization: ${path}`, + `CodeSurface index differs from finalized candidate: ${path}`, err, ) } + try { + assertRawTreeMatchesWorktree(git, canonicalRepoRoot, surface.candidateCommit) + } catch (err) { + if (err instanceof WorktreeAdapterError) throw err + throw new WorktreeAdapterError(`CodeSurface raw tree verification failed: ${path}`, err) + } + const mergeBase = gitText(git, ['merge-base', surface.baseCommit, surface.candidateCommit], path) if (mergeBase !== surface.baseCommit) { throw new WorktreeAdapterError( @@ -288,7 +549,7 @@ function verifyCodeSurfaceWithGit( return { path, - repoRoot: gitText(git, ['rev-parse', '--show-toplevel'], path), + repoRoot: canonicalRepoRoot, contentHash: surfaceContentHash(surface), patchBytes: new Uint8Array(patch), } @@ -376,7 +637,8 @@ export function gitWorktreeAdapter(opts: GitWorktreeAdapterOptions): WorktreeAda } /** Verify a finalized code surface against its current checkout. This rejects - * dirty/ignored files, moved refs, missing Git objects, and byte mismatches. */ + * dirty/ignored files, moved refs, missing Git objects, raw byte/mode + * mismatches, external symlinks, and submodules. */ export function verifyCodeSurface( surface: CodeSurface, worktreeDir?: string, diff --git a/tests/campaign/worktree.test.ts b/tests/campaign/worktree.test.ts index 90cc64fa..bb4b668d 100644 --- a/tests/campaign/worktree.test.ts +++ b/tests/campaign/worktree.test.ts @@ -1,8 +1,16 @@ import { execFileSync } from 'node:child_process' import { createHash } from 'node:crypto' -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { + chmodSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + unlinkSync, + writeFileSync, +} from 'node:fs' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { basename, dirname, join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { surfaceHash } from '../../src/campaign/presets/run-optimization' import { surfaceContentHash } from '../../src/campaign/provenance' @@ -167,6 +175,120 @@ describe('gitWorktreeAdapter — real git worktrees', () => { expect(() => verifyCodeSurface(surface)).toThrow(/hidden index entries/) }) + it('rejects raw tracked bytes hidden by a Git clean filter', async () => { + git(['config', 'filter.hide.clean', "sed 's/^evil$/safe/'"], repoRoot) + git(['config', 'filter.hide.smudge', 'cat'], repoRoot) + writeFileSync(join(repoRoot, '.gitattributes'), 'filtered.txt filter=hide\n') + writeFileSync(join(repoRoot, 'filtered.txt'), 'safe\n') + git(['add', '-A'], repoRoot) + git(['commit', '-q', '-m', 'add filtered file'], repoRoot) + + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'clean-filter-bypass' }) + const surface = await adapter.finalize(wt, 'no-op') + writeFileSync(join(wt.path, 'filtered.txt'), 'evil\n') + + // Git reports the checkout clean because the configured filter maps the + // changed bytes back to the committed blob. Verification must read raw + // filesystem bytes instead of trusting that filtered comparison. + expect(git(['status', '--porcelain=v1'], wt.path)).toBe('') + expect(() => verifyCodeSurface(surface)).toThrow(/raw tracked bytes differ/) + }) + + it('rejects checkout bytes rewritten by a Git smudge filter', async () => { + git(['config', 'filter.rewrite.clean', "sed 's/^evil$/safe/'"], repoRoot) + git(['config', 'filter.rewrite.smudge', "sed 's/^safe$/evil/'"], repoRoot) + writeFileSync(join(repoRoot, '.gitattributes'), 'filtered.txt filter=rewrite\n') + writeFileSync(join(repoRoot, 'filtered.txt'), 'safe\n') + git(['add', '-A'], repoRoot) + git(['commit', '-q', '-m', 'add smudged file'], repoRoot) + + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'smudge-filter-bypass' }) + + expect(git(['status', '--porcelain=v1'], wt.path)).toBe('') + await expect(adapter.finalize(wt, 'no-op')).rejects.toThrow(/raw tracked bytes differ/) + }) + + it('rejects executable-mode changes hidden by core.filemode=false', async () => { + if (process.platform === 'win32') return + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'hidden-mode' }) + const surface = await adapter.finalize(wt, 'no-op') + git(['config', 'core.filemode', 'false'], wt.path) + chmodSync(join(wt.path, 'prompt.txt'), 0o755) + + expect(git(['status', '--porcelain=v1'], wt.path)).toBe('') + expect(() => verifyCodeSurface(surface)).toThrow(/executable mode differs/) + }) + + it('rejects a symbolic worktree locator', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'linked-locator' }) + const surface = await adapter.finalize(wt, 'no-op') + const linkedPath = `${wt.path}-link` + symlinkSync(wt.path, linkedPath, 'dir') + try { + expect(() => verifyCodeSurface({ ...surface, worktreeRef: linkedPath })).toThrow( + /locator must not contain a symbolic link/, + ) + } finally { + unlinkSync(linkedPath) + } + }) + + it('rejects a worktree locator that traverses a symbolic-link parent', async () => { + if (process.platform === 'win32') return + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'linked-parent' }) + const surface = await adapter.finalize(wt, 'no-op') + const linkedParent = `${dirname(wt.path)}-link` + symlinkSync(dirname(wt.path), linkedParent, 'dir') + try { + expect(() => + verifyCodeSurface({ + ...surface, + worktreeRef: join(linkedParent, basename(wt.path)), + }), + ).toThrow(/locator must not contain a symbolic link/) + } finally { + unlinkSync(linkedParent) + } + }) + + it('rejects a tracked symbolic link that escapes the worktree', async () => { + const outside = join(repoRoot, '..', `${basename(repoRoot)}-outside.txt`) + writeFileSync(outside, 'outside bytes\n') + symlinkSync(outside, join(repoRoot, 'outside-link')) + git(['add', 'outside-link'], repoRoot) + git(['commit', '-q', '-m', 'add external link'], repoRoot) + + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'external-link' }) + try { + await expect(adapter.finalize(wt, 'no-op')).rejects.toThrow(/symbolic link escapes/) + } finally { + rmSync(outside, { force: true }) + } + }) + + it('rejects Git submodules whose checked-out bytes are outside the tree identity', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'submodule' }) + const nested = join(wt.path, 'vendor', 'nested') + mkdirSync(nested, { recursive: true }) + git(['init', '-q', '-b', 'main'], nested) + git(['config', 'user.email', 'test@test.dev'], nested) + git(['config', 'user.name', 'Test'], nested) + writeFileSync(join(nested, 'module.txt'), 'module bytes\n') + git(['add', '-A'], nested) + git(['commit', '-q', '-m', 'nested'], nested) + + await expect(adapter.finalize(wt, 'add embedded repository')).rejects.toThrow( + /Git submodule.*not bound/, + ) + }) + it('rejects an ignored file instead of evaluating bytes outside Git identity', async () => { const adapter = gitWorktreeAdapter({ repoRoot }) const wt = await adapter.create({ baseRef: 'main', label: 'ignored-dirty' }) From 95702ea079ef616bb01837aa83cdadeeeecc3733 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 10 Jul 2026 01:35:22 -0600 Subject: [PATCH 04/10] fix(campaign): canonicalize code surface identity --- src/campaign/surface-identity.ts | 6 +++++- src/campaign/worktree/index.ts | 7 ------- tests/campaign/presets.test.ts | 9 +++++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/campaign/surface-identity.ts b/src/campaign/surface-identity.ts index b79b23f6..373fbdfd 100644 --- a/src/campaign/surface-identity.ts +++ b/src/campaign/surface-identity.ts @@ -49,7 +49,11 @@ export function codeSurfaceIdentityMaterial(surface: CodeSurface): string { baseCommit: surface.baseCommit, baseTree: surface.baseTree, candidateTree: surface.candidateTree, - patch: surface.patch, + patch: { + format: surface.patch.format, + sha256: surface.patch.sha256, + byteLength: surface.patch.byteLength, + }, }) } diff --git a/src/campaign/worktree/index.ts b/src/campaign/worktree/index.ts index c7a65df6..d33349df 100644 --- a/src/campaign/worktree/index.ts +++ b/src/campaign/worktree/index.ts @@ -486,13 +486,6 @@ function verifyCodeSurfaceWithGit( ) } - const candidateObject = resolveCommit(git, path, surface.candidateCommit) - if (candidateObject !== surface.candidateCommit) { - throw new WorktreeAdapterError( - `CodeSurface candidate object mismatch: expected ${surface.candidateCommit}, got ${candidateObject}`, - ) - } - const candidateTree = gitText( git, ['rev-parse', '--verify', `${surface.candidateCommit}^{tree}`], diff --git a/tests/campaign/presets.test.ts b/tests/campaign/presets.test.ts index 928cfbbd..9f08eab0 100644 --- a/tests/campaign/presets.test.ts +++ b/tests/campaign/presets.test.ts @@ -1183,8 +1183,17 @@ describe('MutableSurface widening', () => { const sameBytesElsewhere = fakeCodeSurface('/wt/b', '1') const differentBytes = fakeCodeSurface('/wt/c', '2') const aAgain: CodeSurface = { ...fakeCodeSurface('/wt/a', '1'), summary: 'ignored in hash' } + const reorderedPatch: CodeSurface = { + ...a, + patch: { + byteLength: a.patch.byteLength, + sha256: a.patch.sha256, + format: a.patch.format, + }, + } expect(surfaceHash(a)).toBe(surfaceHash(aAgain)) // summary not part of identity expect(surfaceHash(a)).toBe(surfaceHash(sameBytesElsewhere)) + expect(surfaceHash(a)).toBe(surfaceHash(reorderedPatch)) expect(surfaceHash(a)).not.toBe(surfaceHash(differentBytes)) expect(surfaceHash(a)).toMatch(/^[a-f0-9]{16}$/) }) From ecde3882d0831adc94c48e1a54ca36897ae6d3d6 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 10 Jul 2026 01:40:01 -0600 Subject: [PATCH 05/10] fix(campaign): reject external symlink trampolines --- src/campaign/worktree/index.ts | 8 +++++++- tests/campaign/worktree.test.ts | 13 +++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/campaign/worktree/index.ts b/src/campaign/worktree/index.ts index d33349df..bb1d97de 100644 --- a/src/campaign/worktree/index.ts +++ b/src/campaign/worktree/index.ts @@ -23,7 +23,7 @@ import { readSync, realpathSync, } from 'node:fs' -import { basename, isAbsolute, join, relative, resolve, sep } from 'node:path' +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' import { assertCodeSurfaceIdentity, surfaceContentHash } from '../surface-identity' import type { CodeSurface } from '../types' @@ -329,6 +329,12 @@ function assertSymlinkTargetIsBound( `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`, ) } + const lexicalTarget = resolve(dirname(linkPath), target) + if (!isWithinRoot(root, lexicalTarget)) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`, + ) + } let resolvedTarget: string try { resolvedTarget = realpathSync(linkPath) diff --git a/tests/campaign/worktree.test.ts b/tests/campaign/worktree.test.ts index bb4b668d..29c214b1 100644 --- a/tests/campaign/worktree.test.ts +++ b/tests/campaign/worktree.test.ts @@ -272,6 +272,19 @@ describe('gitWorktreeAdapter — real git worktrees', () => { } }) + it('rejects a symbolic link that leaves and re-enters through untracked content', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'external-trampoline' }) + const outside = `${wt.path}-outside` + mkdirSync(outside) + symlinkSync(join(wt.path, 'prompt.txt'), join(outside, 'back')) + symlinkSync(`../${basename(outside)}/back`, join(wt.path, 'trampoline')) + + await expect(adapter.finalize(wt, 'add external trampoline')).rejects.toThrow( + /symbolic link escapes/, + ) + }) + it('rejects Git submodules whose checked-out bytes are outside the tree identity', async () => { const adapter = gitWorktreeAdapter({ repoRoot }) const wt = await adapter.create({ baseRef: 'main', label: 'submodule' }) From 1c0312dd6ad0ec9aff04f269d98aca4262c18aff Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 10 Jul 2026 01:57:30 -0600 Subject: [PATCH 06/10] fix(campaign): unify code surface identity consumers --- scripts/verify-package-exports.mjs | 1 + src/campaign/proposers/composite.test.ts | 36 ++++++++++- src/campaign/proposers/composite.ts | 3 +- src/campaign/proposers/policy-edit.ts | 4 +- src/contract/self-improve.test.ts | 67 ++++++++++++++++++++- src/contract/self-improve.ts | 8 +-- tests/campaign/policy-edit-proposer.test.ts | 9 +-- 7 files changed, 111 insertions(+), 17 deletions(-) diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index b047c974..083b81ed 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -86,6 +86,7 @@ try { 'verifyCodeSurface', 'resolveWorktreePath', 'assertCodeSurfaceIdentity', + 'codeSurfaceIdentityMaterial', 'surfaceHash', 'surfaceContentHash', ]) { diff --git a/src/campaign/proposers/composite.test.ts b/src/campaign/proposers/composite.test.ts index 5b1259ea..05bf4ad4 100644 --- a/src/campaign/proposers/composite.test.ts +++ b/src/campaign/proposers/composite.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import type { ProposeContext, SurfaceProposer } from '../types' +import type { CodeSurface, ProposeContext, SurfaceProposer } from '../types' import { compositeProposer } from './composite' function ctxOf(populationSize: number): ProposeContext { @@ -32,6 +32,23 @@ function stub( } } +function codeSurface(worktreeRef: string): CodeSurface { + return { + kind: 'code', + worktreeRef, + baseRef: 'main', + baseCommit: '1'.repeat(40), + baseTree: '2'.repeat(40), + candidateCommit: '3'.repeat(40), + candidateTree: '4'.repeat(40), + patch: { + format: 'git-diff-binary', + sha256: `sha256:${'5'.repeat(64)}`, + byteLength: 1, + }, + } +} + describe('compositeProposer (N proposers, one generation pool)', () => { it('splits the population budget and merges candidates with member provenance', async () => { const composite = compositeProposer({ @@ -69,6 +86,23 @@ describe('compositeProposer (N proposers, one generation pool)', () => { expect(sameLabel.startsWith('a:')).toBe(true) }) + it('dedupes content-identical code surfaces across different worktree paths', async () => { + const proposer = (kind: string, surface: CodeSurface): SurfaceProposer => ({ + kind, + propose: async () => [{ surface, label: 'v', rationale: 'r' }], + }) + const first = codeSurface('/tmp/candidate-a') + const sameBytesElsewhere = codeSurface('/tmp/candidate-b') + const composite = compositeProposer({ + proposers: [proposer('a', first), proposer('b', sameBytesElsewhere)], + }) + + const pool = await composite.propose(ctxOf(2)) + + expect(pool).toHaveLength(1) + expect((pool[0] as { surface: CodeSurface }).surface.worktreeRef).toBe(first.worktreeRef) + }) + it('isolates a failing member; throws only when ALL members fail', async () => { const oneDown = compositeProposer({ proposers: [stub('boom', [], { fail: true }), stub('ok', ['x1', 'x2'])], diff --git a/src/campaign/proposers/composite.ts b/src/campaign/proposers/composite.ts index c326becd..90a884fb 100644 --- a/src/campaign/proposers/composite.ts +++ b/src/campaign/proposers/composite.ts @@ -31,6 +31,7 @@ * component attribution — see the experiment-optimal research brief. */ +import { surfaceContentHash } from '../surface-identity' import type { GenerationRecord, MutableSurface, @@ -99,7 +100,7 @@ export function compositeProposer( : (proposal as MutableSurface) const label = isCandidate ? (proposal as ProposedCandidate).label : 'candidate' const rationale = isCandidate ? (proposal as ProposedCandidate).rationale : '' - const key = typeof surface === 'string' ? surface : JSON.stringify(surface) + const key = surfaceContentHash(surface) if (seen.has(key)) continue seen.add(key) pool.push({ surface, label: `${member.kind}:${label}`, rationale }) diff --git a/src/campaign/proposers/policy-edit.ts b/src/campaign/proposers/policy-edit.ts index e4f2e140..0eaa52c2 100644 --- a/src/campaign/proposers/policy-edit.ts +++ b/src/campaign/proposers/policy-edit.ts @@ -16,7 +16,7 @@ import { policyEditsFromFindings, } from '../../analyst/policy-edit' import type { AnalystFinding } from '../../analyst/types' -import { assertCodeSurfaceIdentity } from '../surface-identity' +import { assertCodeSurfaceIdentity, surfaceContentHash } from '../surface-identity' import type { MutableSurface, ProposeContext, ProposedCandidate, SurfaceProposer } from '../types' export interface PolicyEditProposerOptions { @@ -118,5 +118,5 @@ function coerceCandidateSurface(surface: unknown): MutableSurface { } function sameSurface(a: MutableSurface, b: MutableSurface): boolean { - return JSON.stringify(a) === JSON.stringify(b) + return surfaceContentHash(a) === surfaceContentHash(b) } diff --git a/src/contract/self-improve.test.ts b/src/contract/self-improve.test.ts index fedf26b3..31556903 100644 --- a/src/contract/self-improve.test.ts +++ b/src/contract/self-improve.test.ts @@ -8,7 +8,8 @@ */ import { describe, expect, it } from 'vitest' -import type { Gate, JudgeConfig, SurfaceProposer } from '../campaign/types' +import { surfaceHash } from '../campaign/surface-identity' +import type { CodeSurface, Gate, JudgeConfig, SurfaceProposer } from '../campaign/types' import type { DispatchContext, Scenario } from './index' import { type SelfImproveProgressEvent, selfImprove } from './self-improve' @@ -37,6 +38,23 @@ async function stubAgent( return { text: String(surface) } } +function codeSurface(worktreeRef: string): CodeSurface { + return { + kind: 'code', + worktreeRef, + baseRef: 'main', + baseCommit: '1'.repeat(40), + baseTree: '2'.repeat(40), + candidateCommit: '3'.repeat(40), + candidateTree: '4'.repeat(40), + patch: { + format: 'git-diff-binary', + sha256: `sha256:${'5'.repeat(64)}`, + byteLength: 1, + }, + } +} + describe('selfImprove — power analysis wiring', () => { it('attaches the MDE analysis to the result and emits power.estimated', async () => { const events: SelfImproveProgressEvent[] = [] @@ -62,6 +80,53 @@ describe('selfImprove — power analysis wiring', () => { }) }) +describe('selfImprove — hosted code-surface identity', () => { + it('uses the content identity in snapshots instead of the mutable worktree path', async () => { + const baseline = codeSurface('/tmp/candidate-a') + const sameBytesElsewhere = codeSurface('/tmp/candidate-b') + const payloads: Array<{ + events?: Array<{ + baseline?: { surfaceHash: string } + generations: Array<{ surfaceHash: string }> + }> + }> = [] + const fetchImpl: typeof fetch = async (_input, init) => { + if (typeof init?.body === 'string') payloads.push(JSON.parse(init.body)) + return new Response(JSON.stringify({ accepted: 1, rejected: [] }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + } + const proposer: SurfaceProposer = { + kind: 'code-candidate', + propose: async () => [sameBytesElsewhere], + } + + await selfImprove({ + agent: stubAgent, + scenarios, + judge, + baselineSurface: baseline, + proposer, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.5 }, + hostedTenant: { + endpoint: 'https://ingest.example', + apiKey: 'test-key', + tenantId: 'test-tenant', + fetchImpl, + }, + }) + + const expected = surfaceHash(baseline) + const event = payloads + .flatMap((payload) => payload.events ?? []) + .find((candidate) => candidate.baseline?.surfaceHash === expected) + expect(event).toBeDefined() + expect(event?.generations[0]?.surfaceHash).toBe(expected) + expect(surfaceHash(sameBytesElsewhere)).toBe(expected) + }) +}) + describe('selfImprove — neutralize (placebo arm) passthrough', () => { it('forwards `neutralize`: runs the placebo arm and hands its scores to the gate', async () => { let neutralizeCalled = false diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index 3be248f2..6270ccee 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -41,6 +41,7 @@ import { fsCampaignStorage, inMemoryCampaignStorage, } from '../campaign/storage' +import { surfaceHash } from '../campaign/surface-identity' import type { CampaignCellResult, DispatchContext, @@ -620,7 +621,7 @@ async function shipEvalRunToHosted( function snapshotFromCampaign( index: number, - surface: MutableSurface | undefined, + surface: MutableSurface, campaign: RunImprovementLoopResult['baselineCampaign'], durationMs: number, ): EvalRunGenerationSnapshot { @@ -644,10 +645,7 @@ async function shipEvalRunToHosted( cells.length === 0 ? 0 : cells.reduce((s, c) => s + c.compositeMean, 0) / cells.length return { index, - surfaceHash: - typeof surface === 'string' - ? hashString(surface) - : hashString(JSON.stringify(surface ?? '')), + surfaceHash: surfaceHash(surface), surface, cells, compositeMean, diff --git a/tests/campaign/policy-edit-proposer.test.ts b/tests/campaign/policy-edit-proposer.test.ts index 8089819c..df29791a 100644 --- a/tests/campaign/policy-edit-proposer.test.ts +++ b/tests/campaign/policy-edit-proposer.test.ts @@ -184,7 +184,7 @@ describe('policyEditProposer', () => { expect(JSON.parse(String(out[0]!.surface))).toEqual({ budget: { maxTurns: 6 } }) }) - it('passes through CodeSurface-shaped candidate surfaces', async () => { + it('skips metadata-only CodeSurface edits that do not change candidate identity', async () => { const codeSurface: CodeSurface = { kind: 'code', worktreeRef: '/tmp/policy-edit-candidate', @@ -208,12 +208,7 @@ describe('policyEditProposer', () => { const proposer = policyEditProposer() const out = await proposer.propose(ctx([codeEdit], codeSurface)) - expect(out[0]!.surface).toEqual({ - kind: 'code', - worktreeRef: '/tmp/policy-edit-candidate', - ...CODE_IDENTITY, - summary: 'updated summary', - }) + expect(out).toEqual([]) }) it('rejects a path-only CodeSurface instead of preserving mutable identity', async () => { From 14ac8d882ee318fde4569d5f6944286aa0c7477f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 10 Jul 2026 15:16:41 -0600 Subject: [PATCH 07/10] fix(campaign): stabilize binary patch identity --- CHANGELOG.md | 19 ++++++++++--- src/campaign/worktree/index.ts | 5 ++++ tests/campaign/worktree.test.ts | 48 +++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd889009..3e1c112f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,21 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ### Changed -- `CodeSurface` is now a finalized, content-addressed code candidate. `gitWorktreeAdapter.finalize()` records exact base/candidate commits, the final tree, and the SHA-256 + byte length of the raw binary Git patch; `surfaceHash` and `surfaceContentHash` no longer use filesystem paths. -- `resolveWorktreePath()` now verifies the candidate before returning its checkout. Dirty or ignored files, moved refs, missing objects, wrong trees, raw byte or executable-mode mismatches, external symlinks, submodules, and patch-byte mismatches fail instead of being evaluated under stale identity. Raw file hashing bypasses Git clean/smudge filters so repository configuration cannot hide different executable bytes. Use `verifyCodeSurface()` when a verification receipt is needed directly. - -This intentionally rejects path-only `CodeSurface` objects; callers must finalize code candidates through a worktree adapter. +- `CodeSurface` is now a finalized, content-addressed code candidate. + `gitWorktreeAdapter.finalize()` records exact base/candidate commits, the final tree, and the SHA-256 + byte length of the raw binary Git patch; `surfaceHash` and `surfaceContentHash` no longer use filesystem paths. + Binary-patch generation pins Git compression at command scope so repository or global config cannot change the digest for identical trees. +- `resolveWorktreePath()` now verifies the candidate before returning its checkout. + Dirty or ignored files, moved refs, missing objects, wrong trees, raw byte or executable-mode mismatches, external symlinks, submodules, and patch-byte mismatches fail instead of being evaluated under stale identity. + Raw file hashing bypasses Git clean/smudge filters so repository configuration cannot hide different executable bytes. + Use `verifyCodeSurface()` when a verification receipt is needed directly. + +### Breaking + +- `resolveWorktreePath()` no longer returns a best-effort, unchecked locator: it verifies the finalized candidate and throws on any identity mismatch. + There is intentionally no lenient fallback on the evaluation path. +- Path-only `CodeSurface` objects are invalid. + Every field in the finalized identity is required. + Downstream callers must migrate to adapter-finalized candidates when adopting 0.112.0. ## [0.111.0] — 2026-07-09 — repository-clustered paired inference diff --git a/src/campaign/worktree/index.ts b/src/campaign/worktree/index.ts index bb1d97de..72e121dd 100644 --- a/src/campaign/worktree/index.ts +++ b/src/campaign/worktree/index.ts @@ -129,6 +129,11 @@ function sha256(bytes: Uint8Array): `sha256:${string}` { } const PATCH_ARGS = [ + // Git binary patches embed a zlib stream. Pin no-compression at command + // scope so repository/global config and compressor heuristics cannot change + // the candidate identity for the same base and final tree. + '-c', + 'core.compression=0', 'diff', '--binary', '--full-index', diff --git a/tests/campaign/worktree.test.ts b/tests/campaign/worktree.test.ts index 29c214b1..373860a5 100644 --- a/tests/campaign/worktree.test.ts +++ b/tests/campaign/worktree.test.ts @@ -138,6 +138,54 @@ describe('gitWorktreeAdapter — real git worktrees', () => { expect(surfaceHash(surfaceA)).toBe(surfaceHash(surfaceB)) }) + it('pins binary patch bytes across repository and global compression config', async () => { + const priorHome = process.env.HOME + const priorXdgConfigHome = process.env.XDG_CONFIG_HOME + const isolatedHome = mkdtempSync(join(tmpdir(), 'wt-git-home-')) + process.env.HOME = isolatedHome + process.env.XDG_CONFIG_HOME = isolatedHome + try { + const baseline = Buffer.concat( + Array.from({ length: 1024 }, (_, i) => Buffer.from(`record-${i % 17}\0`.repeat(32))), + ) + const candidate = Buffer.from(baseline) + for (let i = 0; i < candidate.length; i += 4096) { + candidate[i] = ((candidate[i] ?? 0) + 1) & 0xff + } + writeFileSync(join(repoRoot, 'fixture.bin'), baseline) + git(['add', 'fixture.bin'], repoRoot) + git(['commit', '-q', '-m', 'add binary fixture'], repoRoot) + + const adapter = gitWorktreeAdapter({ repoRoot }) + git(['config', '--global', 'core.compression', '9'], repoRoot) + git(['config', 'core.compression', '1'], repoRoot) + const repoConfigured = await adapter.create({ baseRef: 'main', label: 'repo-compression' }) + writeFileSync(join(repoConfigured.path, 'fixture.bin'), candidate) + const surfaceA = await adapter.finalize(repoConfigured, 'binary candidate A') + + git(['config', '--unset', 'core.compression'], repoRoot) + const globalConfigured = await adapter.create({ + baseRef: 'main', + label: 'global-compression', + }) + writeFileSync(join(globalConfigured.path, 'fixture.bin'), candidate) + const surfaceB = await adapter.finalize(globalConfigured, 'binary candidate B') + + expect(surfaceA.patch).toEqual(surfaceB.patch) + expect(surfaceContentHash(surfaceA)).toBe(surfaceContentHash(surfaceB)) + + git(['config', 'core.compression', '6'], repoRoot) + expect(() => verifyCodeSurface(surfaceA)).not.toThrow() + expect(() => verifyCodeSurface(surfaceB)).not.toThrow() + } finally { + if (priorHome === undefined) delete process.env.HOME + else process.env.HOME = priorHome + if (priorXdgConfigHome === undefined) delete process.env.XDG_CONFIG_HOME + else process.env.XDG_CONFIG_HOME = priorXdgConfigHome + rmSync(isolatedHome, { recursive: true, force: true }) + } + }) + it('changes both content hashes when one candidate byte changes', async () => { const adapter = gitWorktreeAdapter({ repoRoot }) const a = await adapter.create({ baseRef: 'main', label: 'bytes-a' }) From 3f22a4c26e3d2784b40497356b50772c234cbe63 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 10 Jul 2026 15:38:32 -0600 Subject: [PATCH 08/10] fix(tests): isolate nested git hooks --- tests/campaign/worktree.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/campaign/worktree.test.ts b/tests/campaign/worktree.test.ts index 373860a5..71acea9a 100644 --- a/tests/campaign/worktree.test.ts +++ b/tests/campaign/worktree.test.ts @@ -341,6 +341,9 @@ describe('gitWorktreeAdapter — real git worktrees', () => { git(['init', '-q', '-b', 'main'], nested) git(['config', 'user.email', 'test@test.dev'], nested) git(['config', 'user.name', 'Test'], nested) + const nestedHooks = join(nested, '.empty-hooks') + mkdirSync(nestedHooks) + git(['config', 'core.hooksPath', nestedHooks], nested) writeFileSync(join(nested, 'module.txt'), 'module bytes\n') git(['add', '-A'], nested) git(['commit', '-q', '-m', 'nested'], nested) From 9db6ca29008bee21c17343a2528db592b89ec6cb Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 10 Jul 2026 15:53:23 -0600 Subject: [PATCH 09/10] fix(campaign): isolate patch generation --- CHANGELOG.md | 2 +- src/campaign/worktree/index.ts | 85 +++++++++++++++++++++++++++++---- tests/campaign/worktree.test.ts | 81 +++++++++++++++++++++++++++++-- 3 files changed, 153 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e1c112f..aa32cb8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- - `CodeSurface` is now a finalized, content-addressed code candidate. `gitWorktreeAdapter.finalize()` records exact base/candidate commits, the final tree, and the SHA-256 + byte length of the raw binary Git patch; `surfaceHash` and `surfaceContentHash` no longer use filesystem paths. - Binary-patch generation pins Git compression at command scope so repository or global config cannot change the digest for identical trees. + Binary-patch generation runs against an isolated bare repository with fixed diff options, config, attributes, compression, and locale, so ambient repository, global, system, or environment settings cannot change the digest for identical trees. - `resolveWorktreePath()` now verifies the candidate before returning its checkout. Dirty or ignored files, moved refs, missing objects, wrong trees, raw byte or executable-mode mismatches, external symlinks, submodules, and patch-byte mismatches fail instead of being evaluated under stale identity. Raw file hashing bypasses Git clean/smudge filters so repository configuration cannot hide different executable bytes. diff --git a/src/campaign/worktree/index.ts b/src/campaign/worktree/index.ts index 72e121dd..881e8d21 100644 --- a/src/campaign/worktree/index.ts +++ b/src/campaign/worktree/index.ts @@ -18,11 +18,15 @@ import { constants as fsConstants, fstatSync, lstatSync, + mkdirSync, + mkdtempSync, openSync, readlinkSync, readSync, realpathSync, + rmSync, } from 'node:fs' +import { devNull, tmpdir } from 'node:os' import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' import { assertCodeSurfaceIdentity, surfaceContentHash } from '../surface-identity' import type { CodeSurface } from '../types' @@ -31,7 +35,8 @@ const MAX_GIT_OUTPUT_BYTES = 256 * 1024 * 1024 const FILE_HASH_CHUNK_BYTES = 1024 * 1024 type GitOutput = string | Uint8Array -type GitRunner = (args: string[], cwd: string) => GitOutput +type GitEnvironment = Readonly> +type GitRunner = (args: string[], cwd: string, env?: GitEnvironment) => GitOutput const GIT_REPOSITORY_ENV = new Set([ 'GIT_ALTERNATE_OBJECT_DIRECTORIES', @@ -87,19 +92,30 @@ export interface GitWorktreeAdapterOptions { /** Branch-name prefix. Default: `improve`. */ branchPrefix?: string /** Test seam — defaults to a real `git` runner. The return value must contain - * stdout verbatim; trimming a binary diff changes candidate identity. */ + * stdout verbatim, and runners that execute Git must forward the optional + * environment overrides used to isolate patch generation. */ git?: GitRunner } -function defaultGit(args: string[], cwd: string): Uint8Array { +function defaultGit(args: string[], cwd: string, overrides?: GitEnvironment): Uint8Array { try { const env = { ...process.env } for (const key of Object.keys(env)) { - if (GIT_REPOSITORY_ENV.has(key) || key === 'GIT_CONFIG' || key.startsWith('GIT_CONFIG_')) { + if ( + GIT_REPOSITORY_ENV.has(key) || + key === 'GIT_CONFIG' || + key.startsWith('GIT_CONFIG_') || + key.startsWith('GIT_ATTR_') || + key === 'GIT_DIFF_OPTS' || + key === 'GIT_EXTERNAL_DIFF' + ) { delete env[key] } } + Object.assign(env, overrides) env.GIT_NO_REPLACE_OBJECTS = '1' + env.LC_ALL = 'C' + env.LANG = 'C' return execFileSync('git', args, { cwd, env, maxBuffer: MAX_GIT_OUTPUT_BYTES }) } catch (err) { const stderr = @@ -110,9 +126,9 @@ function defaultGit(args: string[], cwd: string): Uint8Array { } } -function gitBytes(git: GitRunner, args: string[], cwd: string): Buffer { +function gitBytes(git: GitRunner, args: string[], cwd: string, env?: GitEnvironment): Buffer { try { - const output = git(args, cwd) + const output = git(args, cwd, env) return typeof output === 'string' ? Buffer.from(output, 'utf8') : Buffer.from(output) } catch (err) { if (err instanceof WorktreeAdapterError) throw err @@ -120,8 +136,8 @@ function gitBytes(git: GitRunner, args: string[], cwd: string): Buffer { } } -function gitText(git: GitRunner, args: string[], cwd: string): string { - return gitBytes(git, args, cwd).toString('utf8').trim() +function gitText(git: GitRunner, args: string[], cwd: string, env?: GitEnvironment): string { + return gitBytes(git, args, cwd, env).toString('utf8').trim() } function sha256(bytes: Uint8Array): `sha256:${string}` { @@ -134,7 +150,16 @@ const PATCH_ARGS = [ // the candidate identity for the same base and final tree. '-c', 'core.compression=0', + '-c', + `core.attributesFile=${devNull}`, + '-c', + 'core.quotePath=true', + '-c', + 'diff.suppressBlankEmpty=false', 'diff', + '--unified=3', + '--inter-hunk-context=0', + `-O${devNull}`, '--binary', '--full-index', '--no-color', @@ -147,13 +172,55 @@ const PATCH_ARGS = [ '--dst-prefix=b/', ] as const +const CANONICAL_GIT_ENV: GitEnvironment = { + GIT_ATTR_GLOBAL: devNull, + GIT_ATTR_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: devNull, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_SYSTEM: devNull, +} + +/** Render the transport patch from immutable objects through fresh Git metadata. + * Source-repository config, info attributes, templates, and caller environment + * therefore cannot alter bytes for the same two trees. */ function patchBytes( git: GitRunner, cwd: string, baseCommit: string, candidateCommit: string, ): Buffer { - return gitBytes(git, [...PATCH_ARGS, baseCommit, candidateCommit, '--'], cwd) + const scratch = mkdtempSync(join(tmpdir(), 'agent-eval-patch-')) + const bareRepo = join(scratch, 'repo.git') + const emptyTemplate = join(scratch, 'empty-template') + mkdirSync(emptyTemplate) + try { + const objectFormat = gitObjectHashAlgorithm(candidateCommit) + const sourceObjects = realpathSync(gitText(git, ['rev-parse', '--git-path', 'objects'], cwd)) + gitText( + git, + [ + 'init', + '--bare', + '--quiet', + ...(objectFormat === 'sha256' ? ['--object-format=sha256'] : []), + `--template=${emptyTemplate}`, + bareRepo, + ], + scratch, + CANONICAL_GIT_ENV, + ) + return gitBytes( + git, + [`--git-dir=${bareRepo}`, ...PATCH_ARGS, baseCommit, candidateCommit, '--'], + scratch, + { + ...CANONICAL_GIT_ENV, + GIT_ALTERNATE_OBJECT_DIRECTORIES: sourceObjects, + }, + ) + } finally { + rmSync(scratch, { recursive: true, force: true }) + } } function resolveCommit(git: GitRunner, cwd: string, ref: string): string { diff --git a/tests/campaign/worktree.test.ts b/tests/campaign/worktree.test.ts index 71acea9a..1fa9f825 100644 --- a/tests/campaign/worktree.test.ts +++ b/tests/campaign/worktree.test.ts @@ -9,7 +9,7 @@ import { unlinkSync, writeFileSync, } from 'node:fs' -import { tmpdir } from 'node:os' +import { devNull, tmpdir } from 'node:os' import { basename, dirname, join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { surfaceHash } from '../../src/campaign/presets/run-optimization' @@ -76,6 +76,9 @@ describe('gitWorktreeAdapter — real git worktrees', () => { 'git', [ 'diff', + '--unified=3', + '--inter-hunk-context=0', + `-O${devNull}`, '--binary', '--full-index', '--no-color', @@ -120,6 +123,32 @@ describe('gitWorktreeAdapter — real git worktrees', () => { expect(verifyCodeSurface(surface).path).toBe(wt.path) }) + it('finalizes and verifies SHA-256 repositories', async () => { + const shaRoot = mkdtempSync(join(tmpdir(), 'wt-sha256-repo-')) + try { + git(['init', '-q', '-b', 'main', '--object-format=sha256'], shaRoot) + git(['config', 'user.email', 'test@test.dev'], shaRoot) + git(['config', 'user.name', 'Test'], shaRoot) + const emptyHooks = join(shaRoot, '.empty-hooks') + mkdirSync(emptyHooks) + git(['config', 'core.hooksPath', emptyHooks], shaRoot) + writeFileSync(join(shaRoot, 'prompt.txt'), 'baseline prompt\n') + git(['add', '-A'], shaRoot) + git(['commit', '-q', '-m', 'init'], shaRoot) + + const adapter = gitWorktreeAdapter({ repoRoot: shaRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'sha256' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate prompt\n') + const surface = await adapter.finalize(wt, 'sha256 candidate') + + expect(surface.baseCommit).toMatch(/^[a-f0-9]{64}$/) + expect(surface.candidateCommit).toMatch(/^[a-f0-9]{64}$/) + expect(verifyCodeSurface(surface).path).toBe(wt.path) + } finally { + rmSync(shaRoot, { recursive: true, force: true }) + } + }) + it('hashes candidate bytes, not worktree paths or commit metadata', async () => { const adapter = gitWorktreeAdapter({ repoRoot }) const a = await adapter.create({ baseRef: 'main', label: 'same-a' }) @@ -138,7 +167,7 @@ describe('gitWorktreeAdapter — real git worktrees', () => { expect(surfaceHash(surfaceA)).toBe(surfaceHash(surfaceB)) }) - it('pins binary patch bytes across repository and global compression config', async () => { + it('pins patch bytes across repository and global diff config', async () => { const priorHome = process.env.HOME const priorXdgConfigHome = process.env.XDG_CONFIG_HOME const isolatedHome = mkdtempSync(join(tmpdir(), 'wt-git-home-')) @@ -153,28 +182,70 @@ describe('gitWorktreeAdapter — real git worktrees', () => { candidate[i] = ((candidate[i] ?? 0) + 1) & 0xff } writeFileSync(join(repoRoot, 'fixture.bin'), baseline) + writeFileSync( + join(repoRoot, 'context.txt'), + ['GLOBAL heading', 'one', '', 'two', '', 'three', 'LOCAL heading', 'tail', ''].join('\n'), + ) + writeFileSync(join(repoRoot, '.gitattributes'), 'context.txt diff=fixture\n') git(['add', 'fixture.bin'], repoRoot) + git(['add', 'context.txt', '.gitattributes'], repoRoot) git(['commit', '-q', '-m', 'add binary fixture'], repoRoot) const adapter = gitWorktreeAdapter({ repoRoot }) + const globalOrder = join(isolatedHome, 'global-order') + const repoOrder = join(isolatedHome, 'repo-order') + writeFileSync(globalOrder, 'fixture.bin\ncontext.txt\n') + writeFileSync(repoOrder, 'context.txt\nfixture.bin\n') git(['config', '--global', 'core.compression', '9'], repoRoot) + git(['config', '--global', 'diff.context', '20'], repoRoot) + git(['config', '--global', 'diff.interHunkContext', '20'], repoRoot) + git(['config', '--global', 'diff.suppressBlankEmpty', 'true'], repoRoot) + git(['config', '--global', 'diff.orderFile', globalOrder], repoRoot) + git(['config', '--global', 'diff.fixture.xfuncname', '^GLOBAL'], repoRoot) git(['config', 'core.compression', '1'], repoRoot) - const repoConfigured = await adapter.create({ baseRef: 'main', label: 'repo-compression' }) + git(['config', 'diff.context', '0'], repoRoot) + git(['config', 'diff.interHunkContext', '0'], repoRoot) + git(['config', 'diff.suppressBlankEmpty', 'false'], repoRoot) + git(['config', 'diff.orderFile', repoOrder], repoRoot) + git(['config', 'diff.fixture.xfuncname', '^LOCAL'], repoRoot) + const repoConfigured = await adapter.create({ baseRef: 'main', label: 'repo-config' }) writeFileSync(join(repoConfigured.path, 'fixture.bin'), candidate) + writeFileSync( + join(repoConfigured.path, 'context.txt'), + ['GLOBAL heading', 'ONE', '', 'two', '', 'THREE', 'LOCAL heading', 'tail', ''].join('\n'), + ) const surfaceA = await adapter.finalize(repoConfigured, 'binary candidate A') - git(['config', '--unset', 'core.compression'], repoRoot) + for (const key of [ + 'core.compression', + 'diff.context', + 'diff.interHunkContext', + 'diff.suppressBlankEmpty', + 'diff.orderFile', + 'diff.fixture.xfuncname', + ]) { + git(['config', '--unset', key], repoRoot) + } const globalConfigured = await adapter.create({ baseRef: 'main', - label: 'global-compression', + label: 'global-config', }) writeFileSync(join(globalConfigured.path, 'fixture.bin'), candidate) + writeFileSync( + join(globalConfigured.path, 'context.txt'), + ['GLOBAL heading', 'ONE', '', 'two', '', 'THREE', 'LOCAL heading', 'tail', ''].join('\n'), + ) const surfaceB = await adapter.finalize(globalConfigured, 'binary candidate B') expect(surfaceA.patch).toEqual(surfaceB.patch) expect(surfaceContentHash(surfaceA)).toBe(surfaceContentHash(surfaceB)) git(['config', 'core.compression', '6'], repoRoot) + git(['config', 'diff.context', '10'], repoRoot) + git(['config', 'diff.interHunkContext', '10'], repoRoot) + git(['config', 'diff.suppressBlankEmpty', 'true'], repoRoot) + git(['config', 'diff.orderFile', repoOrder], repoRoot) + git(['config', 'diff.fixture.xfuncname', '^LOCAL'], repoRoot) expect(() => verifyCodeSurface(surfaceA)).not.toThrow() expect(() => verifyCodeSurface(surfaceB)).not.toThrow() } finally { From 20199cdef9da3f266207a506c6e557b7bc7dc865 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 10 Jul 2026 16:08:15 -0600 Subject: [PATCH 10/10] fix(provenance): version content identity schema --- CHANGELOG.md | 1 + src/campaign/provenance.ts | 4 ++-- tests/campaign/presets.test.ts | 2 +- tests/contract-self-improve.test.ts | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa32cb8b..a80ba25f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- - `CodeSurface` is now a finalized, content-addressed code candidate. `gitWorktreeAdapter.finalize()` records exact base/candidate commits, the final tree, and the SHA-256 + byte length of the raw binary Git patch; `surfaceHash` and `surfaceContentHash` no longer use filesystem paths. Binary-patch generation runs against an isolated bare repository with fixed diff options, config, attributes, compression, and locale, so ambient repository, global, system, or environment settings cannot change the digest for identical trees. +- `LoopProvenanceRecord.schema` is now `tangle.loop-provenance.v2`, distinguishing records that use content-addressed code-surface hashes from v1 records whose code hashes included mutable paths. - `resolveWorktreePath()` now verifies the candidate before returning its checkout. Dirty or ignored files, moved refs, missing objects, wrong trees, raw byte or executable-mode mismatches, external symlinks, submodules, and patch-byte mismatches fail instead of being evaluated under stale identity. Raw file hashing bypasses Git clean/smudge filters so repository configuration cannot hide different executable bytes. diff --git a/src/campaign/provenance.ts b/src/campaign/provenance.ts index ced927b9..ae7f8968 100644 --- a/src/campaign/provenance.ts +++ b/src/campaign/provenance.ts @@ -78,7 +78,7 @@ export interface LoopProvenanceBackend { * the bare hosted event) + backend provenance. */ export interface LoopProvenanceRecord { - schema: 'tangle.loop-provenance.v1' + schema: 'tangle.loop-provenance.v2' runId: string runDir: string timestamp: string @@ -186,7 +186,7 @@ export function buildLoopProvenanceRecord const winnerHoldoutComposite = meanHoldoutComposite(args.winnerOnHoldout) const record: LoopProvenanceRecord = { - schema: 'tangle.loop-provenance.v1', + schema: 'tangle.loop-provenance.v2', runId: args.runId, runDir: args.runDir, timestamp: args.timestamp, diff --git a/tests/campaign/presets.test.ts b/tests/campaign/presets.test.ts index 9f08eab0..9b6edd3e 100644 --- a/tests/campaign/presets.test.ts +++ b/tests/campaign/presets.test.ts @@ -499,7 +499,7 @@ describe('loop provenance emission (transaction-extraction shape, offline)', () // (4) the structured record + OTel spans are DURABLE (written to storage). expect(storage.read(recordPath)).toBeDefined() - expect(JSON.parse(storage.read(recordPath)!).schema).toBe('tangle.loop-provenance.v1') + expect(JSON.parse(storage.read(recordPath)!).schema).toBe('tangle.loop-provenance.v2') const spanLines = storage.read(spansPath)!.split('\n') expect(spanLines.length).toBe(spans.length) // Spans pivot on the OTLP-ingestable tangle.* attributes the otel adapter reads. diff --git a/tests/contract-self-improve.test.ts b/tests/contract-self-improve.test.ts index 9089c3ec..1e1febc2 100644 --- a/tests/contract-self-improve.test.ts +++ b/tests/contract-self-improve.test.ts @@ -142,7 +142,7 @@ describe('selfImprove provenance emission (durable by default)', () => { // (3) the provenance record is on the result AND fired through onProvenance. expect(captured).toBeDefined() - expect(result.provenance.schema).toBe('tangle.loop-provenance.v1') + expect(result.provenance.schema).toBe('tangle.loop-provenance.v2') expect(result.provenance.winnerRationale).toBe(RATIONALE) // real content hashes distinguish baseline from winner + verify bytes. @@ -181,7 +181,7 @@ describe('selfImprove provenance emission (durable by default)', () => { // no runDir ⇒ mem://… ⇒ in-memory storage ⇒ nothing on disk }) // The provenance record is still produced in-memory + on the result. - expect(result.provenance.schema).toBe('tangle.loop-provenance.v1') + expect(result.provenance.schema).toBe('tangle.loop-provenance.v2') expect(result.provenance.runDir.startsWith('mem://')).toBe(true) }) })