diff --git a/.claude/skills/agent-eval/SKILL.md b/.claude/skills/agent-eval/SKILL.md index 5e8278c6..a07d6340 100644 --- a/.claude/skills/agent-eval/SKILL.md +++ b/.claude/skills/agent-eval/SKILL.md @@ -976,7 +976,7 @@ Do you have ≥5 well-defined transformations to test (e.g. directive enum)? **Recipe by tier:** - **Prompt-tier (string surface)** — start with `gepaDriver()`. Escalate to a custom `Mutator` + `evolutionaryDriver` only when you have a known directive enum that beats LLM-discovery. -- **Code-tier (`CodeSurface`)** — use agent-runtime's `improvementDriver` with a `reflectiveGenerator` or `agenticGenerator`. The substrate's drivers don't reason about code diffs. +- **Code-tier (`CodeSurface`)** — use agent-runtime's `improvementDriver` with a `reflectiveGenerator` or `agenticGenerator`, then finalize through `gitWorktreeAdapter` and call `verifyCodeSurface` before execution. The substrate's drivers don't reason about code diffs. - **Knowledge-tier** — out of scope here; agent-knowledge owns this. ### Surface menu — what string to pick diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index e2586c67..1c9e0c9b 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -17,3 +17,5 @@ {"skill":"critical-audit","ts":"2026-06-21T12:57:05Z","project":"agent-eval","target":"PR #264 review comment 4762012574","verdict":"APPROVE_AFTER_FIXES","score":"8/10","findings":"1 medium, 9 low; 8 fixed, 1 not_actionable, 1 deferred","files":[".gitignore","CHANGELOG.md","src/agent-profile.ts","src/agent-profile.test.ts","src/contract/define-agent-eval.ts","src/contract/index.ts","tests/contract-define-agent-eval.test.ts"]} {"skill":"critical-audit","ts":"2026-07-09T21:54:37Z","project":"agent-eval","target":"clustered paired binary inference for repository-disjoint coding benchmarks","verdict":"APPROVE_AFTER_FIXES","score":"8/10","findings":"3 medium found and resolved: estimand mismatch, one-cluster interval, insufficient bootstrap draws","files":["src/clustered-paired-binary.ts","src/statistics.ts","src/index.ts","tests/clustered-paired-binary.test.ts",".claude/skills/agent-eval/SKILL.md"]} {"skill":"critical-audit","ts":"2026-07-09T22:35:41Z","project":"agent-eval","target":"clustered paired binary inference at abe1d89a11fe1b40b75d0e4d96ccf68dfa2403b1","verdict":"APPROVE","score":"8/10","findings":"1 low accepted risk: independent-package parity vector remains before paper use","files":["src/clustered-paired-binary.ts","src/statistics.ts","src/index.ts","tests/clustered-paired-binary.test.ts",".claude/skills/agent-eval/SKILL.md"]} +{"skill":"agent-eval","ts":"2026-07-09T22:27:32Z","project":"agent-eval","target":"immutable CodeSurface identity","outcome":"implemented content-addressed Git identity and fail-loud verification","branch":"feat/immutable-code-surface","model_runs":0} +{"skill":"harden","ts":"2026-07-09T22:27:32Z","project":"agent-eval","target":"CodeSurface Git trust boundary","outcome":"fixed inherited Git environment repository-redirection bypass; added real-Git adversarial coverage","branch":"feat/immutable-code-surface","model_runs":0} diff --git a/CHANGELOG.md b/CHANGELOG.md index 32bc1c9a..3382905b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.113.0] — 2026-07-10 — immutable code candidates + +### Changed + +- `CodeSurface` is now a finalized, content-addressed code candidate. + `gitWorktreeAdapter.finalize()` records exact base/candidate commits, the final tree, and the SHA-256 + byte length of the raw binary Git patch; `surfaceHash` and `surfaceContentHash` no longer use filesystem paths. + Binary-patch generation runs against an isolated bare repository with fixed diff options, config, attributes, compression, and locale, so ambient repository, global, system, or environment settings cannot change the digest for identical trees. +- `LoopProvenanceRecord.schema` is now `tangle.loop-provenance.v2`, distinguishing records that use content-addressed code-surface hashes from v1 records whose code hashes included mutable paths. +- `resolveWorktreePath()` now verifies the candidate before returning its checkout. + Dirty or ignored files, moved refs, missing objects, wrong trees, raw byte or executable-mode mismatches, external symlinks, submodules, and patch-byte mismatches fail instead of being evaluated under stale identity. + Raw file hashing bypasses Git clean/smudge filters so repository configuration cannot hide different executable bytes. + Use `verifyCodeSurface()` when a verification receipt is needed directly. + +### Breaking + +- `resolveWorktreePath()` no longer returns a best-effort, unchecked locator: it verifies the finalized candidate and throws on any identity mismatch. + There is intentionally no lenient fallback on the evaluation path. +- Path-only `CodeSurface` objects are invalid. + Every field in the finalized identity is required. + Downstream callers must migrate to adapter-finalized candidates when adopting 0.113.0. + ## [0.112.0] — 2026-07-10 — complete agent-surface findings ### Added diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 59d4b596..1a5c2e9d 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.112.0" +version = "0.113.0" description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 03ed89b8..75948575 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -58,7 +58,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.112.0" + __version__ = "0.113.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index a3a87c32..a4e7b33d 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.10" [[package]] name = "agent-eval-rpc" -version = "0.112.0" +version = "0.113.0" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/docs/design/loop-taxonomy.md b/docs/design/loop-taxonomy.md index da0f7d37..ec6eb74e 100644 --- a/docs/design/loop-taxonomy.md +++ b/docs/design/loop-taxonomy.md @@ -162,7 +162,13 @@ imports agent-eval and implements it. `MutableSurface` is the thing the proposer changes. It has tiers, least → most invasive. `MutableSurface = string | CodeSurface` spans all of them: `string` -for tiers 1–2, `CodeSurface{ worktreeRef }` for tier 4. +for tiers 1–2, and a finalized `CodeSurface` for tier 4. A code surface's +worktree path is only its locator; exact base/candidate commits, final tree, +and binary-patch digest are its portable identity. Call `verifyCodeSurface` +before executing the checkout so a moved ref or post-finalization mutation +fails before measurement. Verification hashes raw files and executable modes +without Git filters and rejects external symlinks or submodules whose bytes are +not represented by the candidate tree. | Tier | Surface | Generator that changes it | Blast radius | |---|---|---|---| diff --git a/docs/improvement-glossary.md b/docs/improvement-glossary.md index fa0c4b38..af6898f4 100644 --- a/docs/improvement-glossary.md +++ b/docs/improvement-glossary.md @@ -30,7 +30,7 @@ Grounded to `agent-eval/src/campaign/types.ts` unless noted. | Term | Plain sentence | If you know DSPy/GEPA | |---|---|---| -| **surface** | The one thing being changed this run — a prompt string, a JSON config string, or a code/worktree ref (`MutableSurface = string \| CodeSurface`, `types.ts:158`). | The optimized artifact: a signature/predictor's instruction text, or the module config. | +| **surface** | The one thing being changed this run — a prompt string, a JSON config string, or a finalized code candidate (`MutableSurface = string \| CodeSurface`, `types.ts`). | The optimized artifact: a signature/predictor's instruction text, or the module config. | | **proposer** | The strategy that, given the current surface + what failed, proposes the next batch of candidate surfaces to measure — it does **not** run the agent or score anything (`SurfaceProposer.propose`, `types.ts:286`). | The optimizer / teleprompter (MIPRO, BootstrapFewShot, GEPA's reflective proposer). | | **candidate** | One proposed surface plus its human `label` and `rationale`, ready to be measured (`ProposedCandidate`, `types.ts:166`). | One trial instruction/program the optimizer wants to evaluate. | | **generation** | One round of *propose → measure → rank → promote*; the loop runs up to `maxGenerations` of them (`GenerationRecord`, `types.ts:549`). | One GEPA iteration / optimization step. | diff --git a/package.json b/package.json index 60123c4b..c307de9a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.112.0", + "version": "0.113.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index c0a74b91..083b81ed 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -80,8 +80,17 @@ try { throw new Error('missing belief-state export analyzeBeliefPolicy') } const campaign = await import('@tangle-network/agent-eval/campaign') - if (!('compositeProposer' in campaign)) { - throw new Error('missing campaign export compositeProposer') + for (const name of [ + 'compositeProposer', + 'gitWorktreeAdapter', + 'verifyCodeSurface', + 'resolveWorktreePath', + 'assertCodeSurfaceIdentity', + 'codeSurfaceIdentityMaterial', + 'surfaceHash', + 'surfaceContentHash', + ]) { + if (!(name in campaign)) throw new Error('missing campaign export ' + name) } `, ], diff --git a/src/campaign/analyst-surface.test.ts b/src/campaign/analyst-surface.test.ts index 41329889..cac00fa9 100644 --- a/src/campaign/analyst-surface.test.ts +++ b/src/campaign/analyst-surface.test.ts @@ -61,9 +61,9 @@ describe('buildAnalystSurfaceDispatch — runs the analyst with the surface as a it('fails loud if handed a code-tier surface (analyst prompt is prompt-tier)', async () => { const dispatch = buildAnalystSurfaceDispatch({ analystOptions: { ai: {} as never } }) - await expect(dispatch({ kind: 'code', worktreeRef: 'wt/abc' }, scenario, ctx)).rejects.toThrow( - /prompt-tier/, - ) + await expect( + dispatch({ kind: 'code', worktreeRef: 'wt/abc' } as never, scenario, ctx), + ).rejects.toThrow(/prompt-tier/) }) }) diff --git a/src/campaign/index.ts b/src/campaign/index.ts index 0ba1bd16..30c1af87 100644 --- a/src/campaign/index.ts +++ b/src/campaign/index.ts @@ -286,6 +286,11 @@ export { type SkillPatchRejection, } from './skill-patch' export { type CampaignStorage, fsCampaignStorage, inMemoryCampaignStorage } from './storage' +// ── Code-surface content identity ──────────────────────────────────── +export { + assertCodeSurfaceIdentity, + codeSurfaceIdentityMaterial, +} from './surface-identity' export { isTransientTransportFailure, type TransientFailureOptions } from './transient-failure' export type { CampaignAggregates, @@ -331,9 +336,11 @@ export type { export { isProposedCandidate, labelTrustRank } from './types' // ── Worktree adapter (VCS-pluggable; code-tier surfaces) ───────────── export { + type CodeSurfaceVerification, type GitWorktreeAdapterOptions, gitWorktreeAdapter, resolveWorktreePath, + verifyCodeSurface, type Worktree, type WorktreeAdapter, WorktreeAdapterError, diff --git a/src/campaign/presets/run-optimization.ts b/src/campaign/presets/run-optimization.ts index 96562b8c..e5e96ce9 100644 --- a/src/campaign/presets/run-optimization.ts +++ b/src/campaign/presets/run-optimization.ts @@ -15,10 +15,10 @@ * re-score + release gate + optional PR. */ -import { createHash } from 'node:crypto' import { type Objective, paretoFrontier } from '../../pareto' import { type RunCampaignOptions, runCampaign } from '../run-campaign' import { campaignBreakdown, campaignMeanComposite } from '../score-utils' +import { surfaceHash } from '../surface-identity' import { type CampaignResult, type GenerationRecord, @@ -355,19 +355,4 @@ function computeParetoFrontier(scored: ParetoParent[]): ParetoParent[] { return paretoFrontier(scored, objectives).frontier } -/** - * Short (16-char) sha256 fingerprint of a `MutableSurface`: hashes text content for prompt surfaces, or the worktree + base ref pair for code surfaces. - */ -export function surfaceHash(surface: MutableSurface): string { - // Prompt/tool surfaces (string) hash by content; code surfaces hash by the - // worktree + base ref pair (the content lives in git, not in the string). - const material = - typeof surface === 'string' - ? surface - : JSON.stringify({ - kind: surface.kind, - worktreeRef: surface.worktreeRef, - baseRef: surface.baseRef ?? null, - }) - return createHash('sha256').update(material).digest('hex').slice(0, 16) -} +export { surfaceHash } from '../surface-identity' diff --git a/src/campaign/proposers/composite.test.ts b/src/campaign/proposers/composite.test.ts index 5b1259ea..05bf4ad4 100644 --- a/src/campaign/proposers/composite.test.ts +++ b/src/campaign/proposers/composite.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import type { ProposeContext, SurfaceProposer } from '../types' +import type { CodeSurface, ProposeContext, SurfaceProposer } from '../types' import { compositeProposer } from './composite' function ctxOf(populationSize: number): ProposeContext { @@ -32,6 +32,23 @@ function stub( } } +function codeSurface(worktreeRef: string): CodeSurface { + return { + kind: 'code', + worktreeRef, + baseRef: 'main', + baseCommit: '1'.repeat(40), + baseTree: '2'.repeat(40), + candidateCommit: '3'.repeat(40), + candidateTree: '4'.repeat(40), + patch: { + format: 'git-diff-binary', + sha256: `sha256:${'5'.repeat(64)}`, + byteLength: 1, + }, + } +} + describe('compositeProposer (N proposers, one generation pool)', () => { it('splits the population budget and merges candidates with member provenance', async () => { const composite = compositeProposer({ @@ -69,6 +86,23 @@ describe('compositeProposer (N proposers, one generation pool)', () => { expect(sameLabel.startsWith('a:')).toBe(true) }) + it('dedupes content-identical code surfaces across different worktree paths', async () => { + const proposer = (kind: string, surface: CodeSurface): SurfaceProposer => ({ + kind, + propose: async () => [{ surface, label: 'v', rationale: 'r' }], + }) + const first = codeSurface('/tmp/candidate-a') + const sameBytesElsewhere = codeSurface('/tmp/candidate-b') + const composite = compositeProposer({ + proposers: [proposer('a', first), proposer('b', sameBytesElsewhere)], + }) + + const pool = await composite.propose(ctxOf(2)) + + expect(pool).toHaveLength(1) + expect((pool[0] as { surface: CodeSurface }).surface.worktreeRef).toBe(first.worktreeRef) + }) + it('isolates a failing member; throws only when ALL members fail', async () => { const oneDown = compositeProposer({ proposers: [stub('boom', [], { fail: true }), stub('ok', ['x1', 'x2'])], diff --git a/src/campaign/proposers/composite.ts b/src/campaign/proposers/composite.ts index c326becd..90a884fb 100644 --- a/src/campaign/proposers/composite.ts +++ b/src/campaign/proposers/composite.ts @@ -31,6 +31,7 @@ * component attribution — see the experiment-optimal research brief. */ +import { surfaceContentHash } from '../surface-identity' import type { GenerationRecord, MutableSurface, @@ -99,7 +100,7 @@ export function compositeProposer( : (proposal as MutableSurface) const label = isCandidate ? (proposal as ProposedCandidate).label : 'candidate' const rationale = isCandidate ? (proposal as ProposedCandidate).rationale : '' - const key = typeof surface === 'string' ? surface : JSON.stringify(surface) + const key = surfaceContentHash(surface) if (seen.has(key)) continue seen.add(key) pool.push({ surface, label: `${member.kind}:${label}`, rationale }) diff --git a/src/campaign/proposers/policy-edit.ts b/src/campaign/proposers/policy-edit.ts index ef5b7338..0eaa52c2 100644 --- a/src/campaign/proposers/policy-edit.ts +++ b/src/campaign/proposers/policy-edit.ts @@ -16,6 +16,7 @@ import { policyEditsFromFindings, } from '../../analyst/policy-edit' import type { AnalystFinding } from '../../analyst/types' +import { assertCodeSurfaceIdentity, surfaceContentHash } from '../surface-identity' import type { MutableSurface, ProposeContext, ProposedCandidate, SurfaceProposer } from '../types' export interface PolicyEditProposerOptions { @@ -108,7 +109,8 @@ function coerceCandidateSurface(surface: unknown): MutableSurface { if (surface && typeof surface === 'object') { const obj = surface as Record if (obj.kind === 'code' && typeof obj.worktreeRef === 'string') { - return surface as MutableSurface + assertCodeSurfaceIdentity(surface) + return surface } return JSON.stringify(surface, null, 2) } @@ -116,5 +118,5 @@ function coerceCandidateSurface(surface: unknown): MutableSurface { } function sameSurface(a: MutableSurface, b: MutableSurface): boolean { - return JSON.stringify(a) === JSON.stringify(b) + return surfaceContentHash(a) === surfaceContentHash(b) } diff --git a/src/campaign/provenance.ts b/src/campaign/provenance.ts index c47ce729..ae7f8968 100644 --- a/src/campaign/provenance.ts +++ b/src/campaign/provenance.ts @@ -37,24 +37,10 @@ import type { import { summarizeBackendIntegrity } from '../integrity/backend-integrity' import type { RunRecord } from '../run-record' import type { CampaignStorage } from './storage' +import { surfaceContentHash } from './surface-identity' import type { CampaignResult, GateDecision, GateResult, MutableSurface, Scenario } from './types' -/** Stable sha256 (full hex) of a surface's effective text. Code surfaces hash - * their worktree+base identity since the content lives in git. Distinct from - * `surfaceHash` (16-char content fingerprint used as a loop identity key); - * this is the byte-identical-verifiable content hash the provenance record + - * `RunRecord.promptHash` carry. */ -export function surfaceContentHash(surface: MutableSurface): string { - const material = - typeof surface === 'string' - ? surface - : JSON.stringify({ - kind: surface.kind, - worktreeRef: surface.worktreeRef, - baseRef: surface.baseRef ?? null, - }) - return `sha256:${createHash('sha256').update(material).digest('hex')}` -} +export { surfaceContentHash } from './surface-identity' export interface LoopProvenanceCandidate { /** Generation index this candidate was proposed in. */ @@ -92,7 +78,7 @@ export interface LoopProvenanceBackend { * the bare hosted event) + backend provenance. */ export interface LoopProvenanceRecord { - schema: 'tangle.loop-provenance.v1' + schema: 'tangle.loop-provenance.v2' runId: string runDir: string timestamp: string @@ -200,7 +186,7 @@ export function buildLoopProvenanceRecord const winnerHoldoutComposite = meanHoldoutComposite(args.winnerOnHoldout) const record: LoopProvenanceRecord = { - schema: 'tangle.loop-provenance.v1', + schema: 'tangle.loop-provenance.v2', runId: args.runId, runDir: args.runDir, timestamp: args.timestamp, diff --git a/src/campaign/surface-identity.ts b/src/campaign/surface-identity.ts new file mode 100644 index 00000000..373fbdfd --- /dev/null +++ b/src/campaign/surface-identity.ts @@ -0,0 +1,69 @@ +import { createHash } from 'node:crypto' +import type { CodeSurface, MutableSurface } from './types' + +const GIT_OBJECT_ID = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/ +const SHA256 = /^sha256:[a-f0-9]{64}$/ + +/** Fail when a code surface does not carry a complete immutable identity. */ +export function assertCodeSurfaceIdentity(surface: unknown): asserts surface is CodeSurface { + if (!surface || typeof surface !== 'object') { + throw new TypeError('CodeSurface must be an object') + } + const candidate = surface as Partial + if (candidate.kind !== 'code') throw new TypeError('CodeSurface.kind must be "code"') + if (typeof candidate.worktreeRef !== 'string' || candidate.worktreeRef.trim().length === 0) { + throw new TypeError('CodeSurface.worktreeRef must be a non-empty locator') + } + if (typeof candidate.baseRef !== 'string' || candidate.baseRef.trim().length === 0) { + throw new TypeError('CodeSurface.baseRef must be a non-empty ref label') + } + for (const [field, value] of [ + ['baseCommit', candidate.baseCommit], + ['baseTree', candidate.baseTree], + ['candidateCommit', candidate.candidateCommit], + ['candidateTree', candidate.candidateTree], + ] as const) { + if (typeof value !== 'string' || !GIT_OBJECT_ID.test(value)) { + throw new TypeError(`CodeSurface.${field} must be a full Git object id`) + } + } + const patch = candidate.patch + if (!patch || typeof patch !== 'object' || patch.format !== 'git-diff-binary') { + throw new TypeError('CodeSurface.patch.format must be "git-diff-binary"') + } + if (typeof patch.sha256 !== 'string' || !SHA256.test(patch.sha256)) { + throw new TypeError('CodeSurface.patch.sha256 must be a sha256 digest') + } + if (!Number.isSafeInteger(patch.byteLength) || patch.byteLength < 0) { + throw new TypeError('CodeSurface.patch.byteLength must be a non-negative safe integer') + } +} + +/** Canonical, location-independent identity of a finalized code candidate. + * Commit metadata is excluded: two commits with the same base, final tree, + * and patch bytes are the same executable candidate. */ +export function codeSurfaceIdentityMaterial(surface: CodeSurface): string { + assertCodeSurfaceIdentity(surface) + return JSON.stringify({ + schema: 'tangle.code-surface.v1', + baseCommit: surface.baseCommit, + baseTree: surface.baseTree, + candidateTree: surface.candidateTree, + patch: { + format: surface.patch.format, + sha256: surface.patch.sha256, + byteLength: surface.patch.byteLength, + }, + }) +} + +/** Full SHA-256 content identity for a prompt or finalized code surface. */ +export function surfaceContentHash(surface: MutableSurface): `sha256:${string}` { + const material = typeof surface === 'string' ? surface : codeSurfaceIdentityMaterial(surface) + return `sha256:${createHash('sha256').update(material).digest('hex')}` +} + +/** Short loop key derived from the same content identity as provenance. */ +export function surfaceHash(surface: MutableSurface): string { + return surfaceContentHash(surface).slice('sha256:'.length, 'sha256:'.length + 16) +} diff --git a/src/campaign/types.ts b/src/campaign/types.ts index 0523e5ce..789e53f0 100644 --- a/src/campaign/types.ts +++ b/src/campaign/types.ts @@ -132,20 +132,35 @@ export interface JudgeScore { // ── Optimization (population + generations + mutator) ───────────────── -/** A tier-4 code surface — a candidate change to the agent's +/** A tier-4 code surface — a finalized candidate change to the agent's * IMPLEMENTATION, not its prompt. Produced by autoresearch (reads codebase + - * trace findings → opens a worktree). Measured by checking out `worktreeRef` - * and running the worker against the changed code. See the improvement-tier - * table in `docs/design/loop-taxonomy.md`. */ + * trace findings → opens a worktree). `worktreeRef` locates the candidate; + * the exact commits, tree, and binary-patch digest identify it. See the + * improvement-tier table in `docs/design/loop-taxonomy.md`. */ export interface CodeSurface { - kind: 'code' - /** Worktree path or git ref holding the candidate code change. The - * consumer's `dispatchWithSurface` checks this out before running. */ - worktreeRef: string - /** Base ref the change is measured against. Default: the repo's main. */ - baseRef?: string + readonly kind: 'code' + /** Worktree path or git ref holding the candidate code change. This is a + * mutable locator and is deliberately excluded from content hashes. */ + readonly worktreeRef: string + /** Human-readable ref the worktree was forked from. Not identity-bearing. */ + readonly baseRef: string + /** Exact commit the candidate was forked from. */ + readonly baseCommit: string + /** Exact tree object for `baseCommit`. */ + readonly baseTree: string + /** Exact finalized candidate commit. */ + readonly candidateCommit: string + /** Exact tree object for `candidateCommit`. */ + readonly candidateTree: string + /** Identity of the exact patch artifact. The deployable candidate bundle + * carries the same descriptor plus its base64-encoded content. */ + readonly patch: { + readonly format: 'git-diff-binary' + readonly sha256: `sha256:${string}` + readonly byteLength: number + } /** Human summary of what changed — rendered into the auto-PR body. */ - summary?: string + readonly summary?: string } /** The mutable surface a proposer changes. Tiers (see diff --git a/src/campaign/worktree/index.ts b/src/campaign/worktree/index.ts index 27b86884..881e8d21 100644 --- a/src/campaign/worktree/index.ts +++ b/src/campaign/worktree/index.ts @@ -2,8 +2,8 @@ * VCS-pluggable worktree adapter. One improvement = one worktree, PR-like * (multiple commits allowed). A code-tier proposer's `propose()` creates a * worktree, an agent commits the change into it, and `finalize()` returns a - * `CodeSurface{ worktreeRef }` the measurement checks out to run the worker - * against the changed code. On promotion the worktree becomes the PR branch. + * content-addressed `CodeSurface` the measurement verifies before running. + * On promotion the worktree becomes the PR branch. * * The interface is VCS-agnostic so a future `jj` ([jj-vcs](https://github.com/jj-vcs/jj)) * adapter can slot in without touching proposer code. Only the git adapter @@ -11,25 +11,63 @@ */ import { execFileSync } from 'node:child_process' -import { existsSync } from 'node:fs' -import { basename, isAbsolute, join } from 'node:path' +import { createHash } from 'node:crypto' +import { + closeSync, + existsSync, + constants as fsConstants, + fstatSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readlinkSync, + readSync, + realpathSync, + rmSync, +} from 'node:fs' +import { devNull, tmpdir } from 'node:os' +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' +import { assertCodeSurfaceIdentity, surfaceContentHash } from '../surface-identity' import type { CodeSurface } from '../types' +const MAX_GIT_OUTPUT_BYTES = 256 * 1024 * 1024 +const FILE_HASH_CHUNK_BYTES = 1024 * 1024 + +type GitOutput = string | Uint8Array +type GitEnvironment = Readonly> +type GitRunner = (args: string[], cwd: string, env?: GitEnvironment) => GitOutput + +const GIT_REPOSITORY_ENV = new Set([ + 'GIT_ALTERNATE_OBJECT_DIRECTORIES', + 'GIT_COMMON_DIR', + 'GIT_DIR', + 'GIT_INDEX_FILE', + 'GIT_NAMESPACE', + 'GIT_OBJECT_DIRECTORY', + 'GIT_PREFIX', + 'GIT_QUARANTINE_PATH', + 'GIT_WORK_TREE', +]) + export interface Worktree { /** Absolute path to the checked-out worktree directory. */ - path: string + readonly path: string /** The branch the worktree is on (becomes the PR branch on promotion). */ - branch: string + readonly branch: string /** The ref the worktree was forked from. */ - baseRef: string + readonly baseRef: string + /** Exact commit `baseRef` resolved to before the worktree was created. */ + readonly baseCommit: string + /** Exact tree object for `baseCommit`. */ + readonly baseTree: string } export interface WorktreeAdapter { /** Create an isolated worktree on a fresh branch off `baseRef`. */ create(opts: { baseRef: string; label: string }): Promise - /** Commit any pending changes in the worktree, then return a CodeSurface - * pointing at it. The agent has already written its change into - * `worktree.path` by the time this is called. */ + /** Commit pending changes, freeze the exact Git objects + binary patch, and + * verify the worktree still matches that identity. */ finalize(worktree: Worktree, summary: string): Promise /** Remove the worktree (and its branch) — called for losing candidates. */ discard(worktree: Worktree): Promise @@ -53,13 +91,32 @@ export interface GitWorktreeAdapterOptions { worktreeDir?: string /** Branch-name prefix. Default: `improve`. */ branchPrefix?: string - /** Test seam — defaults to a real `git` runner. */ - git?: (args: string[], cwd: string) => string + /** Test seam — defaults to a real `git` runner. The return value must contain + * stdout verbatim, and runners that execute Git must forward the optional + * environment overrides used to isolate patch generation. */ + git?: GitRunner } -function defaultGit(args: string[], cwd: string): string { +function defaultGit(args: string[], cwd: string, overrides?: GitEnvironment): Uint8Array { try { - return execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() + const env = { ...process.env } + for (const key of Object.keys(env)) { + if ( + GIT_REPOSITORY_ENV.has(key) || + key === 'GIT_CONFIG' || + key.startsWith('GIT_CONFIG_') || + key.startsWith('GIT_ATTR_') || + key === 'GIT_DIFF_OPTS' || + key === 'GIT_EXTERNAL_DIFF' + ) { + delete env[key] + } + } + Object.assign(env, overrides) + env.GIT_NO_REPLACE_OBJECTS = '1' + env.LC_ALL = 'C' + env.LANG = 'C' + return execFileSync('git', args, { cwd, env, maxBuffer: MAX_GIT_OUTPUT_BYTES }) } catch (err) { const stderr = err && typeof err === 'object' && 'stderr' in err @@ -69,6 +126,506 @@ function defaultGit(args: string[], cwd: string): string { } } +function gitBytes(git: GitRunner, args: string[], cwd: string, env?: GitEnvironment): Buffer { + try { + const output = git(args, cwd, env) + return typeof output === 'string' ? Buffer.from(output, 'utf8') : Buffer.from(output) + } catch (err) { + if (err instanceof WorktreeAdapterError) throw err + throw new WorktreeAdapterError(`git ${args.join(' ')} failed: ${String(err)}`, err) + } +} + +function gitText(git: GitRunner, args: string[], cwd: string, env?: GitEnvironment): string { + return gitBytes(git, args, cwd, env).toString('utf8').trim() +} + +function sha256(bytes: Uint8Array): `sha256:${string}` { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + +const PATCH_ARGS = [ + // Git binary patches embed a zlib stream. Pin no-compression at command + // scope so repository/global config and compressor heuristics cannot change + // the candidate identity for the same base and final tree. + '-c', + 'core.compression=0', + '-c', + `core.attributesFile=${devNull}`, + '-c', + 'core.quotePath=true', + '-c', + 'diff.suppressBlankEmpty=false', + 'diff', + '--unified=3', + '--inter-hunk-context=0', + `-O${devNull}`, + '--binary', + '--full-index', + '--no-color', + '--no-ext-diff', + '--no-textconv', + '--no-renames', + '--diff-algorithm=myers', + '--no-indent-heuristic', + '--src-prefix=a/', + '--dst-prefix=b/', +] as const + +const CANONICAL_GIT_ENV: GitEnvironment = { + GIT_ATTR_GLOBAL: devNull, + GIT_ATTR_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: devNull, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_SYSTEM: devNull, +} + +/** Render the transport patch from immutable objects through fresh Git metadata. + * Source-repository config, info attributes, templates, and caller environment + * therefore cannot alter bytes for the same two trees. */ +function patchBytes( + git: GitRunner, + cwd: string, + baseCommit: string, + candidateCommit: string, +): Buffer { + const scratch = mkdtempSync(join(tmpdir(), 'agent-eval-patch-')) + const bareRepo = join(scratch, 'repo.git') + const emptyTemplate = join(scratch, 'empty-template') + mkdirSync(emptyTemplate) + try { + const objectFormat = gitObjectHashAlgorithm(candidateCommit) + const sourceObjects = realpathSync(gitText(git, ['rev-parse', '--git-path', 'objects'], cwd)) + gitText( + git, + [ + 'init', + '--bare', + '--quiet', + ...(objectFormat === 'sha256' ? ['--object-format=sha256'] : []), + `--template=${emptyTemplate}`, + bareRepo, + ], + scratch, + CANONICAL_GIT_ENV, + ) + return gitBytes( + git, + [`--git-dir=${bareRepo}`, ...PATCH_ARGS, baseCommit, candidateCommit, '--'], + scratch, + { + ...CANONICAL_GIT_ENV, + GIT_ALTERNATE_OBJECT_DIRECTORIES: sourceObjects, + }, + ) + } finally { + rmSync(scratch, { recursive: true, force: true }) + } +} + +function resolveCommit(git: GitRunner, cwd: string, ref: string): string { + return gitText(git, ['rev-parse', '--verify', `${ref}^{commit}`], cwd) +} + +function unresolvedWorktreePath(surface: CodeSurface, worktreeDir?: string): string { + if (isAbsolute(surface.worktreeRef)) return surface.worktreeRef + if (worktreeDir) return join(worktreeDir, basename(surface.worktreeRef)) + return surface.worktreeRef +} + +interface GitTreeEntry { + mode: string + objectId: string + path: string +} + +function displayGitPath(path: string): string { + return JSON.stringify(path) +} + +function parseGitTreeEntries(bytes: Uint8Array): GitTreeEntry[] { + const input = Buffer.from(bytes) + const entries: GitTreeEntry[] = [] + let start = 0 + while (start < input.length) { + const end = input.indexOf(0, start) + if (end < 0) throw new WorktreeAdapterError('Git tree output was not NUL-terminated') + const record = input.subarray(start, end) + start = end + 1 + if (record.length === 0) continue + + const tab = record.indexOf(0x09) + if (tab < 0) throw new WorktreeAdapterError('Git tree entry did not contain a path') + const metadata = record.subarray(0, tab).toString('ascii').split(' ') + if (metadata.length !== 3) throw new WorktreeAdapterError('Git tree entry was malformed') + const mode = metadata[0] + const type = metadata[1] + const objectId = metadata[2] + if (!mode || !type || !objectId) { + throw new WorktreeAdapterError('Git tree entry was malformed') + } + const pathBytes = record.subarray(tab + 1) + const path = pathBytes.toString('utf8') + if (!Buffer.from(path, 'utf8').equals(pathBytes)) { + throw new WorktreeAdapterError('CodeSurface paths must be valid UTF-8') + } + if (type !== 'blob' && type !== 'commit') { + throw new WorktreeAdapterError( + `CodeSurface contains unsupported Git object type ${type} at ${displayGitPath(path)}`, + ) + } + entries.push({ mode, objectId, path }) + } + return entries +} + +function parseGitPathList(bytes: Uint8Array): string[] { + const input = Buffer.from(bytes) + const paths: string[] = [] + let start = 0 + while (start < input.length) { + const end = input.indexOf(0, start) + if (end < 0) throw new WorktreeAdapterError('Git path output was not NUL-terminated') + const pathBytes = input.subarray(start, end) + start = end + 1 + if (pathBytes.length === 0) continue + const path = pathBytes.toString('utf8') + if (!Buffer.from(path, 'utf8').equals(pathBytes)) { + throw new WorktreeAdapterError('CodeSurface paths must be valid UTF-8') + } + paths.push(path) + } + return paths +} + +function isWithinRoot(root: string, candidate: string): boolean { + const rel = relative(root, candidate) + return rel === '' || (!isAbsolute(rel) && rel !== '..' && !rel.startsWith(`..${sep}`)) +} + +function assertSafeRelativePath(root: string, path: string): string { + if (path.length === 0 || isAbsolute(path)) { + throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`) + } + const absolutePath = resolve(root, path) + if (!isWithinRoot(root, absolutePath) || absolutePath === root) { + throw new WorktreeAdapterError(`CodeSurface path escapes its worktree: ${displayGitPath(path)}`) + } + + const segments = path.split(/[\\/]/u) + let parent = root + for (const segment of segments.slice(0, -1)) { + if (segment.length === 0 || segment === '.' || segment === '..') { + throw new WorktreeAdapterError(`CodeSurface contains unsafe path ${displayGitPath(path)}`) + } + parent = join(parent, segment) + const stat = lstatSync(parent) + if (!stat.isDirectory() || stat.isSymbolicLink()) { + throw new WorktreeAdapterError( + `CodeSurface path traverses a non-directory or symbolic link: ${displayGitPath(path)}`, + ) + } + } + return absolutePath +} + +function gitObjectHashAlgorithm(objectId: string): 'sha1' | 'sha256' { + if (/^[a-f0-9]{40}$/.test(objectId)) return 'sha1' + if (/^[a-f0-9]{64}$/.test(objectId)) return 'sha256' + throw new WorktreeAdapterError(`Unsupported Git object id: ${objectId}`) +} + +function hashGitBlobBytes(bytes: Uint8Array, objectId: string): string { + const hash = createHash(gitObjectHashAlgorithm(objectId)) + hash.update(`blob ${bytes.byteLength}\0`) + hash.update(bytes) + return hash.digest('hex') +} + +function hashGitBlobFile(path: string, objectId: string): { hash: string; executable: boolean } { + const before = lstatSync(path) + if (!before.isFile() || before.isSymbolicLink()) { + throw new WorktreeAdapterError(`CodeSurface expected a regular file at ${displayGitPath(path)}`) + } + + const noFollow = process.platform === 'win32' ? 0 : fsConstants.O_NOFOLLOW + const fd = openSync(path, fsConstants.O_RDONLY | noFollow) + try { + const opened = fstatSync(fd) + if ( + !opened.isFile() || + opened.dev !== before.dev || + opened.ino !== before.ino || + opened.size !== before.size + ) { + throw new WorktreeAdapterError( + `CodeSurface file changed while it was being verified: ${displayGitPath(path)}`, + ) + } + + const hash = createHash(gitObjectHashAlgorithm(objectId)) + hash.update(`blob ${opened.size}\0`) + const chunk = Buffer.allocUnsafe(FILE_HASH_CHUNK_BYTES) + let total = 0 + while (true) { + const count = readSync(fd, chunk, 0, chunk.length, null) + if (count === 0) break + total += count + hash.update(chunk.subarray(0, count)) + } + if (total !== opened.size) { + throw new WorktreeAdapterError( + `CodeSurface file changed while it was being verified: ${displayGitPath(path)}`, + ) + } + return { hash: hash.digest('hex'), executable: (opened.mode & 0o111) !== 0 } + } finally { + closeSync(fd) + } +} + +function assertSymlinkTargetIsBound( + root: string, + linkPath: string, + trackedPaths: ReadonlySet, +): void { + const targetBytes = readlinkSync(linkPath, { encoding: 'buffer' }) + const target = targetBytes.toString('utf8') + if (!Buffer.from(target, 'utf8').equals(targetBytes)) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link has an unsafe target at ${displayGitPath(linkPath)}`, + ) + } + if (isAbsolute(target)) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`, + ) + } + const lexicalTarget = resolve(dirname(linkPath), target) + if (!isWithinRoot(root, lexicalTarget)) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`, + ) + } + let resolvedTarget: string + try { + resolvedTarget = realpathSync(linkPath) + } catch (err) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link target is missing or cyclic at ${displayGitPath(linkPath)}`, + err, + ) + } + if (!isWithinRoot(root, resolvedTarget)) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link escapes its worktree at ${displayGitPath(linkPath)}`, + ) + } + + const targetRelative = relative(root, resolvedTarget).split(sep).join('/') + const targetIsTracked = + trackedPaths.has(targetRelative) || + [...trackedPaths].some((trackedPath) => trackedPath.startsWith(`${targetRelative}/`)) + if (!targetIsTracked) { + throw new WorktreeAdapterError( + `CodeSurface symbolic link resolves to untracked content at ${displayGitPath(linkPath)}`, + ) + } +} + +function assertRawTreeMatchesWorktree(git: GitRunner, root: string, candidateCommit: string): void { + const entries = parseGitTreeEntries( + gitBytes(git, ['ls-tree', '-r', '-z', '--full-tree', candidateCommit], root), + ) + const trackedPaths = new Set(entries.map((entry) => entry.path)) + + for (const entry of entries) { + const absolutePath = assertSafeRelativePath(root, entry.path) + if (entry.mode === '160000' || entry.mode === '040000') { + throw new WorktreeAdapterError( + `CodeSurface contains a Git submodule whose executable bytes are not bound: ${displayGitPath(entry.path)}`, + ) + } + if (entry.mode === '120000') { + const stat = lstatSync(absolutePath) + if (!stat.isSymbolicLink()) { + throw new WorktreeAdapterError( + `CodeSurface expected a symbolic link at ${displayGitPath(entry.path)}`, + ) + } + const targetBytes = readlinkSync(absolutePath, { encoding: 'buffer' }) + const actualObjectId = hashGitBlobBytes(targetBytes, entry.objectId) + if (actualObjectId !== entry.objectId) { + throw new WorktreeAdapterError( + `CodeSurface raw symbolic-link bytes differ from the candidate tree at ${displayGitPath(entry.path)}`, + ) + } + assertSymlinkTargetIsBound(root, absolutePath, trackedPaths) + continue + } + if (entry.mode !== '100644' && entry.mode !== '100755') { + throw new WorktreeAdapterError( + `CodeSurface contains unsupported Git mode ${entry.mode} at ${displayGitPath(entry.path)}`, + ) + } + + const actual = hashGitBlobFile(absolutePath, entry.objectId) + if (actual.hash !== entry.objectId) { + throw new WorktreeAdapterError( + `CodeSurface raw tracked bytes differ from the candidate tree because the worktree changed after finalization at ${displayGitPath(entry.path)}`, + ) + } + if (process.platform !== 'win32' && actual.executable !== (entry.mode === '100755')) { + throw new WorktreeAdapterError( + `CodeSurface executable mode differs from the candidate tree at ${displayGitPath(entry.path)}`, + ) + } + } +} + +export interface CodeSurfaceVerification { + /** Verified worktree path. */ + path: string + /** Git's canonical root for the verified checkout. */ + repoRoot: string + /** Recomputed full content identity. */ + contentHash: `sha256:${string}` + /** Exact verified binary-patch bytes. Candidate-bundle builders encode this + * directly instead of reproducing Git diff options. */ + patchBytes: Uint8Array +} + +function verifyCodeSurfaceWithGit( + surface: CodeSurface, + path: string, + git: GitRunner, +): CodeSurfaceVerification { + assertCodeSurfaceIdentity(surface) + if (!existsSync(path)) { + throw new WorktreeAdapterError(`CodeSurface worktree does not exist: ${path}`) + } + const lexicalRoot = resolve(path) + const canonicalRoot = realpathSync(path) + if ( + lstatSync(lexicalRoot).isSymbolicLink() || + (process.platform !== 'win32' && lexicalRoot !== canonicalRoot) + ) { + throw new WorktreeAdapterError( + `CodeSurface worktree locator must not contain a symbolic link: ${path}`, + ) + } + const repoRoot = gitText(git, ['rev-parse', '--show-toplevel'], path) + const canonicalRepoRoot = realpathSync(repoRoot) + if (canonicalRepoRoot !== canonicalRoot) { + throw new WorktreeAdapterError( + `CodeSurface worktree locator is not the repository root: expected ${repoRoot}, got ${path}`, + ) + } + + const indexFlags = gitText(git, ['ls-files', '-v'], path) + .split('\n') + .filter((line) => line.length > 0 && (line.startsWith('S ') || /^[a-z]/.test(line))) + if (indexFlags.length > 0) { + throw new WorktreeAdapterError( + `CodeSurface worktree uses hidden index entries that cannot be verified: ${path}\n${indexFlags.join('\n')}`, + ) + } + + const extraPaths = [ + ...parseGitPathList(gitBytes(git, ['ls-files', '--others', '--exclude-standard', '-z'], path)), + ...parseGitPathList( + gitBytes(git, ['ls-files', '--others', '--ignored', '--exclude-standard', '-z'], path), + ), + ] + if (extraPaths.length > 0) { + throw new WorktreeAdapterError( + `CodeSurface worktree changed after finalization: ${path}\n${extraPaths.join('\n')}`, + ) + } + + const candidateCommit = resolveCommit(git, path, 'HEAD') + if (candidateCommit !== surface.candidateCommit) { + throw new WorktreeAdapterError( + `CodeSurface candidate commit mismatch: expected ${surface.candidateCommit}, got ${candidateCommit}`, + ) + } + + const baseCommit = resolveCommit(git, path, surface.baseCommit) + if (baseCommit !== surface.baseCommit) { + throw new WorktreeAdapterError( + `CodeSurface base commit mismatch: expected ${surface.baseCommit}, got ${baseCommit}`, + ) + } + + const baseTree = gitText(git, ['rev-parse', '--verify', `${surface.baseCommit}^{tree}`], path) + if (baseTree !== surface.baseTree) { + throw new WorktreeAdapterError( + `CodeSurface base tree mismatch: expected ${surface.baseTree}, got ${baseTree}`, + ) + } + + const candidateTree = gitText( + git, + ['rev-parse', '--verify', `${surface.candidateCommit}^{tree}`], + path, + ) + if (candidateTree !== surface.candidateTree) { + throw new WorktreeAdapterError( + `CodeSurface tree mismatch: expected ${surface.candidateTree}, got ${candidateTree}`, + ) + } + + try { + gitText( + git, + [ + 'diff-index', + '--cached', + '--quiet', + '--no-ext-diff', + '--no-textconv', + surface.candidateCommit, + '--', + ], + path, + ) + } catch (err) { + throw new WorktreeAdapterError( + `CodeSurface index differs from finalized candidate: ${path}`, + err, + ) + } + + try { + assertRawTreeMatchesWorktree(git, canonicalRepoRoot, surface.candidateCommit) + } catch (err) { + if (err instanceof WorktreeAdapterError) throw err + throw new WorktreeAdapterError(`CodeSurface raw tree verification failed: ${path}`, err) + } + + const mergeBase = gitText(git, ['merge-base', surface.baseCommit, surface.candidateCommit], path) + if (mergeBase !== surface.baseCommit) { + throw new WorktreeAdapterError( + `CodeSurface candidate ${surface.candidateCommit} does not descend from base ${surface.baseCommit}`, + ) + } + + const patch = patchBytes(git, path, surface.baseCommit, surface.candidateCommit) + const actualPatchHash = sha256(patch) + if (actualPatchHash !== surface.patch.sha256 || patch.byteLength !== surface.patch.byteLength) { + throw new WorktreeAdapterError( + `CodeSurface patch mismatch: expected ${surface.patch.sha256}/${surface.patch.byteLength} bytes, got ${actualPatchHash}/${patch.byteLength} bytes`, + ) + } + + return { + path, + repoRoot: canonicalRepoRoot, + contentHash: surfaceContentHash(surface), + patchBytes: new Uint8Array(patch), + } +} + /** Slugify a label into a branch-safe segment. */ function slug(label: string): string { return ( @@ -93,40 +650,76 @@ export function gitWorktreeAdapter(opts: GitWorktreeAdapterOptions): WorktreeAda const id = `${slug(label)}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}` const branch = `${branchPrefix}/${id}` const path = join(worktreeDir, id) - git(['worktree', 'add', '-b', branch, path, baseRef], opts.repoRoot) - return { path, branch, baseRef } + const baseCommit = resolveCommit(git, opts.repoRoot, baseRef) + const baseTree = gitText( + git, + ['rev-parse', '--verify', `${baseCommit}^{tree}`], + opts.repoRoot, + ) + gitText(git, ['worktree', 'add', '-b', branch, path, baseCommit], opts.repoRoot) + return { path, branch, baseRef, baseCommit, baseTree } }, async finalize(worktree, summary) { // Stage + commit any pending changes the agent left in the worktree. // A no-op commit is refused by git, so only commit when the tree is dirty. - const status = git(['status', '--porcelain'], worktree.path) + const status = gitText( + git, + ['status', '--porcelain=v1', '--untracked-files=all'], + worktree.path, + ) if (status.length > 0) { - git(['add', '-A'], worktree.path) - git(['commit', '-m', summary], worktree.path) + gitText(git, ['add', '-A'], worktree.path) + gitText(git, ['commit', '-m', summary], worktree.path) } - return { + const candidateCommit = resolveCommit(git, worktree.path, 'HEAD') + const candidateTree = gitText( + git, + ['rev-parse', '--verify', `${candidateCommit}^{tree}`], + worktree.path, + ) + const patch = patchBytes(git, worktree.path, worktree.baseCommit, candidateCommit) + const surface: CodeSurface = { kind: 'code', worktreeRef: worktree.path, baseRef: worktree.baseRef, + baseCommit: worktree.baseCommit, + baseTree: worktree.baseTree, + candidateCommit, + candidateTree, + patch: { + format: 'git-diff-binary', + sha256: sha256(patch), + byteLength: patch.byteLength, + }, summary, } + verifyCodeSurfaceWithGit(surface, worktree.path, git) + return surface }, async discard(worktree) { // Remove the worktree, then delete its branch. Force-remove because the // worktree may hold uncommitted experiment state we're discarding. - git(['worktree', 'remove', '--force', worktree.path], opts.repoRoot) - git(['branch', '-D', worktree.branch], opts.repoRoot) + gitText(git, ['worktree', 'remove', '--force', worktree.path], opts.repoRoot) + gitText(git, ['branch', '-D', worktree.branch], opts.repoRoot) }, } } -/** Resolve a `CodeSurface`'s worktreeRef to a directory the measurement can - * run the worker in. A path ref is returned as-is; anything else is treated - * as a ref under the adapter's worktree dir. */ +/** Verify a finalized code surface against its current checkout. This rejects + * dirty/ignored files, moved refs, missing Git objects, raw byte/mode + * mismatches, external symlinks, and submodules. */ +export function verifyCodeSurface( + surface: CodeSurface, + worktreeDir?: string, +): CodeSurfaceVerification { + const path = unresolvedWorktreePath(surface, worktreeDir) + return verifyCodeSurfaceWithGit(surface, path, defaultGit) +} + +/** Resolve a code candidate for evaluation only after verifying its immutable + * identity against the checkout at `worktreeRef`. */ export function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string { - if (isAbsolute(surface.worktreeRef) && existsSync(surface.worktreeRef)) return surface.worktreeRef - if (worktreeDir) return join(worktreeDir, basename(surface.worktreeRef)) - return surface.worktreeRef + return verifyCodeSurface(surface, worktreeDir).path } diff --git a/src/contract/self-improve.test.ts b/src/contract/self-improve.test.ts index fedf26b3..31556903 100644 --- a/src/contract/self-improve.test.ts +++ b/src/contract/self-improve.test.ts @@ -8,7 +8,8 @@ */ import { describe, expect, it } from 'vitest' -import type { Gate, JudgeConfig, SurfaceProposer } from '../campaign/types' +import { surfaceHash } from '../campaign/surface-identity' +import type { CodeSurface, Gate, JudgeConfig, SurfaceProposer } from '../campaign/types' import type { DispatchContext, Scenario } from './index' import { type SelfImproveProgressEvent, selfImprove } from './self-improve' @@ -37,6 +38,23 @@ async function stubAgent( return { text: String(surface) } } +function codeSurface(worktreeRef: string): CodeSurface { + return { + kind: 'code', + worktreeRef, + baseRef: 'main', + baseCommit: '1'.repeat(40), + baseTree: '2'.repeat(40), + candidateCommit: '3'.repeat(40), + candidateTree: '4'.repeat(40), + patch: { + format: 'git-diff-binary', + sha256: `sha256:${'5'.repeat(64)}`, + byteLength: 1, + }, + } +} + describe('selfImprove — power analysis wiring', () => { it('attaches the MDE analysis to the result and emits power.estimated', async () => { const events: SelfImproveProgressEvent[] = [] @@ -62,6 +80,53 @@ describe('selfImprove — power analysis wiring', () => { }) }) +describe('selfImprove — hosted code-surface identity', () => { + it('uses the content identity in snapshots instead of the mutable worktree path', async () => { + const baseline = codeSurface('/tmp/candidate-a') + const sameBytesElsewhere = codeSurface('/tmp/candidate-b') + const payloads: Array<{ + events?: Array<{ + baseline?: { surfaceHash: string } + generations: Array<{ surfaceHash: string }> + }> + }> = [] + const fetchImpl: typeof fetch = async (_input, init) => { + if (typeof init?.body === 'string') payloads.push(JSON.parse(init.body)) + return new Response(JSON.stringify({ accepted: 1, rejected: [] }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + } + const proposer: SurfaceProposer = { + kind: 'code-candidate', + propose: async () => [sameBytesElsewhere], + } + + await selfImprove({ + agent: stubAgent, + scenarios, + judge, + baselineSurface: baseline, + proposer, + budget: { generations: 1, populationSize: 1, holdoutFraction: 0.5 }, + hostedTenant: { + endpoint: 'https://ingest.example', + apiKey: 'test-key', + tenantId: 'test-tenant', + fetchImpl, + }, + }) + + const expected = surfaceHash(baseline) + const event = payloads + .flatMap((payload) => payload.events ?? []) + .find((candidate) => candidate.baseline?.surfaceHash === expected) + expect(event).toBeDefined() + expect(event?.generations[0]?.surfaceHash).toBe(expected) + expect(surfaceHash(sameBytesElsewhere)).toBe(expected) + }) +}) + describe('selfImprove — neutralize (placebo arm) passthrough', () => { it('forwards `neutralize`: runs the placebo arm and hands its scores to the gate', async () => { let neutralizeCalled = false diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index 3be248f2..6270ccee 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -41,6 +41,7 @@ import { fsCampaignStorage, inMemoryCampaignStorage, } from '../campaign/storage' +import { surfaceHash } from '../campaign/surface-identity' import type { CampaignCellResult, DispatchContext, @@ -620,7 +621,7 @@ async function shipEvalRunToHosted( function snapshotFromCampaign( index: number, - surface: MutableSurface | undefined, + surface: MutableSurface, campaign: RunImprovementLoopResult['baselineCampaign'], durationMs: number, ): EvalRunGenerationSnapshot { @@ -644,10 +645,7 @@ async function shipEvalRunToHosted( cells.length === 0 ? 0 : cells.reduce((s, c) => s + c.compositeMean, 0) / cells.length return { index, - surfaceHash: - typeof surface === 'string' - ? hashString(surface) - : hashString(JSON.stringify(surface ?? '')), + surfaceHash: surfaceHash(surface), surface, cells, compositeMean, diff --git a/tests/campaign/policy-edit-proposer.test.ts b/tests/campaign/policy-edit-proposer.test.ts index 65c4e848..df29791a 100644 --- a/tests/campaign/policy-edit-proposer.test.ts +++ b/tests/campaign/policy-edit-proposer.test.ts @@ -4,6 +4,19 @@ import { makeFinding } from '../../src/analyst/types' import { policyEditProposer } from '../../src/campaign/proposers/policy-edit' import type { CodeSurface, ProposeContext } from '../../src/campaign/types' +const CODE_IDENTITY = { + baseRef: 'main', + baseCommit: 'a'.repeat(40), + baseTree: 'b'.repeat(40), + candidateCommit: 'c'.repeat(40), + candidateTree: 'd'.repeat(40), + patch: { + format: 'git-diff-binary' as const, + sha256: `sha256:${'e'.repeat(64)}` as const, + byteLength: 123, + }, +} + function ctx( findings: unknown[], currentSurface: ProposeContext['currentSurface'] = 'Base prompt.', @@ -171,10 +184,11 @@ describe('policyEditProposer', () => { expect(JSON.parse(String(out[0]!.surface))).toEqual({ budget: { maxTurns: 6 } }) }) - it('passes through CodeSurface-shaped candidate surfaces', async () => { + it('skips metadata-only CodeSurface edits that do not change candidate identity', async () => { const codeSurface: CodeSurface = { kind: 'code', worktreeRef: '/tmp/policy-edit-candidate', + ...CODE_IDENTITY, summary: 'old summary', } const codeEdit = makePolicyEdit({ @@ -194,11 +208,29 @@ describe('policyEditProposer', () => { const proposer = policyEditProposer() const out = await proposer.propose(ctx([codeEdit], codeSurface)) - expect(out[0]!.surface).toEqual({ - kind: 'code', - worktreeRef: '/tmp/policy-edit-candidate', - summary: 'updated summary', + expect(out).toEqual([]) + }) + + it('rejects a path-only CodeSurface instead of preserving mutable identity', async () => { + const codeEdit = makePolicyEdit({ + axis: 'agent_profile', + target: { surface: 'runtime-config', path: 'summary' }, + change: { kind: 'json', mode: 'set', path: 'summary', value: 'updated' }, + claim: 'Candidate code surface must already be finalized.', + expectedGain: { metric: 'holdout.composite', direction: 'increase', amount: 0.04 }, + confidence: 0.85, + risk: 'low', + source: { + findingIds: ['f_code_surface'], + analystIds: ['trace-analyst'], + evidenceRefs: [{ kind: 'artifact', uri: 'file:///tmp/policy-edit-candidate' }], + }, }) + const proposer = policyEditProposer() + + await expect( + proposer.propose(ctx([codeEdit], { kind: 'code', worktreeRef: '/tmp/path-only' } as never)), + ).rejects.toThrow(/baseRef/) }) it('fails loud when judge-derived findings try to steer proposals', async () => { diff --git a/tests/campaign/presets.test.ts b/tests/campaign/presets.test.ts index fd080065..9b6edd3e 100644 --- a/tests/campaign/presets.test.ts +++ b/tests/campaign/presets.test.ts @@ -28,6 +28,24 @@ import { surfaceContentHash, surfaceHash, } from '../../src/campaign/index' + +function fakeCodeSurface(worktreeRef: string, identityDigit: string): CodeSurface { + return { + kind: 'code', + worktreeRef, + baseRef: 'main', + baseCommit: 'a'.repeat(40), + baseTree: 'b'.repeat(40), + candidateCommit: identityDigit.repeat(40), + candidateTree: identityDigit.repeat(40), + patch: { + format: 'git-diff-binary', + sha256: `sha256:${identityDigit.repeat(64)}`, + byteLength: 1, + }, + } +} + import type { RunRecord } from '../../src/run-record' interface FakeScenario extends Scenario { @@ -481,7 +499,7 @@ describe('loop provenance emission (transaction-extraction shape, offline)', () // (4) the structured record + OTel spans are DURABLE (written to storage). expect(storage.read(recordPath)).toBeDefined() - expect(JSON.parse(storage.read(recordPath)!).schema).toBe('tangle.loop-provenance.v1') + expect(JSON.parse(storage.read(recordPath)!).schema).toBe('tangle.loop-provenance.v2') const spanLines = storage.read(spansPath)!.split('\n') expect(spanLines.length).toBe(spans.length) // Spans pivot on the OTLP-ingestable tangle.* attributes the otel adapter reads. @@ -1160,17 +1178,23 @@ describe('MutableSurface widening', () => { expect(surfaceHash('hello')).not.toBe(surfaceHash('world')) }) - it('surfaceHash distinguishes code surfaces by worktree + base ref', () => { - const a: CodeSurface = { kind: 'code', worktreeRef: '/wt/a', baseRef: 'main' } - const b: CodeSurface = { kind: 'code', worktreeRef: '/wt/b', baseRef: 'main' } - const aAgain: CodeSurface = { - kind: 'code', - worktreeRef: '/wt/a', - baseRef: 'main', - summary: 'ignored in hash', + it('surfaceHash identifies code content independently of its worktree path', () => { + const a = fakeCodeSurface('/wt/a', '1') + const sameBytesElsewhere = fakeCodeSurface('/wt/b', '1') + const differentBytes = fakeCodeSurface('/wt/c', '2') + const aAgain: CodeSurface = { ...fakeCodeSurface('/wt/a', '1'), summary: 'ignored in hash' } + const reorderedPatch: CodeSurface = { + ...a, + patch: { + byteLength: a.patch.byteLength, + sha256: a.patch.sha256, + format: a.patch.format, + }, } expect(surfaceHash(a)).toBe(surfaceHash(aAgain)) // summary not part of identity - expect(surfaceHash(a)).not.toBe(surfaceHash(b)) + expect(surfaceHash(a)).toBe(surfaceHash(sameBytesElsewhere)) + expect(surfaceHash(a)).toBe(surfaceHash(reorderedPatch)) + expect(surfaceHash(a)).not.toBe(surfaceHash(differentBytes)) expect(surfaceHash(a)).toMatch(/^[a-f0-9]{16}$/) }) @@ -1188,9 +1212,10 @@ describe('MutableSurface widening', () => { }) { return new Array(populationSize).fill(0).map( (_, i): MutableSurface => ({ - kind: 'code', - worktreeRef: `/wt/gen${generation}-cand${i}`, - baseRef: 'main', + ...fakeCodeSurface( + `/wt/gen${generation}-cand${i}`, + (generation * populationSize + i + 1).toString(16), + ), summary: `candidate ${i}`, }), ) @@ -1203,7 +1228,7 @@ describe('MutableSurface widening', () => { const result = await runOptimization({ scenarios: SCENARIOS, - baselineSurface: { kind: 'code', worktreeRef: '/wt/main', baseRef: 'main' }, + baselineSurface: fakeCodeSurface('/wt/main', 'f'), dispatchWithSurface, proposer: codeProposer, populationSize: 2, diff --git a/tests/campaign/skill-opt.test.ts b/tests/campaign/skill-opt.test.ts index a27a6afb..02d90933 100644 --- a/tests/campaign/skill-opt.test.ts +++ b/tests/campaign/skill-opt.test.ts @@ -217,7 +217,7 @@ describe('skillOptProposer', () => { }) await expect( proposer.propose({ - currentSurface: { kind: 'code', worktreeRef: '/wt/a' }, + currentSurface: { kind: 'code', worktreeRef: '/wt/a' } as never, history: [], findings: [], populationSize: 1, diff --git a/tests/campaign/worktree.test.ts b/tests/campaign/worktree.test.ts index 4a724dba..1fa9f825 100644 --- a/tests/campaign/worktree.test.ts +++ b/tests/campaign/worktree.test.ts @@ -1,9 +1,25 @@ import { execFileSync } from 'node:child_process' -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { createHash } from 'node:crypto' +import { + chmodSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + unlinkSync, + writeFileSync, +} from 'node:fs' +import { devNull, tmpdir } from 'node:os' +import { basename, dirname, join } from 'node:path' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { gitWorktreeAdapter, resolveWorktreePath } from '../../src/campaign/worktree' +import { surfaceHash } from '../../src/campaign/presets/run-optimization' +import { surfaceContentHash } from '../../src/campaign/provenance' +import { + gitWorktreeAdapter, + resolveWorktreePath, + verifyCodeSurface, + WorktreeAdapterError, +} from '../../src/campaign/worktree' function git(args: string[], cwd: string): string { return execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() @@ -33,6 +49,8 @@ describe('gitWorktreeAdapter — real git worktrees', () => { expect(wt.branch).toMatch(/^improve\/tighten-the-rubric-/) expect(wt.baseRef).toBe('main') + expect(wt.baseCommit).toBe(git(['rev-parse', 'main'], repoRoot)) + expect(wt.baseTree).toBe(git(['rev-parse', 'main^{tree}'], repoRoot)) // The worktree is a real checkout with the baseline content. expect(git(['rev-parse', '--abbrev-ref', 'HEAD'], wt.path)).toBe(wt.branch) const worktrees = git(['worktree', 'list'], repoRoot) @@ -50,6 +68,39 @@ describe('gitWorktreeAdapter — real git worktrees', () => { expect(surface.kind).toBe('code') expect(surface.worktreeRef).toBe(wt.path) expect(surface.baseRef).toBe('main') + expect(surface.baseCommit).toBe(git(['rev-parse', 'main'], repoRoot)) + expect(surface.baseTree).toBe(git(['rev-parse', 'main^{tree}'], repoRoot)) + expect(surface.candidateCommit).toBe(git(['rev-parse', 'HEAD'], wt.path)) + expect(surface.candidateTree).toBe(git(['rev-parse', 'HEAD^{tree}'], wt.path)) + const patch = execFileSync( + 'git', + [ + 'diff', + '--unified=3', + '--inter-hunk-context=0', + `-O${devNull}`, + '--binary', + '--full-index', + '--no-color', + '--no-ext-diff', + '--no-textconv', + '--no-renames', + '--diff-algorithm=myers', + '--no-indent-heuristic', + '--src-prefix=a/', + '--dst-prefix=b/', + surface.baseCommit, + surface.candidateCommit, + '--', + ], + { cwd: wt.path }, + ) + expect(surface.patch).toEqual({ + format: 'git-diff-binary', + sha256: `sha256:${createHash('sha256').update(patch).digest('hex')}`, + byteLength: patch.byteLength, + }) + expect(Buffer.from(verifyCodeSurface(surface).patchBytes)).toEqual(patch) expect(surface.summary).toBe('improve the prompt') // The change is committed on the branch (one commit beyond base). const log = git(['log', '--oneline', 'main..HEAD'], wt.path) @@ -61,9 +112,432 @@ describe('gitWorktreeAdapter — real git worktrees', () => { it('finalize on a clean worktree makes no empty commit', async () => { const adapter = gitWorktreeAdapter({ repoRoot }) const wt = await adapter.create({ baseRef: 'main', label: 'noop' }) - await adapter.finalize(wt, 'no changes') + const surface = await adapter.finalize(wt, 'no changes') // No commits beyond base — git refuses empty commits and we don't force one. expect(git(['log', '--oneline', 'main..HEAD'], wt.path)).toBe('') + expect(surface.candidateCommit).toBe(surface.baseCommit) + expect(surface.patch.byteLength).toBe(0) + expect(surface.patch.sha256).toBe( + `sha256:${createHash('sha256').update(Buffer.alloc(0)).digest('hex')}`, + ) + expect(verifyCodeSurface(surface).path).toBe(wt.path) + }) + + it('finalizes and verifies SHA-256 repositories', async () => { + const shaRoot = mkdtempSync(join(tmpdir(), 'wt-sha256-repo-')) + try { + git(['init', '-q', '-b', 'main', '--object-format=sha256'], shaRoot) + git(['config', 'user.email', 'test@test.dev'], shaRoot) + git(['config', 'user.name', 'Test'], shaRoot) + const emptyHooks = join(shaRoot, '.empty-hooks') + mkdirSync(emptyHooks) + git(['config', 'core.hooksPath', emptyHooks], shaRoot) + writeFileSync(join(shaRoot, 'prompt.txt'), 'baseline prompt\n') + git(['add', '-A'], shaRoot) + git(['commit', '-q', '-m', 'init'], shaRoot) + + const adapter = gitWorktreeAdapter({ repoRoot: shaRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'sha256' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate prompt\n') + const surface = await adapter.finalize(wt, 'sha256 candidate') + + expect(surface.baseCommit).toMatch(/^[a-f0-9]{64}$/) + expect(surface.candidateCommit).toMatch(/^[a-f0-9]{64}$/) + expect(verifyCodeSurface(surface).path).toBe(wt.path) + } finally { + rmSync(shaRoot, { recursive: true, force: true }) + } + }) + + it('hashes candidate bytes, not worktree paths or commit metadata', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const a = await adapter.create({ baseRef: 'main', label: 'same-a' }) + const b = await adapter.create({ baseRef: 'main', label: 'same-b' }) + writeFileSync(join(a.path, 'prompt.txt'), 'identical candidate bytes\n') + writeFileSync(join(b.path, 'prompt.txt'), 'identical candidate bytes\n') + + const surfaceA = await adapter.finalize(a, 'candidate message A') + const surfaceB = await adapter.finalize(b, 'candidate message B') + + expect(surfaceA.worktreeRef).not.toBe(surfaceB.worktreeRef) + expect(surfaceA.candidateCommit).not.toBe(surfaceB.candidateCommit) + expect(surfaceA.patch).toEqual(surfaceB.patch) + expect(surfaceA.candidateTree).toBe(surfaceB.candidateTree) + expect(surfaceContentHash(surfaceA)).toBe(surfaceContentHash(surfaceB)) + expect(surfaceHash(surfaceA)).toBe(surfaceHash(surfaceB)) + }) + + it('pins patch bytes across repository and global diff config', async () => { + const priorHome = process.env.HOME + const priorXdgConfigHome = process.env.XDG_CONFIG_HOME + const isolatedHome = mkdtempSync(join(tmpdir(), 'wt-git-home-')) + process.env.HOME = isolatedHome + process.env.XDG_CONFIG_HOME = isolatedHome + try { + const baseline = Buffer.concat( + Array.from({ length: 1024 }, (_, i) => Buffer.from(`record-${i % 17}\0`.repeat(32))), + ) + const candidate = Buffer.from(baseline) + for (let i = 0; i < candidate.length; i += 4096) { + candidate[i] = ((candidate[i] ?? 0) + 1) & 0xff + } + writeFileSync(join(repoRoot, 'fixture.bin'), baseline) + writeFileSync( + join(repoRoot, 'context.txt'), + ['GLOBAL heading', 'one', '', 'two', '', 'three', 'LOCAL heading', 'tail', ''].join('\n'), + ) + writeFileSync(join(repoRoot, '.gitattributes'), 'context.txt diff=fixture\n') + git(['add', 'fixture.bin'], repoRoot) + git(['add', 'context.txt', '.gitattributes'], repoRoot) + git(['commit', '-q', '-m', 'add binary fixture'], repoRoot) + + const adapter = gitWorktreeAdapter({ repoRoot }) + const globalOrder = join(isolatedHome, 'global-order') + const repoOrder = join(isolatedHome, 'repo-order') + writeFileSync(globalOrder, 'fixture.bin\ncontext.txt\n') + writeFileSync(repoOrder, 'context.txt\nfixture.bin\n') + git(['config', '--global', 'core.compression', '9'], repoRoot) + git(['config', '--global', 'diff.context', '20'], repoRoot) + git(['config', '--global', 'diff.interHunkContext', '20'], repoRoot) + git(['config', '--global', 'diff.suppressBlankEmpty', 'true'], repoRoot) + git(['config', '--global', 'diff.orderFile', globalOrder], repoRoot) + git(['config', '--global', 'diff.fixture.xfuncname', '^GLOBAL'], repoRoot) + git(['config', 'core.compression', '1'], repoRoot) + git(['config', 'diff.context', '0'], repoRoot) + git(['config', 'diff.interHunkContext', '0'], repoRoot) + git(['config', 'diff.suppressBlankEmpty', 'false'], repoRoot) + git(['config', 'diff.orderFile', repoOrder], repoRoot) + git(['config', 'diff.fixture.xfuncname', '^LOCAL'], repoRoot) + const repoConfigured = await adapter.create({ baseRef: 'main', label: 'repo-config' }) + writeFileSync(join(repoConfigured.path, 'fixture.bin'), candidate) + writeFileSync( + join(repoConfigured.path, 'context.txt'), + ['GLOBAL heading', 'ONE', '', 'two', '', 'THREE', 'LOCAL heading', 'tail', ''].join('\n'), + ) + const surfaceA = await adapter.finalize(repoConfigured, 'binary candidate A') + + for (const key of [ + 'core.compression', + 'diff.context', + 'diff.interHunkContext', + 'diff.suppressBlankEmpty', + 'diff.orderFile', + 'diff.fixture.xfuncname', + ]) { + git(['config', '--unset', key], repoRoot) + } + const globalConfigured = await adapter.create({ + baseRef: 'main', + label: 'global-config', + }) + writeFileSync(join(globalConfigured.path, 'fixture.bin'), candidate) + writeFileSync( + join(globalConfigured.path, 'context.txt'), + ['GLOBAL heading', 'ONE', '', 'two', '', 'THREE', 'LOCAL heading', 'tail', ''].join('\n'), + ) + const surfaceB = await adapter.finalize(globalConfigured, 'binary candidate B') + + expect(surfaceA.patch).toEqual(surfaceB.patch) + expect(surfaceContentHash(surfaceA)).toBe(surfaceContentHash(surfaceB)) + + git(['config', 'core.compression', '6'], repoRoot) + git(['config', 'diff.context', '10'], repoRoot) + git(['config', 'diff.interHunkContext', '10'], repoRoot) + git(['config', 'diff.suppressBlankEmpty', 'true'], repoRoot) + git(['config', 'diff.orderFile', repoOrder], repoRoot) + git(['config', 'diff.fixture.xfuncname', '^LOCAL'], repoRoot) + expect(() => verifyCodeSurface(surfaceA)).not.toThrow() + expect(() => verifyCodeSurface(surfaceB)).not.toThrow() + } finally { + if (priorHome === undefined) delete process.env.HOME + else process.env.HOME = priorHome + if (priorXdgConfigHome === undefined) delete process.env.XDG_CONFIG_HOME + else process.env.XDG_CONFIG_HOME = priorXdgConfigHome + rmSync(isolatedHome, { recursive: true, force: true }) + } + }) + + it('changes both content hashes when one candidate byte changes', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const a = await adapter.create({ baseRef: 'main', label: 'bytes-a' }) + const b = await adapter.create({ baseRef: 'main', label: 'bytes-b' }) + writeFileSync(join(a.path, 'prompt.txt'), 'candidate A\n') + writeFileSync(join(b.path, 'prompt.txt'), 'candidate B\n') + + const surfaceA = await adapter.finalize(a, 'candidate') + const surfaceB = await adapter.finalize(b, 'candidate') + + expect(surfaceA.patch.sha256).not.toBe(surfaceB.patch.sha256) + expect(surfaceContentHash(surfaceA)).not.toBe(surfaceContentHash(surfaceB)) + expect(surfaceHash(surfaceA)).not.toBe(surfaceHash(surfaceB)) + }) + + it('rejects tracked mutation after finalization before resolving the path', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'mutated' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'finalized bytes\n') + const surface = await adapter.finalize(wt, 'finalize') + + writeFileSync(join(wt.path, 'prompt.txt'), 'mutated after finalize\n') + + expect(() => verifyCodeSurface(surface)).toThrow(/changed after finalization/) + expect(() => resolveWorktreePath(surface)).toThrow(/changed after finalization/) + }) + + it('rejects assume-unchanged entries that can hide tracked mutations', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'hidden-mutation' }) + const surface = await adapter.finalize(wt, 'no-op') + git(['update-index', '--assume-unchanged', 'prompt.txt'], wt.path) + writeFileSync(join(wt.path, 'prompt.txt'), 'hidden mutation\n') + + expect(() => verifyCodeSurface(surface)).toThrow(/hidden index entries/) + }) + + it('rejects raw tracked bytes hidden by a Git clean filter', async () => { + git(['config', 'filter.hide.clean', "sed 's/^evil$/safe/'"], repoRoot) + git(['config', 'filter.hide.smudge', 'cat'], repoRoot) + writeFileSync(join(repoRoot, '.gitattributes'), 'filtered.txt filter=hide\n') + writeFileSync(join(repoRoot, 'filtered.txt'), 'safe\n') + git(['add', '-A'], repoRoot) + git(['commit', '-q', '-m', 'add filtered file'], repoRoot) + + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'clean-filter-bypass' }) + const surface = await adapter.finalize(wt, 'no-op') + writeFileSync(join(wt.path, 'filtered.txt'), 'evil\n') + + // Git reports the checkout clean because the configured filter maps the + // changed bytes back to the committed blob. Verification must read raw + // filesystem bytes instead of trusting that filtered comparison. + expect(git(['status', '--porcelain=v1'], wt.path)).toBe('') + expect(() => verifyCodeSurface(surface)).toThrow(/raw tracked bytes differ/) + }) + + it('rejects checkout bytes rewritten by a Git smudge filter', async () => { + git(['config', 'filter.rewrite.clean', "sed 's/^evil$/safe/'"], repoRoot) + git(['config', 'filter.rewrite.smudge', "sed 's/^safe$/evil/'"], repoRoot) + writeFileSync(join(repoRoot, '.gitattributes'), 'filtered.txt filter=rewrite\n') + writeFileSync(join(repoRoot, 'filtered.txt'), 'safe\n') + git(['add', '-A'], repoRoot) + git(['commit', '-q', '-m', 'add smudged file'], repoRoot) + + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'smudge-filter-bypass' }) + + expect(git(['status', '--porcelain=v1'], wt.path)).toBe('') + await expect(adapter.finalize(wt, 'no-op')).rejects.toThrow(/raw tracked bytes differ/) + }) + + it('rejects executable-mode changes hidden by core.filemode=false', async () => { + if (process.platform === 'win32') return + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'hidden-mode' }) + const surface = await adapter.finalize(wt, 'no-op') + git(['config', 'core.filemode', 'false'], wt.path) + chmodSync(join(wt.path, 'prompt.txt'), 0o755) + + expect(git(['status', '--porcelain=v1'], wt.path)).toBe('') + expect(() => verifyCodeSurface(surface)).toThrow(/executable mode differs/) + }) + + it('rejects a symbolic worktree locator', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'linked-locator' }) + const surface = await adapter.finalize(wt, 'no-op') + const linkedPath = `${wt.path}-link` + symlinkSync(wt.path, linkedPath, 'dir') + try { + expect(() => verifyCodeSurface({ ...surface, worktreeRef: linkedPath })).toThrow( + /locator must not contain a symbolic link/, + ) + } finally { + unlinkSync(linkedPath) + } + }) + + it('rejects a worktree locator that traverses a symbolic-link parent', async () => { + if (process.platform === 'win32') return + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'linked-parent' }) + const surface = await adapter.finalize(wt, 'no-op') + const linkedParent = `${dirname(wt.path)}-link` + symlinkSync(dirname(wt.path), linkedParent, 'dir') + try { + expect(() => + verifyCodeSurface({ + ...surface, + worktreeRef: join(linkedParent, basename(wt.path)), + }), + ).toThrow(/locator must not contain a symbolic link/) + } finally { + unlinkSync(linkedParent) + } + }) + + it('rejects a tracked symbolic link that escapes the worktree', async () => { + const outside = join(repoRoot, '..', `${basename(repoRoot)}-outside.txt`) + writeFileSync(outside, 'outside bytes\n') + symlinkSync(outside, join(repoRoot, 'outside-link')) + git(['add', 'outside-link'], repoRoot) + git(['commit', '-q', '-m', 'add external link'], repoRoot) + + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'external-link' }) + try { + await expect(adapter.finalize(wt, 'no-op')).rejects.toThrow(/symbolic link escapes/) + } finally { + rmSync(outside, { force: true }) + } + }) + + it('rejects a symbolic link that leaves and re-enters through untracked content', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'external-trampoline' }) + const outside = `${wt.path}-outside` + mkdirSync(outside) + symlinkSync(join(wt.path, 'prompt.txt'), join(outside, 'back')) + symlinkSync(`../${basename(outside)}/back`, join(wt.path, 'trampoline')) + + await expect(adapter.finalize(wt, 'add external trampoline')).rejects.toThrow( + /symbolic link escapes/, + ) + }) + + it('rejects Git submodules whose checked-out bytes are outside the tree identity', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'submodule' }) + const nested = join(wt.path, 'vendor', 'nested') + mkdirSync(nested, { recursive: true }) + git(['init', '-q', '-b', 'main'], nested) + git(['config', 'user.email', 'test@test.dev'], nested) + git(['config', 'user.name', 'Test'], nested) + const nestedHooks = join(nested, '.empty-hooks') + mkdirSync(nestedHooks) + git(['config', 'core.hooksPath', nestedHooks], nested) + writeFileSync(join(nested, 'module.txt'), 'module bytes\n') + git(['add', '-A'], nested) + git(['commit', '-q', '-m', 'nested'], nested) + + await expect(adapter.finalize(wt, 'add embedded repository')).rejects.toThrow( + /Git submodule.*not bound/, + ) + }) + + it('rejects an ignored file instead of evaluating bytes outside Git identity', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'ignored-dirty' }) + writeFileSync(join(wt.path, '.gitignore'), '*.cache\n') + writeFileSync(join(wt.path, 'agent.cache'), 'mutable execution input\n') + + await expect(adapter.finalize(wt, 'add ignore rule')).rejects.toThrow( + /changed after finalization.*agent\.cache/s, + ) + }) + + it('rejects a surface pointed at the wrong finalized ref', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const a = await adapter.create({ baseRef: 'main', label: 'wrong-ref-a' }) + const b = await adapter.create({ baseRef: 'main', label: 'wrong-ref-b' }) + writeFileSync(join(a.path, 'prompt.txt'), 'candidate A\n') + writeFileSync(join(b.path, 'prompt.txt'), 'candidate B\n') + const surfaceA = await adapter.finalize(a, 'candidate A') + await adapter.finalize(b, 'candidate B') + + expect(() => verifyCodeSurface({ ...surfaceA, worktreeRef: b.path })).toThrow( + /candidate commit mismatch/, + ) + }) + + it('rejects a forged patch digest even when the checkout is unchanged', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'forged-digest' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate\n') + const surface = await adapter.finalize(wt, 'candidate') + + expect(() => + verifyCodeSurface({ + ...surface, + patch: { ...surface.patch, sha256: `sha256:${'0'.repeat(64)}` }, + }), + ).toThrow(/patch mismatch/) + }) + + it('rejects a surface locator pointed at another repository', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'right-repo' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate\n') + const surface = await adapter.finalize(wt, 'candidate') + + const otherRepo = mkdtempSync(join(tmpdir(), 'wt-wrong-repo-')) + try { + git(['init', '-q', '-b', 'main'], otherRepo) + git(['config', 'user.email', 'test@test.dev'], otherRepo) + git(['config', 'user.name', 'Test'], otherRepo) + const emptyHooks = join(otherRepo, '.empty-hooks') + mkdirSync(emptyHooks) + git(['config', 'core.hooksPath', emptyHooks], otherRepo) + writeFileSync(join(otherRepo, 'prompt.txt'), 'unrelated\n') + git(['add', '-A'], otherRepo) + git(['commit', '-q', '-m', 'unrelated'], otherRepo) + + expect(() => verifyCodeSurface({ ...surface, worktreeRef: otherRepo })).toThrow( + /candidate commit mismatch/, + ) + } finally { + rmSync(otherRepo, { recursive: true, force: true }) + } + }) + + it('does not let inherited Git environment redirect verification', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'env-right-repo' }) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate\n') + const surface = await adapter.finalize(wt, 'candidate') + + const otherRepo = mkdtempSync(join(tmpdir(), 'wt-env-wrong-repo-')) + const priorGitDir = process.env.GIT_DIR + const priorGitWorkTree = process.env.GIT_WORK_TREE + try { + git(['init', '-q', '-b', 'main'], otherRepo) + git(['config', 'user.email', 'test@test.dev'], otherRepo) + git(['config', 'user.name', 'Test'], otherRepo) + const emptyHooks = join(otherRepo, '.empty-hooks') + mkdirSync(emptyHooks) + git(['config', 'core.hooksPath', emptyHooks], otherRepo) + writeFileSync(join(otherRepo, 'prompt.txt'), 'unrelated\n') + git(['add', '-A'], otherRepo) + git(['commit', '-q', '-m', 'unrelated'], otherRepo) + + process.env.GIT_DIR = git(['rev-parse', '--absolute-git-dir'], wt.path) + process.env.GIT_WORK_TREE = wt.path + + expect(() => verifyCodeSurface({ ...surface, worktreeRef: otherRepo })).toThrow( + /candidate commit mismatch/, + ) + } finally { + if (priorGitDir === undefined) delete process.env.GIT_DIR + else process.env.GIT_DIR = priorGitDir + if (priorGitWorkTree === undefined) delete process.env.GIT_WORK_TREE + else process.env.GIT_WORK_TREE = priorGitWorkTree + rmSync(otherRepo, { recursive: true, force: true }) + } + }) + + it('keeps the exact fork commit when the human base ref advances', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'moving-base' }) + const originalBase = wt.baseCommit + + writeFileSync(join(repoRoot, 'later.txt'), 'main advanced\n') + git(['add', 'later.txt'], repoRoot) + git(['commit', '-q', '-m', 'advance main'], repoRoot) + writeFileSync(join(wt.path, 'prompt.txt'), 'candidate from original base\n') + + const surface = await adapter.finalize(wt, 'candidate') + expect(surface.baseCommit).toBe(originalBase) + expect(surface.baseCommit).not.toBe(git(['rev-parse', 'main'], repoRoot)) + expect(verifyCodeSurface(surface).path).toBe(wt.path) }) it('discard removes the worktree and its branch', async () => { @@ -95,18 +569,16 @@ describe('gitWorktreeAdapter — real git worktrees', () => { it('surfaces a typed error when git fails', async () => { const adapter = gitWorktreeAdapter({ repoRoot }) await expect(adapter.create({ baseRef: 'no-such-ref', label: 'x' })).rejects.toThrow( - /git worktree add .* failed/, + WorktreeAdapterError, ) }) }) describe('resolveWorktreePath', () => { - it('returns an existing absolute path as-is', () => { - const p = mkdtempSync(join(tmpdir(), 'wt-resolve-')) - try { - expect(resolveWorktreePath({ kind: 'code', worktreeRef: p })).toBe(p) - } finally { - rmSync(p, { recursive: true, force: true }) - } + it('returns a verified absolute worktree path', async () => { + const adapter = gitWorktreeAdapter({ repoRoot }) + const wt = await adapter.create({ baseRef: 'main', label: 'resolve' }) + const surface = await adapter.finalize(wt, 'no-op') + expect(resolveWorktreePath(surface)).toBe(wt.path) }) }) diff --git a/tests/contract-self-improve.test.ts b/tests/contract-self-improve.test.ts index 9089c3ec..1e1febc2 100644 --- a/tests/contract-self-improve.test.ts +++ b/tests/contract-self-improve.test.ts @@ -142,7 +142,7 @@ describe('selfImprove provenance emission (durable by default)', () => { // (3) the provenance record is on the result AND fired through onProvenance. expect(captured).toBeDefined() - expect(result.provenance.schema).toBe('tangle.loop-provenance.v1') + expect(result.provenance.schema).toBe('tangle.loop-provenance.v2') expect(result.provenance.winnerRationale).toBe(RATIONALE) // real content hashes distinguish baseline from winner + verify bytes. @@ -181,7 +181,7 @@ describe('selfImprove provenance emission (durable by default)', () => { // no runDir ⇒ mem://… ⇒ in-memory storage ⇒ nothing on disk }) // The provenance record is still produced in-memory + on the result. - expect(result.provenance.schema).toBe('tangle.loop-provenance.v1') + expect(result.provenance.schema).toBe('tangle.loop-provenance.v2') expect(result.provenance.runDir.startsWith('mem://')).toBe(true) }) })