From 4b0a0aa62c6dbfbbdf308bced2900081037abb5a Mon Sep 17 00:00:00 2001 From: Christopher Date: Sat, 27 Jun 2026 08:44:12 +1000 Subject: [PATCH] test(eval): guard workspace path override compatibility --- apps/cli/test/eval.integration.test.ts | 25 +++++++++++++ apps/cli/test/fixtures/mock-run-evaluation.ts | 4 +++ .../test/evaluation/workspace/setup.test.ts | 36 +++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index 484095448..2f75409e6 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -505,6 +505,31 @@ describe('agentv eval CLI', () => { } }, 30_000); + it('keeps --workspace-path as a static workspace override for existing workspaces', async () => { + const fixture = await createFixture(); + try { + const workspacePath = path.join(fixture.baseDir, 'prepared-workspace'); + await mkdir(workspacePath, { recursive: true }); + + const result = await runCli(fixture, [ + 'eval', + fixture.testFilePath, + '--workspace-path', + workspacePath, + ]); + + expect(result.exitCode).toBe(0); + const diagnostics = await readDiagnostics(fixture); + expect(diagnostics).toMatchObject({ + workspaceMode: 'static', + workspacePath, + resultCount: 2, + }); + } finally { + await rm(fixture.baseDir, { recursive: true, force: true }); + } + }, 30_000); + it('passes run-level budget tracking through to the evaluator', async () => { const fixture = await createFixture(); try { diff --git a/apps/cli/test/fixtures/mock-run-evaluation.ts b/apps/cli/test/fixtures/mock-run-evaluation.ts index b7ce3515f..0099f5571 100644 --- a/apps/cli/test/fixtures/mock-run-evaluation.ts +++ b/apps/cli/test/fixtures/mock-run-evaluation.ts @@ -19,6 +19,8 @@ interface RunEvaluationOptionsLike { readonly evalCases?: ReadonlyArray; readonly verbose?: boolean; readonly maxConcurrency?: number; + readonly workspaceMode?: string; + readonly workspacePath?: string; readonly trials?: { readonly count: number; readonly strategy: string; @@ -181,6 +183,8 @@ async function maybeWriteDiagnostics( envLocalOnly: process.env.CLI_ENV_LOCAL_ONLY ?? null, budgetUsd: options.budgetUsd ?? null, maxConcurrency: options.maxConcurrency ?? null, + workspaceMode: options.workspaceMode ?? null, + workspacePath: options.workspacePath ?? null, trials: options.trials ?? null, threshold: options.threshold ?? null, hasRunBudgetTracker: options.runBudgetTracker !== undefined, diff --git a/packages/core/test/evaluation/workspace/setup.test.ts b/packages/core/test/evaluation/workspace/setup.test.ts index b5f61790e..6c7160ddc 100644 --- a/packages/core/test/evaluation/workspace/setup.test.ts +++ b/packages/core/test/evaluation/workspace/setup.test.ts @@ -113,4 +113,40 @@ describe('prepareSharedWorkspaceSetup', () => { expect(readFileSync(path.join(repoDir, 'tracked.txt'), 'utf8')).toBe('clean\n'); expect(existsSync(path.join(repoDir, 'stale.txt'))).toBe(false); }, 30_000); + + it('uses CLI workspacePath as an existing static workspace without materializing repos', async () => { + const existingWorkspace = path.join(tmpDir, 'existing-workspace'); + mkdirSync(existingWorkspace, { recursive: true }); + writeFileSync(path.join(existingWorkspace, 'marker.txt'), 'already prepared\n', 'utf8'); + + const evalCase: EvalTest = { + id: 'case-1', + question: 'test', + criteria: 'ok', + workspace: { + repos: [ + { + path: './repo-a', + repo: 'https://example.com/repo-a.git', + commit: 'main', + }, + ], + }, + }; + + setup = await prepareSharedWorkspaceSetup({ + evalRunId: 'test-cli-workspace-path', + evalCases: [evalCase], + evalDir: tmpDir, + workspacePath: existingWorkspace, + workers: 1, + }); + + expect(setup.sharedWorkspacePath).toBe(existingWorkspace); + expect(setup.repoManager).toBeUndefined(); + expect(readFileSync(path.join(existingWorkspace, 'marker.txt'), 'utf8')).toBe( + 'already prepared\n', + ); + expect(existsSync(path.join(existingWorkspace, 'repo-a'))).toBe(false); + }); });