fix(agent-providers): stabilize Copilot and Pi SDK runs

docs/learnings/best-practices/prefer-copilot-sdk-tcp-for-owned-runtimes.md

@@ -0,0 +1,62 @@
+---
+title: "Prefer Copilot SDK TCP transport for owned runtimes"
+module: copilot-sdk provider
+date: 2026-06-23
+problem_type: best_practice
+component: tooling
+severity: medium
+tags:
+  - copilot-sdk
+  - subprocess
+  - epipe
+  - runtime-lifecycle
+  - provider
+applies_when:
+  - Using @github/copilot-sdk to launch a local Copilot runtime process
+  - Seeing uncaught stdin EPIPE after a Copilot SDK session appears to finish
+  - Choosing between RuntimeConnection.forTcp and RuntimeConnection.forStdio
+---
+
+# Prefer Copilot SDK TCP transport for owned runtimes
+
+## Context
+
+AgentV's `copilot-sdk` provider can run against an external Copilot runtime URL or own
+the local Copilot runtime process for an eval invocation. When AgentV owns the runtime,
+`@github/copilot-sdk@1.0.3` offers both stdio and TCP runtime connections.
+
+During eval smoke testing, the stdio transport could complete the assistant turn and
+then crash the Node process with an uncaught child `stdin` `EPIPE`. The upstream SDK has
+a matching lifecycle issue: `github/copilot-sdk#1427`.
+
+## Guidance
+
+Prefer `RuntimeConnection.forTcp()` when AgentV launches or auto-resolves the local
+Copilot runtime. Keep `RuntimeConnection.forStdio()` only as backward compatibility for
+older SDK releases that do not expose TCP.
+
+This is not a reason to add global `uncaughtException` handlers or swallow all `EPIPE`
+errors. The narrow fix is to choose the SDK-supported transport that avoids the stdio
+stdin lifecycle edge case.
+
+## Why This Matters
+
+The provider result is only useful if AgentV can finish writing `index.jsonl`,
+per-case artifacts, and benchmark summaries. A post-turn stdio `EPIPE` can crash the
+process before result finalization even when the model already produced a usable answer.
+
+TCP also keeps lifecycle handling inside the SDK transport boundary. AgentV should not
+reach into private SDK fields such as `forceStopping`, and it should not install
+process-wide exception filters for one provider's subprocess pipe behavior.
+
+## When to Apply
+
+- Local owned Copilot runtime: use `RuntimeConnection.forTcp({ path, args })`.
+- External Copilot runtime URL: use `RuntimeConnection.forUri(url)`.
+- Old SDK without TCP support: fall back to `RuntimeConnection.forStdio({ path, args })`.
+
+## Related
+
+- `packages/core/src/evaluation/providers/copilot-sdk.ts` — runtime connection selection
+- `packages/core/test/evaluation/providers/copilot-sdk.test.ts` — TCP/URI constructor coverage
+- Upstream issue: `github/copilot-sdk#1427`

package.json

@@ -19,6 +19,7 @@
     "agentv": "bun apps/cli/src/cli.ts",
     "agentv:buildrun": "bun run build && bun apps/cli/dist/cli.js",
     "beads:check": "bun scripts/check-beads-context.ts",
+    "debug:pi-sdk-tools": "bun scripts/debug-pi-sdk-tools.ts",
     "validate:examples": "EVAL_CRITERIA=placeholder CUSTOM_SYSTEM_PROMPT=placeholder bun scripts/validate-example-evals.ts",
     "eval:baseline-check": "bun scripts/check-eval-baselines.ts",
     "release": "bun scripts/release.ts",

packages/core/src/evaluation/providers/copilot-sdk.ts

@@ -356,7 +356,12 @@ export class CopilotSdkProvider implements Provider {
       }
 
       if (!clientOptions.connection && (this.config.cliPath || this.config.args?.length)) {
-        if (sdk.RuntimeConnection?.forStdio) {
+        if (sdk.RuntimeConnection?.forTcp) {
+          clientOptions.connection = sdk.RuntimeConnection.forTcp({
+            ...(this.config.cliPath ? { path: this.config.cliPath } : {}),
+            ...(this.config.args?.length ? { args: this.config.args } : {}),
+          });
+        } else if (sdk.RuntimeConnection?.forStdio) {
           clientOptions.connection = sdk.RuntimeConnection.forStdio({
             ...(this.config.cliPath ? { path: this.config.cliPath } : {}),
             ...(this.config.args?.length ? { args: this.config.args } : {}),
@@ -369,7 +374,16 @@ export class CopilotSdkProvider implements Provider {
         // node:sqlite (unavailable in Bun). Auto-resolve the platform-specific native
         // binary from @github/copilot-{platform}-{arch} when available.
         const nativePath = resolvePlatformCliPath();
-        if (nativePath && sdk.RuntimeConnection?.forStdio && !clientOptions.connection) {
+        if (nativePath && sdk.RuntimeConnection?.forTcp && !clientOptions.connection) {
+          // Prefer the SDK's supported TCP transport for owned runtimes. In
+          // @github/copilot-sdk@1.0.3 the stdio transport can rethrow child
+          // stdin EPIPE as an uncaughtException when the runtime exits outside
+          // forceStop(); see github/copilot-sdk#1427.
+          clientOptions.connection = sdk.RuntimeConnection.forTcp({
+            path: nativePath,
+            ...(this.config.args?.length ? { args: this.config.args } : {}),
+          });
+        } else if (nativePath && sdk.RuntimeConnection?.forStdio && !clientOptions.connection) {
           clientOptions.connection = sdk.RuntimeConnection.forStdio({
             path: nativePath,
             ...(this.config.args?.length ? { args: this.config.args } : {}),

packages/core/src/evaluation/providers/pi-coding-agent.ts

@@ -44,6 +44,8 @@ let piCodingAgentModule: typeof import('@earendil-works/pi-coding-agent') | null
 let piAiModule: typeof import('@earendil-works/pi-ai') | null = null;
 let loadingPromise: Promise<void> | null = null;
 
+const PI_BUILT_IN_TOOL_NAMES = new Set(['read', 'bash', 'edit', 'write', 'grep', 'find', 'ls']);
+
 async function promptInstall(): Promise<boolean> {
   if (!process.stdout.isTTY) return false;
   const rl = createInterface({ input: process.stdin, output: process.stdout });
@@ -240,19 +242,8 @@ async function loadSdkModules() {
   // After doLoadSdkModules resolves, both modules are guaranteed non-null.
   const piSdk = piCodingAgentModule as NonNullable<typeof piCodingAgentModule>;
   const piAi = piAiModule as NonNullable<typeof piAiModule>;
-  const toolMap: Record<string, unknown> = {
-    read: piSdk.readTool,
-    bash: piSdk.bashTool,
-    edit: piSdk.editTool,
-    write: piSdk.writeTool,
-    grep: piSdk.grepTool,
-    find: piSdk.findTool,
-    ls: piSdk.lsTool,
-  };
   return {
     createAgentSession: piSdk.createAgentSession,
-    codingTools: piSdk.codingTools,
-    toolMap,
     SessionManager: piSdk.SessionManager,
     getModel: piAi.getModel,
     // biome-ignore lint/suspicious/noExplicitAny: registerBuiltInApiProviders exists at runtime but not in type defs
@@ -337,7 +328,7 @@ export class PiCodingAgentProvider implements Provider {
       }
 
       // Select tools based on config
-      const tools = this.resolveTools(sdk);
+      const tools = resolvePiToolNames(this.config.tools);
 
       // Create agent session using the SDK
       const { session } = await sdk.createAgentSession({
@@ -575,22 +566,6 @@ export class PiCodingAgentProvider implements Provider {
     return process.cwd();
   }
 
-  private resolveTools(sdk: Awaited<ReturnType<typeof loadSdkModules>>) {
-    if (!this.config.tools) {
-      return sdk.codingTools;
-    }
-
-    const toolNames = this.config.tools.split(',').map((t) => t.trim().toLowerCase());
-    const selected = [];
-    for (const name of toolNames) {
-      if (name in sdk.toolMap) {
-        selected.push(sdk.toolMap[name]);
-      }
-    }
-    // biome-ignore lint/suspicious/noExplicitAny: tools are typed dynamically from SDK
-    return selected.length > 0 ? (selected as any[]) : sdk.codingTools;
-  }
-
   private resolveLogDirectory(request: ProviderRequest): string | undefined {
     if (this.config.logDir) {
       return path.resolve(this.config.logDir);
@@ -691,6 +666,18 @@ class PiStreamLogger {
   }
 }
 
+function resolvePiToolNames(configTools?: string): readonly string[] | undefined {
+  if (!configTools) return undefined;
+
+  const selected = configTools
+    .split(',')
+    .map((tool) => tool.trim().toLowerCase())
+    .filter((tool) => PI_BUILT_IN_TOOL_NAMES.has(tool));
+
+  // Passing undefined lets the SDK use its default built-ins.
+  return selected.length > 0 ? selected : undefined;
+}
+
 function summarizeSdkEvent(event: unknown): string | undefined {
   if (!event || typeof event !== 'object') return undefined;
   const record = event as Record<string, unknown>;
@@ -846,4 +833,5 @@ export const _internal = {
   findAgentvRoot,
   findManagedSdkInstallRoot,
   resolveGlobalNpmRoot,
+  resolvePiToolNames,
 };

packages/core/test/evaluation/providers/copilot-sdk.test.ts

@@ -80,6 +80,20 @@ function mockCopilotSdk(client: MockClient) {
     CopilotClient: mock(function CopilotClient() {
       return client;
     }),
+    RuntimeConnection: {
+      forTcp: mock((options?: Record<string, unknown>) => ({
+        kind: 'tcp',
+        ...options,
+      })),
+      forStdio: mock((options?: Record<string, unknown>) => ({
+        kind: 'stdio',
+        ...options,
+      })),
+      forUri: mock((url: string) => ({
+        kind: 'uri',
+        url,
+      })),
+    },
   };
 }
 
@@ -144,17 +158,21 @@ describe('CopilotSdkProvider', () => {
     expect(sessionOptions.model).toBe('gpt-5');
   });
 
-  it('passes cliUrl to CopilotClient constructor', async () => {
+  it('passes cliUrl through RuntimeConnection.forUri', async () => {
     const session = createMockSession({
       events: [{ type: 'assistant.message', data: { content: 'response' } }],
     });
     const client = createMockClient(session);
+    const forUri = mock((url: string) => ({ kind: 'uri', url }));
 
     const CopilotClientMock = mock(function CopilotClient() {
       return client;
     });
     mock.module('@github/copilot-sdk', () => ({
       CopilotClient: CopilotClientMock,
+      RuntimeConnection: {
+        forUri,
+      },
     }));
 
     const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js');
@@ -166,20 +184,29 @@ describe('CopilotSdkProvider', () => {
     await provider.invoke({ question: 'Test' });
 
     const constructorArgs = CopilotClientMock.mock.calls[0][0];
-    expect(constructorArgs.cliUrl).toBe('http://localhost:9999');
+    expect(forUri).toHaveBeenCalledWith('http://localhost:9999');
+    expect(constructorArgs.connection).toEqual({
+      kind: 'uri',
+      url: 'http://localhost:9999',
+    });
+    expect(session.disconnect).toHaveBeenCalledTimes(1);
   });
 
-  it('passes args as cliArgs to CopilotClient constructor', async () => {
+  it('passes args to the local TCP runtime and legacy cliArgs constructor option', async () => {
     const session = createMockSession({
       events: [{ type: 'assistant.message', data: { content: 'response' } }],
     });
     const client = createMockClient(session);
+    const forTcp = mock((options?: Record<string, unknown>) => ({ kind: 'tcp', ...options }));
 
     const CopilotClientMock = mock(function CopilotClient() {
       return client;
     });
     mock.module('@github/copilot-sdk', () => ({
       CopilotClient: CopilotClientMock,
+      RuntimeConnection: {
+        forTcp,
+      },
     }));
 
     const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js');
@@ -191,6 +218,11 @@ describe('CopilotSdkProvider', () => {
     await provider.invoke({ question: 'Test' });
 
     const constructorArgs = CopilotClientMock.mock.calls[0][0];
+    expect(forTcp).toHaveBeenCalledWith({ args: ['--verbose', 'enabled'] });
+    expect(constructorArgs.connection).toEqual({
+      kind: 'tcp',
+      args: ['--verbose', 'enabled'],
+    });
     expect(constructorArgs.cliArgs).toEqual(['--verbose', 'enabled']);
   });
 
@@ -199,12 +231,16 @@ describe('CopilotSdkProvider', () => {
       events: [{ type: 'assistant.message', data: { content: 'response' } }],
     });
     const client = createMockClient(session);
+    const forTcp = mock((options?: Record<string, unknown>) => ({ kind: 'tcp', ...options }));
 
     const CopilotClientMock = mock(function CopilotClient() {
       return client;
     });
     mock.module('@github/copilot-sdk', () => ({
       CopilotClient: CopilotClientMock,
+      RuntimeConnection: {
+        forTcp,
+      },
     }));
 
     const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js');
@@ -219,6 +255,10 @@ describe('CopilotSdkProvider', () => {
     // cwd is set so the subprocess resolves relative paths itself — args are NOT pre-resolved
     expect(constructorArgs.cwd).toBe(path.resolve(fixturesRoot));
     expect(constructorArgs.workingDirectory).toBe(path.resolve(fixturesRoot));
+    expect(constructorArgs.connection).toEqual({
+      kind: 'tcp',
+      args: ['--plugin-dir', './plugins', '--shared-dir', '../shared', '--mode', 'agent'],
+    });
     expect(constructorArgs.cliArgs).toEqual([
       '--plugin-dir',
       './plugins',
@@ -302,22 +342,28 @@ describe('CopilotSdkProvider', () => {
     );
   });
 
-  it('reuses client across multiple invocations', async () => {
+  it('reuses external client across multiple invocations', async () => {
     const session = createMockSession({
       events: [{ type: 'assistant.message', data: { content: 'response' } }],
     });
     const client = createMockClient(session);
+    const forUri = mock((url: string) => ({ kind: 'uri', url }));
 
     const CopilotClientMock = mock(function CopilotClient() {
       return client;
     });
     mock.module('@github/copilot-sdk', () => ({
       CopilotClient: CopilotClientMock,
+      RuntimeConnection: {
+        forUri,
+      },
     }));
 
     const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js');
 
-    const provider = new CopilotSdkProvider('test-target', {});
+    const provider = new CopilotSdkProvider('test-target', {
+      cliUrl: 'http://localhost:9999',
+    });
 
     await provider.invoke({ question: 'First' });
     await provider.invoke({ question: 'Second' });
@@ -326,29 +372,41 @@ describe('CopilotSdkProvider', () => {
     expect(CopilotClientMock).toHaveBeenCalledTimes(1);
     // But createSession should be called twice (fresh session per invocation)
     expect(client.createSession).toHaveBeenCalledTimes(2);
+    expect(session.disconnect).toHaveBeenCalledTimes(2);
+    expect(forUri).toHaveBeenCalledTimes(1);
   });
 
-  it('creates fresh session per invocation', async () => {
+  it('reuses local TCP client across multiple invocations', async () => {
     const session = createMockSession({
       events: [{ type: 'assistant.message', data: { content: 'response' } }],
     });
     const client = createMockClient(session);
-    const sdkMock = mockCopilotSdk(client);
+    const forTcp = mock((options?: Record<string, unknown>) => ({ kind: 'tcp', ...options }));
 
-    mock.module('@github/copilot-sdk', () => sdkMock);
+    const CopilotClientMock = mock(function CopilotClient() {
+      return client;
+    });
+    mock.module('@github/copilot-sdk', () => ({
+      CopilotClient: CopilotClientMock,
+      RuntimeConnection: {
+        forTcp,
+      },
+    }));
     const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js');
 
     const provider = new CopilotSdkProvider('test-target', {});
 
     await provider.invoke({ question: 'First' });
     await provider.invoke({ question: 'Second' });
 
-    // Session should be disconnected after each invocation
-    expect(session.disconnect).toHaveBeenCalledTimes(2);
+    expect(CopilotClientMock).toHaveBeenCalledTimes(1);
     expect(client.createSession).toHaveBeenCalledTimes(2);
+    expect(session.disconnect).toHaveBeenCalledTimes(2);
+    expect(forTcp).toHaveBeenCalledTimes(1);
+    expect(CopilotClientMock.mock.calls[0][0].connection.kind).toBe('tcp');
   });
 
-  it('falls back to destroy for older SDK sessions', async () => {
+  it('falls back to destroy for older external SDK sessions', async () => {
     const session = createMockSession({
       events: [{ type: 'assistant.message', data: { content: 'response' } }],
       legacyDestroyOnly: true,
@@ -359,7 +417,9 @@ describe('CopilotSdkProvider', () => {
     mock.module('@github/copilot-sdk', () => sdkMock);
     const { CopilotSdkProvider } = await import('../../../src/evaluation/providers/copilot-sdk.js');
 
-    const provider = new CopilotSdkProvider('test-target', {});
+    const provider = new CopilotSdkProvider('test-target', {
+      cliUrl: 'http://localhost:9999',
+    });
 
     await provider.invoke({ question: 'Test' });