feat: trace Portkey chat completions

.changeset/quiet-portkey-tracing.md

@@ -0,0 +1,5 @@
+---
+"braintrust": minor
+---
+
+Add automatic instrumentation and `wrapPortkey` for Portkey AI v3 chat completions, including streaming responses, token usage, tool calls, and errors.

js/package.json

@@ -212,6 +212,7 @@
     "eslint-plugin-n": "^18.2.1",
     "eslint-plugin-node-import": "^1.0.5",
     "openai": "6.25.0",
+    "portkey-ai": "3.1.0",
     "rollup": "^4.60.3",
     "tar": "^7.5.22",
     "tinybench": "^4.0.1",

js/src/auto-instrumentations/configs/all.ts

@@ -20,6 +20,7 @@ import { googleADKConfigs } from "./google-adk";
 import { googleGenerativeAIConfigs } from "./google-generative-ai";
 import { googleGenAIConfigs } from "./google-genai";
 import { groqConfigs } from "./groq";
+import { portkeyConfigs } from "./portkey";
 import { huggingFaceConfigs } from "./huggingface";
 import { huggingFaceTransformersConfigs } from "./huggingface-transformers";
 import { langchainConfigs } from "./langchain";
@@ -108,6 +109,7 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[]
     { integrations: ["googleADK"], configs: googleADKConfigs },
     { integrations: ["cohere"], configs: cohereConfigs },
     { integrations: ["groq"], configs: groqConfigs },
+    { integrations: ["portkey"], configs: portkeyConfigs },
     {
       integrations: ["genkit"],
       configs: genkitConfigs,

js/src/auto-instrumentations/configs/portkey.ts

@@ -0,0 +1,18 @@
+import type { InstrumentationConfig } from "../orchestrion-js";
+import { portkeyChannels } from "../../instrumentation/plugins/portkey-channels";
+
+export const portkeyConfigs: InstrumentationConfig[] = [
+  {
+    channelName: portkeyChannels.chatCompletionsCreate.channelName,
+    module: {
+      name: "portkey-ai",
+      versionRange: ">=3.0.0 <4.0.0",
+      filePath: "dist/src/apis/chatCompletions.js",
+    },
+    functionQuery: {
+      className: "ChatCompletions",
+      methodName: "create",
+      kind: "Async",
+    },
+  },
+];

js/src/exports.ts

@@ -240,6 +240,7 @@ export { wrapCohere } from "./wrappers/cohere";
 export { wrapVoyageAI } from "./wrappers/voyageai";
 export { wrapTypeSafe } from "./wrappers/typesafe";
 export { wrapGroq } from "./wrappers/groq";
+export { wrapPortkey } from "./wrappers/portkey";
 export { wrapBedrockRuntime } from "./wrappers/bedrock-runtime";
 export { wrapCopilotClient } from "./wrappers/github-copilot";
 export {

js/src/instrumentation/braintrust-plugin.test.ts

@@ -17,6 +17,7 @@ import { MistralPlugin } from "./plugins/mistral-plugin";
 import { OllamaPlugin } from "./plugins/ollama-plugin";
 import { CoherePlugin } from "./plugins/cohere-plugin";
 import { GroqPlugin } from "./plugins/groq-plugin";
+import { PortkeyPlugin } from "./plugins/portkey-plugin";
 import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin";
 import { LangChainPlugin } from "./plugins/langchain-plugin";
 import { LangSmithPlugin } from "./plugins/langsmith-plugin";
@@ -37,6 +38,10 @@ function createPluginClassMock() {
   });
 }
 
+vi.mock("./plugins/portkey-plugin", () => ({
+  PortkeyPlugin: createPluginClassMock(),
+}));
+
 // Preserve the re-exported utility functions (parseMetricsFromUsage, etc.)
 // while mocking out the OpenAIPlugin class — those utilities are also
 // imported and tested at the bottom of this file via braintrust-plugin's
@@ -673,6 +678,22 @@ describe("BraintrustPlugin", () => {
       expect(MistralPlugin).toHaveBeenCalledTimes(1);
     });
 
+    it("enables and tears down Portkey by default", () => {
+      const plugin = new BraintrustPlugin();
+      plugin.enable();
+      const portkey = vi.mocked(PortkeyPlugin).mock.results[0].value;
+      expect(portkey.enable).toHaveBeenCalledTimes(1);
+      plugin.disable();
+      expect(portkey.disable).toHaveBeenCalledTimes(1);
+    });
+
+    it("does not enable Portkey when portkey: false", () => {
+      const plugin = new BraintrustPlugin({ integrations: { portkey: false } });
+      plugin.enable();
+      expect(PortkeyPlugin).not.toHaveBeenCalled();
+      plugin.disable();
+    });
+
     it("should not create GitHubCopilot plugin when gitHubCopilot: false", () => {
       const plugin = new BraintrustPlugin({
         integrations: { gitHubCopilot: false },
@@ -751,6 +772,7 @@ describe("BraintrustPlugin", () => {
           ollama: false,
           cohere: false,
           groq: false,
+          portkey: false,
           gitHubCopilot: false,
           langchain: false,
           langsmith: false,
@@ -777,6 +799,7 @@ describe("BraintrustPlugin", () => {
       expect(OllamaPlugin).not.toHaveBeenCalled();
       expect(CoherePlugin).not.toHaveBeenCalled();
       expect(GroqPlugin).not.toHaveBeenCalled();
+      expect(PortkeyPlugin).not.toHaveBeenCalled();
       expect(GitHubCopilotPlugin).not.toHaveBeenCalled();
       expect(LangChainPlugin).not.toHaveBeenCalled();
       expect(LangSmithPlugin).not.toHaveBeenCalled();

js/src/instrumentation/braintrust-plugin.ts

@@ -19,6 +19,7 @@ import { OllamaPlugin } from "./plugins/ollama-plugin";
 import { GoogleADKPlugin } from "./plugins/google-adk-plugin";
 import { CoherePlugin } from "./plugins/cohere-plugin";
 import { GroqPlugin } from "./plugins/groq-plugin";
+import { PortkeyPlugin } from "./plugins/portkey-plugin";
 import { BedrockRuntimePlugin } from "./plugins/bedrock-runtime-plugin";
 import { GenkitPlugin } from "./plugins/genkit-plugin";
 import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin";
@@ -80,6 +81,7 @@ export class BraintrustPlugin extends BasePlugin {
   private googleADKPlugin: GoogleADKPlugin | null = null;
   private coherePlugin: CoherePlugin | null = null;
   private groqPlugin: GroqPlugin | null = null;
+  private portkeyPlugin: PortkeyPlugin | null = null;
   private bedrockRuntimePlugin: BedrockRuntimePlugin | null = null;
   private genkitPlugin: GenkitPlugin | null = null;
   private gitHubCopilotPlugin: GitHubCopilotPlugin | null = null;
@@ -223,6 +225,11 @@ export class BraintrustPlugin extends BasePlugin {
       this.groqPlugin.enable();
     }
 
+    if (integrations.portkey !== false) {
+      this.portkeyPlugin = new PortkeyPlugin();
+      this.portkeyPlugin.enable();
+    }
+
     if (
       integrations.bedrock !== false &&
       integrations.awsBedrock !== false &&
@@ -401,6 +408,11 @@ export class BraintrustPlugin extends BasePlugin {
       this.groqPlugin = null;
     }
 
+    if (this.portkeyPlugin) {
+      this.portkeyPlugin.disable();
+      this.portkeyPlugin = null;
+    }
+
     if (this.bedrockRuntimePlugin) {
       this.bedrockRuntimePlugin.disable();
       this.bedrockRuntimePlugin = null;

js/src/instrumentation/config.ts

@@ -42,6 +42,7 @@ export interface InstrumentationIntegrationsConfig {
   ollama?: boolean;
   cohere?: boolean;
   groq?: boolean;
+  portkey?: boolean;
   bedrock?: boolean;
   awsBedrock?: boolean;
   awsBedrockRuntime?: boolean;
@@ -138,6 +139,8 @@ const envIntegrationAliases: Record<
   cohere: "cohere",
   groq: "groq",
   "groq-sdk": "groq",
+  portkey: "portkey",
+  "portkey-ai": "portkey",
   bedrock: "bedrock",
   "aws-bedrock": "awsBedrock",
   awsbedrock: "awsBedrock",
@@ -196,6 +199,7 @@ export function getDefaultInstrumentationIntegrations(): Record<
     ollama: true,
     cohere: true,
     groq: true,
+    portkey: true,
     bedrock: true,
     awsBedrock: true,
     awsBedrockRuntime: true,

js/src/instrumentation/plugins/instrumentation-names.test.ts

@@ -21,6 +21,7 @@ import { googleADKChannels } from "./google-adk-channels";
 import { googleGenerativeAIChannels } from "./google-generative-ai-channels";
 import { googleGenAIChannels } from "./google-genai-channels";
 import { groqChannels } from "./groq-channels";
+import { portkeyChannels } from "./portkey-channels";
 import { huggingFaceChannels } from "./huggingface-channels";
 import { langGraphSDKChannels } from "./langgraph-sdk-channels";
 import { langChainChannels } from "./langchain-channels";
@@ -69,6 +70,7 @@ describe("built-in instrumentation provenance names", () => {
     ],
     [googleGenAIChannels.generateContent, INSTRUMENTATION_NAMES.GOOGLE_GENAI],
     [groqChannels.chatCompletionsCreate, INSTRUMENTATION_NAMES.GROQ],
+    [portkeyChannels.chatCompletionsCreate, INSTRUMENTATION_NAMES.PORTKEY],
     [huggingFaceChannels.chatCompletion, INSTRUMENTATION_NAMES.HUGGINGFACE],
     [langGraphSDKChannels.wait, INSTRUMENTATION_NAMES.LANGGRAPH_SDK],
     [langChainChannels.configure, INSTRUMENTATION_NAMES.LANGCHAIN],

js/src/instrumentation/plugins/portkey-channels.ts

@@ -0,0 +1,23 @@
+import { channel, defineChannels } from "../core/channel-definitions";
+import { INSTRUMENTATION_NAMES } from "../../span-origin";
+import type {
+  PortkeyChatCompletion,
+  PortkeyChatCompletionChunk,
+  PortkeyChatCreateParams,
+  PortkeyChatStream,
+} from "../../vendor-sdk-types/portkey";
+
+// Portkey's chat API uses OpenAI-compatible requests and responses, but takes
+// gateway configuration and HTTP request options as separate arguments.
+export const portkeyChannels = defineChannels(
+  "portkey-ai",
+  {
+    chatCompletionsCreate: channel<
+      [PortkeyChatCreateParams, unknown?, unknown?],
+      PortkeyChatCompletion | PortkeyChatStream,
+      Record<string, unknown>,
+      PortkeyChatCompletionChunk
+    >({ channelName: "chat.completions.create", kind: "async" }),
+  },
+  { instrumentationName: INSTRUMENTATION_NAMES.PORTKEY },
+);