feat: Common audio recorder for integrations

.changeset/shared-audio-recording.md

@@ -0,0 +1,7 @@
+---
+"@braintrust/audio": minor
+---
+
+feat(audio): add shared progressive audio recordings
+
+Add an optional, framework-independent audio recorder with progressive segmentation, WAV and Ogg/Opus encoding in a worker, and ordinary attachment uploads. Bound source memory and worker admission; publish ready recordings and playback selections only after upload succeeds. Preserve uploaded audio when later capture or trace publication fails.

integrations/audio/README.md

@@ -0,0 +1,108 @@
+# Braintrust audio recordings
+
+Record timestamped audio as progressive attachments, with Ogg/Opus compression by default or optional WAV output. Integrations supply audio frames, timing, and spans.
+
+## Installation
+
+```sh
+npm install @braintrust/audio
+```
+
+Requires Node 20+. Installed separately from `braintrust`; enable recording through your integration.
+
+## Recording API
+
+Create a recording, supply audio frames, and finish when the session ends:
+
+```ts
+import { Attachment } from "braintrust";
+import { createRecording } from "@braintrust/audio";
+
+const recording = createRecording({
+  span: sessionSpan,
+  flush: () => logger.flush(),
+  createAttachment: (options) => new Attachment(options),
+  audioFormat: "ogg", // Default; use "wav" for uncompressed audio.
+  options: { segmentDurationSeconds: 60 },
+  snapshot: () => ({
+    origin: sessionStartUnixMs,
+    basis: "local_capture",
+    source: (channel_index) => ({ channel_index }),
+  }),
+  turnSelections: () => [], // Add turn playback selections as shown below.
+});
+
+recording.record({
+  pcm, // Int16Array; the recorder copies it.
+  rate: 24000,
+  channels: 1,
+  channel: 0, // Caller; channel 1 is the agent.
+  at: frameOffsetMs, // Milliseconds since sessionStartUnixMs.
+});
+// No more audio will arrive before this offset, on either channel.
+recording.advance(confirmedThroughMs);
+
+// Finish uploads and publish final recording states and selections.
+await recording.finish();
+```
+
+## Configuration
+
+Required inputs:
+
+| Option             | Purpose                                                                                                                                                                                  |
+| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `span`             | Session span that receives recording metadata and audio attachments.                                                                                                                     |
+| `createAttachment` | Creates an uploadable attachment; typically `(options) => new Attachment(options)`.                                                                                                      |
+| `flush`            | Flushes trace updates; typically `() => logger.flush()`.                                                                                                                                 |
+| `snapshot`         | Returns the current timing and source information: `{ origin, basis, source, metadata? }`. `origin` is the session start in Unix milliseconds; `source(channel)` describes each channel. |
+| `turnSelections`   | Returns `{ span, intervals }` entries linking turns to their audio. Use `() => []` when no turn playback links are needed.                                                               |
+
+Optional settings:
+
+| Option                           | Default     | Meaning                                                                    |
+| -------------------------------- | ----------- | -------------------------------------------------------------------------- |
+| `audioFormat`                    | `"ogg"`     | `"ogg"` for compression; `"wav"` for uncompressed audio.                   |
+| `options.segmentDurationSeconds` | `60`        | Target duration per file; buffer pressure can produce shorter files.       |
+| `options.maxDurationSeconds`     | `1800`      | Stop capture at this session offset in seconds.                            |
+| `options.maxBufferBytes`         | `33554432`  | Maximum retained PCM bytes per recording.                                  |
+| `options.flushFraction`          | `0.5`       | Fraction of the buffer limit that triggers an earlier segment.             |
+| `encoder`                        | `undefined` | Custom encoder `{ module: "..." }`; overrides `audioFormat` when supplied. |
+
+## Turn playback
+
+When reviewing a voice trace, you may want to follow a message, e.g. such as “Where is my order?”, and hear how it was spoken, without searching through the call recording. These selections describe which audio to play for that message.
+
+Supply `turnSelections` to `createRecording()`: a callback that returns each turn’s span and when its audio occurred, in milliseconds from the start of the call. The recorder maps those timestamps onto the uploaded audio files, even when a turn crosses a file boundary, and writes `audio.selections` onto the turn’s span. These references can be used to play that turn’s audio.
+
+For example, if the caller said “Where is my order?” from 1.2 to 2.6 seconds into the call:
+
+```ts
+// Inside the createRecording({ ... }) options:
+turnSelections: () => [
+  {
+    span: userTurnSpan, // The span containing “Where is my order?”
+    intervals: [
+      {
+        recording_span_id: sessionSpan.spanId, // Where the call audio is attached.
+        recording_id: "call",
+        channel_index: 0, // Caller audio.
+        start_offset_ms: 1200,
+        end_offset_ms: 2600,
+      },
+    ],
+  },
+];
+```
+
+When a turn or its timing changes, update what `turnSelections()` returns and call `recording.publishSelections()`. `finish()` also publishes the final selections. These selections associate recorded audio with turns; they do not control what gets recorded.
+
+## Development
+
+```sh
+pnpm run build
+pnpm run check:typings
+pnpm test
+pnpm run test:smoke
+pnpm pack
+```

integrations/audio/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@braintrust/audio",
+  "version": "0.0.1",
+  "description": "Shared audio recording, encoding, and export for Braintrust integrations",
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "build": "tsup",
+    "check:typings": "tsc --noEmit",
+    "test": "vitest run",
+    "test:smoke": "node smoke/recording.mjs"
+  },
+  "dependencies": {
+    "@audio/encode-opus": "1.3.1"
+  },
+  "devDependencies": {
+    "@types/node": "^20.10.5",
+    "tsup": "^8.5.1",
+    "typescript": "^5.3.3",
+    "vitest": "4.1.11",
+    "ogg-opus-decoder": "^1.7.3"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "author": "Braintrust Data Inc.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/braintrustdata/braintrust-sdk-javascript.git",
+    "directory": "integrations/audio"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/",
+    "provenance": true
+  }
+}

integrations/audio/smoke/recording.mjs

@@ -0,0 +1,51 @@
+import assert from "node:assert/strict";
+import { createRequire } from "node:module";
+
+// Also run this file from an isolated app installed from `pnpm pack` output.
+const require = createRequire(import.meta.url);
+for (const api of [
+  await import("@braintrust/audio"),
+  require("@braintrust/audio"),
+]) {
+  for (const audioFormat of ["wav", "ogg"]) {
+    const rows = [];
+    const files = [];
+    const recording = api.createRecording({
+      audioFormat,
+      span: { spanId: "session", log: (row) => rows.push(row) },
+      flush: async () => {},
+      createAttachment: ({ data, filename, contentType }) => {
+        files.push(data);
+        return {
+          reference: { key: filename, filename, content_type: contentType },
+          upload: async () => ({ upload_status: "done" }),
+        };
+      },
+      snapshot: () => ({
+        origin: 1000,
+        basis: "test",
+
+        source: (channel_index) => ({ channel_index }),
+      }),
+      turnSelections: () => [],
+    });
+    recording.record({
+      pcm: new Int16Array(480).fill(1000),
+      at: 0,
+      rate: 24000,
+      channels: 1,
+      channel: 0,
+    });
+    await recording.finish();
+    assert.equal(files.length, 1);
+    assert.equal(
+      Buffer.from(files[0].subarray(0, 4)).toString(),
+      audioFormat === "wav" ? "RIFF" : "OggS",
+    );
+    assert.equal(
+      rows.flatMap((r) => r.metadata?.["audio.recordings"] ?? []).at(-1).state,
+      "ready",
+    );
+  }
+}
+console.log("WAV and Ogg recording passed through ESM and CommonJS exports.");

integrations/audio/src/audio-worker.ts

@@ -0,0 +1,46 @@
+import { parentPort } from "node:worker_threads";
+import { encodeCall, pcmChunks } from "./pcm";
+import type { Packet } from "./recorder";
+parentPort?.on(
+  "message",
+  async ({
+    packets,
+    durationMs,
+    encoderModule,
+  }: {
+    packets: Packet[];
+    durationMs: number;
+    encoderModule?: string;
+  }) => {
+    try {
+      const codec = encoderModule ? await import(encoderModule) : undefined;
+      if (codec?.encodeChunks) {
+        // The optional codec preserves resampling history between PCM blocks.
+        const encoded = await codec.encodeChunks(
+          pcmChunks(packets, durationMs),
+          24000,
+          2,
+        );
+        parentPort!.postMessage(
+          { ...encoded, durationMs: Math.round(durationMs * 24) / 24 },
+          [encoded.bytes.buffer],
+        );
+      } else {
+        // Preserve compatibility with codecs exposing the original encode API.
+        const result = encodeCall(packets, durationMs);
+        const encoded = codec
+          ? await codec.encode(
+              new Int16Array(result.bytes.buffer, 44),
+              24000,
+              2,
+            )
+          : { bytes: result.bytes, mimeType: "audio/wav", sampleRate: 24000 };
+        parentPort!.postMessage({ ...encoded, durationMs: result.durationMs }, [
+          encoded.bytes.buffer,
+        ]);
+      }
+    } catch {
+      parentPort!.postMessage({ reason: "encoding_failed" });
+    }
+  },
+);

integrations/audio/src/buffers.ts

@@ -0,0 +1,60 @@
+/** Owns PCM reservations across staging, recorder admission and worker transfer. */
+export class PcmBuffers {
+  private leases = new Map<
+    ArrayBufferLike,
+    { bytes: number; admitted: boolean }
+  >();
+  private retained = 0;
+  constructor(private maximum: number) {}
+  get bytes() {
+    return this.retained;
+  }
+  copy(pcm: Int16Array): Int16Array {
+    const bytes = pcm.byteLength;
+    if (this.retained + bytes > this.maximum) {
+      throw new Error("capture_byte_limit");
+    }
+    const owned = new Int16Array(pcm);
+    this.leases.set(owned.buffer, { bytes, admitted: false });
+    this.retained += bytes;
+    return owned;
+  }
+
+  admit(pcm: Int16Array): boolean {
+    const lease = this.leases.get(pcm.buffer);
+    if (!lease || lease.admitted) {
+      return false;
+    }
+    lease.admitted = true;
+    return true;
+  }
+  discard(pcm: Int16Array) {
+    if (!this.leases.get(pcm.buffer)?.admitted) {
+      this.release(pcm);
+    }
+  }
+  release(pcm: Int16Array) {
+    const lease = this.leases.get(pcm.buffer);
+    if (!lease) {
+      return;
+    }
+    this.leases.delete(pcm.buffer);
+    this.retained -= lease.bytes;
+  }
+  split(pcm: Int16Array, count: number): [Int16Array, Int16Array] {
+    const lease = this.leases.get(pcm.buffer);
+    if (!lease || pcm.byteLength !== lease.bytes) {
+      throw new Error("Invalid PCM ownership");
+    }
+    const head = pcm.slice(0, count),
+      tail = pcm.slice(count);
+    this.leases.delete(pcm.buffer);
+    this.leases.set(head.buffer, { bytes: head.byteLength, admitted: true });
+    this.leases.set(tail.buffer, { bytes: tail.byteLength, admitted: true });
+    return [head, tail];
+  }
+  close() {
+    this.leases.clear();
+    this.retained = 0;
+  }
+}

integrations/audio/src/codec.ts

@@ -0,0 +1,16 @@
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { pathToFileURL } from "node:url";
+/** Codec is loaded inside Braintrust's encoding worker, never on the voice thread. */
+export function oggOpus() {
+  return {
+    module: pathToFileURL(
+      join(
+        dirname(
+          createRequire(__filename).resolve("@braintrust/audio/package.json"),
+        ),
+        "dist/encoder.mjs",
+      ),
+    ).href,
+  };
+}