diff --git a/.github/workflows/release-agent-stt.yaml b/.github/workflows/release-agent-stt.yaml new file mode 100644 index 00000000..3e0ed4f5 --- /dev/null +++ b/.github/workflows/release-agent-stt.yaml @@ -0,0 +1,90 @@ +name: Release Agent STT SDK + +on: + push: + tags: + - "agent-stt/v*" + +permissions: + contents: read + id-token: write + +jobs: + extract-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.extract.outputs.version }} + steps: + - name: Extract version from tag + id: extract + run: | + # Extract version from tag (agent-stt/v1.0.0 -> 1.0.0) + VERSION=${GITHUB_REF#refs/tags/agent-stt/v} + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + test-agent-stt: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Test Agent STT SDK + run: | + make install-dev + make lint-agent-stt + make test-agent-stt + + release-build: + runs-on: ubuntu-latest + needs: [extract-version, test-agent-stt] + outputs: + version: ${{ needs.extract-version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Update package version in sdk/agent_stt/speechmatics/agent_stt/__init__.py + run: | + VERSION="${{ needs.extract-version.outputs.version }}" + sed -i "s/0\.0\.0/$VERSION/g" ./sdk/agent_stt/speechmatics/agent_stt/__init__.py + echo "Updated version to: $VERSION" + cat ./sdk/agent_stt/speechmatics/agent_stt/__init__.py | grep __version__ + + - name: Build Agent STT SDK + run: | + make install-dev + make build-agent-stt + + - name: Upload dist + uses: actions/upload-artifact@v4 + with: + name: agent-stt-release-dist + path: sdk/agent_stt/dist/ + + pypi-publish: + runs-on: ubuntu-latest + needs: [release-build] + environment: + name: pypi-agent-stt + url: https://pypi.org/project/speechmatics-agent-stt/${{ needs.release-build.outputs.version }} + + steps: + - name: Retrieve release dist + uses: actions/download-artifact@v4 + with: + name: agent-stt-release-dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + password: ${{ secrets.PYPI_ORG_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7928371f..be4f6a64 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -90,6 +90,26 @@ jobs: - name: Build Voice Agent SDK run: make build-voice + test-agent-stt: + name: Test Agent STT SDK + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: make install-dev + - name: Lint Agent STT SDK + run: make lint-agent-stt + - name: Test Agent STT SDK + run: make test-agent-stt + - name: Build Agent STT SDK + run: make build-agent-stt + test-tts: name: Test TTS SDK runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 12d4d8ae..b9f9cb99 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,12 @@ # Makefile for Speechmatics Python SDKs .PHONY: help -.PHONY: test-all test-rt test-batch test-flow test-tts test-voice -.PHONY: format-all format-rt format-batch format-flow format-tts format-voice -.PHONY: lint-all lint-rt lint-batch lint-flow lint-tts lint-voice -.PHONY: type-check-all type-check-rt type-check-batch type-check-flow type-check-tts type-check-voice -.PHONY: build-all build-rt build-batch build-flow build-tts build-voice -.PHONY: clean-all clean-rt clean-batch clean-flow clean-tts clean-voice +.PHONY: test-all test-rt test-batch test-flow test-tts test-voice test-agent-stt +.PHONY: format-all format-rt format-batch format-flow format-tts format-voice format-agent-stt +.PHONY: lint-all lint-rt lint-batch lint-flow lint-tts lint-voice lint-agent-stt +.PHONY: type-check-all type-check-rt type-check-batch type-check-flow type-check-tts type-check-voice type-check-agent-stt +.PHONY: build-all build-rt build-batch build-flow build-tts build-voice build-agent-stt +.PHONY: clean-all clean-rt clean-batch clean-flow clean-tts clean-voice clean-agent-stt help: @@ -19,6 +19,7 @@ help: @echo " test-flow Run tests for Flow SDK" @echo " test-tts Run tests for TTS SDK" @echo " test-voice Run tests for Voice Agent SDK" + @echo " test-agent-stt Run tests for Agent STT SDK" @echo "" @echo "Code formatting:" @echo " format-all Auto-fix formatting for all SDKs" @@ -27,6 +28,7 @@ help: @echo " format-flow Auto-fix formatting for Flow SDK" @echo " format-tts Auto-fix formatting for TTS SDK" @echo " format-voice Auto-fix formatting for Voice Agent SDK" + @echo " format-agent-stt Auto-fix formatting for Agent STT SDK" @echo "" @echo "Linting:" @echo " lint-all Run linting for all SDKs" @@ -35,6 +37,7 @@ help: @echo " lint-flow Run linting for Flow SDK" @echo " lint-tts Run linting for TTS SDK" @echo " lint-voice Run linting for Voice Agent SDK" + @echo " lint-agent-stt Run linting for Agent STT SDK" @echo "" @echo "Type checking:" @echo " type-check-all Run type checking for all SDKs" @@ -43,6 +46,7 @@ help: @echo " type-check-flow Run type checking for Flow SDK" @echo " type-check-tts Run type checking for TTS SDK" @echo " type-check-voice Run type checking for Voice Agent SDK" + @echo " type-check-agent-stt Run type checking for Agent STT SDK" @echo "" @echo "Building:" @echo " build-all Build all SDKs" @@ -51,6 +55,7 @@ help: @echo " build-flow Build Flow SDK" @echo " build-tts Build TTS SDK" @echo " build-voice Build Voice Agent SDK" + @echo " build-agent-stt Build Agent STT SDK" @echo "" @echo "Cleaning:" @echo " clean-all Clean all SDKs" @@ -59,10 +64,11 @@ help: @echo " clean-flow Clean Flow SDK build artifacts" @echo " clean-tts Clean TTS SDK build artifacts" @echo " clean-voice Clean Voice Agent SDK build artifacts" + @echo " clean-agent-stt Clean Agent STT SDK build artifacts" @echo "" # Testing targets -test-all: test-rt test-batch test-flow test-tts test-voice +test-all: test-rt test-batch test-flow test-tts test-voice test-agent-stt test-rt: pytest tests/rt/ -v -s @@ -78,8 +84,11 @@ test-tts: test-voice: pytest tests/voice/ -v -s +test-agent-stt: + pytest tests/agent_stt/ -v -s + # Formatting targets -format-all: format-rt format-batch format-flow format-tts format-voice format-tests format-examples +format-all: format-rt format-batch format-flow format-tts format-voice format-agent-stt format-tests format-examples format-rt: cd sdk/rt/speechmatics && black . @@ -101,6 +110,10 @@ format-voice: cd sdk/voice/speechmatics && black . cd sdk/voice/speechmatics && ruff check --fix . +format-agent-stt: + cd sdk/agent_stt/speechmatics && black . + cd sdk/agent_stt/speechmatics && ruff check --fix . + format-tests: cd tests && black . cd tests && ruff check --fix . @@ -110,7 +123,7 @@ format-examples: cd examples && ruff check --fix . # Linting targets -lint-all: lint-rt lint-batch lint-flow lint-tts lint-voice +lint-all: lint-rt lint-batch lint-flow lint-tts lint-voice lint-agent-stt lint-rt: cd sdk/rt/speechmatics && ruff check . @@ -127,8 +140,11 @@ lint-tts: lint-voice: cd sdk/voice/speechmatics && ruff check . +lint-agent-stt: + cd sdk/agent_stt/speechmatics && ruff check . + # Type checking targets -type-check-all: type-check-rt type-check-batch type-check-flow type-check-tts type-check-voice +type-check-all: type-check-rt type-check-batch type-check-flow type-check-tts type-check-voice type-check-agent-stt type-check-rt: cd sdk/rt/speechmatics && mypy . @@ -144,6 +160,9 @@ type-check-tts: type-check-voice: cd sdk/voice/speechmatics && mypy . +type-check-agent-stt: + cd sdk/agent_stt/speechmatics && mypy . + # Installation targets install-dev: python -m pip install --upgrade pip @@ -152,12 +171,13 @@ install-dev: python -m pip install -e sdk/flow[dev] python -m pip install -e sdk/tts[dev] python -m pip install -e sdk/voice[dev] + python -m pip install -e sdk/agent_stt[dev] install-build: python -m pip install --upgrade build # Building targets -build-all: build-rt build-batch build-flow build-tts build-voice +build-all: build-rt build-batch build-flow build-tts build-voice build-agent-stt build-rt: install-build cd sdk/rt && python -m build @@ -174,8 +194,11 @@ build-tts: install-build build-voice: install-build cd sdk/voice && python -m build +build-agent-stt: install-build + cd sdk/agent_stt && python -m build + # Cleaning targets -clean-all: clean-rt clean-batch clean-flow clean-tts clean-voice clean-test clean-examples +clean-all: clean-rt clean-batch clean-flow clean-tts clean-voice clean-agent-stt clean-test clean-examples clean-rt: rm -rf sdk/rt/dist sdk/rt/build sdk/rt/*.egg-info find sdk/rt -name __pycache__ -exec rm -rf {} + 2>/dev/null || true @@ -196,6 +219,10 @@ clean-voice: rm -rf sdk/voice/dist sdk/voice/build sdk/voice/*.egg-info find sdk/voice -name __pycache__ -exec rm -rf {} + 2>/dev/null || true +clean-agent-stt: + rm -rf sdk/agent_stt/dist sdk/agent_stt/build sdk/agent_stt/*.egg-info + find sdk/agent_stt -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + clean-test: find tests -name __pycache__ -exec rm -rf {} + 2>/dev/null || true rm -rf .pytest_cache diff --git a/README.md b/README.md index 9a4faa0c..1bf015cd 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ [![PyPI - batch](https://img.shields.io/pypi/v/speechmatics-batch?label=batch)](https://pypi.org/project/speechmatics-batch/) [![PyPI - rt](https://img.shields.io/pypi/v/speechmatics-rt?label=rt)](https://pypi.org/project/speechmatics-rt/) [![PyPI - voice](https://img.shields.io/pypi/v/speechmatics-voice?label=voice)](https://pypi.org/project/speechmatics-voice/) +[![PyPI - agent-stt](https://img.shields.io/pypi/v/speechmatics-agent-stt?label=agent-stt)](https://pypi.org/project/speechmatics-agent-stt/) [![Python Versions](https://img.shields.io/pypi/pyversions/speechmatics-batch.svg)](https://pypi.org/project/speechmatics-batch/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/speechmatics/speechmatics-python-sdk/blob/main/LICENSE) [![Build Status](https://github.com/speechmatics/speechmatics-python-sdk/actions/workflows/test.yaml/badge.svg)](https://github.com/speechmatics/speechmatics-python-sdk/actions/workflows/test.yaml) @@ -57,6 +58,9 @@ pip install speechmatics-rt # Voice agents pip install speechmatics-voice +# Voice agents on the Agent STT service +pip install speechmatics-agent-stt + # Text-to-speech pip install speechmatics-tts ``` @@ -84,6 +88,12 @@ pip install speechmatics-tts - Speaker diarization and turn detection - Optional ML-based smart turn: `pip install speechmatics-voice[smart]` +**[speechmatics-agent-stt](./sdk/agent_stt/README.md)** - Agent STT SDK + +- Segment-level transcription for voice agents +- Server-side VAD and turn detection, or bring your own (Pipecat, LiveKit) +- Runs no models locally + **[speechmatics-tts](./sdk/tts/README.md)** - Text-to-speech - Convert text to natural-sounding speech @@ -937,6 +947,7 @@ Each SDK package includes detailed documentation: | **speechmatics-batch** | [README](./sdk/batch/README.md) • [Migration Guide](./sdk/batch/MIGRATION.md) | Async batch transcription | | **speechmatics-rt** | [README](./sdk/rt/README.md) • [Migration Guide](./sdk/rt/MIGRATION.md) | Realtime Streaming | | **speechmatics-voice** | [README](./sdk/voice/README.md) | Voice agent SDK | +| **speechmatics-agent-stt** | [README](./sdk/agent_stt/README.md) • [Plan](./sdk/agent_stt/PLAN.md) | Agent STT SDK | | **speechmatics-tts** | [README](./sdk/tts/README.md) | Text-to-speech | ### Speechmatics Academy diff --git a/examples/agent_stt/README.md b/examples/agent_stt/README.md new file mode 100644 index 00000000..1069db35 --- /dev/null +++ b/examples/agent_stt/README.md @@ -0,0 +1,24 @@ +# Agent STT examples + +Set `SPEECHMATICS_API_KEY` first. To point at a local Voice Agent Service, set +`SPEECHMATICS_RT_URL` (for example `ws://localhost:8000/v2`); the `/agent` segment is appended +when it is missing. + +The service needs 16 kHz raw PCM, so the file examples take a 16 kHz WAV and default to +`tests/voice/assets/audio_01_16kHz.wav`. + +| Example | What it shows | +| --- | --- | +| [file/main.py](file/main.py) | File transcription with the service's VAD; segments, turn events, transcript at the end | +| [realtime_file/main.py](realtime_file/main.py) | The same file paced at wall-clock speed, with the lag of each message behind the audio | +| [client_vad/main.py](client_vad/main.py) | `TurnDetectionMode.EXTERNAL`: the application owns turn boundaries and calls `finalize()`, as Pipecat and LiveKit do | +| [microphone/main.py](microphone/main.py) | Live microphone with diarization and speaker-labelled transcript (needs `pyaudio`) | +| [microphone_windows/main.py](microphone_windows/main.py) | The same, set up for Windows: device selection, in-place partials, Ctrl+C shutdown | + +```bash +python examples/agent_stt/file/main.py +python examples/agent_stt/realtime_file/main.py +python examples/agent_stt/client_vad/main.py +python examples/agent_stt/microphone/main.py +py examples\agent_stt\microphone_windows\main.py +``` diff --git a/examples/agent_stt/client_vad/main.py b/examples/agent_stt/client_vad/main.py new file mode 100644 index 00000000..51ea7a9a --- /dev/null +++ b/examples/agent_stt/client_vad/main.py @@ -0,0 +1,56 @@ +"""Drive turn boundaries from the application instead of the service. + +This is the mode host frameworks use: Pipecat, LiveKit and others already run a VAD, so the +service's VAD is switched off and each turn is closed by calling `finalize()`, which sends +ForceEndOfUtterance stamped with the audio position at the moment of the call. + +The fixed interval below stands in for the host framework's own end-of-speech signal. + +Run with: python examples/agent_stt/client_vad/main.py [path/to/16kHz.wav] +""" + +import asyncio +import sys +import wave + +from speechmatics.agent_stt import AgentSttAsyncClient +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import TurnDetectionMode + +DEFAULT_AUDIO_FILE = "./tests/voice/assets/audio_01_16kHz.wav" +CHUNK_SIZE = 1024 +TURN_SECONDS = 5.0 + + +async def main(path: str) -> None: + config = TranscriptionConfig(language="en", enable_partials=True, turn_detection_mode=TurnDetectionMode.EXTERNAL) + + # Uses SPEECHMATICS_API_KEY from the environment + async with AgentSttAsyncClient(config=config) as client: + + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + print(f"[final] {message['segment']['transcript']}") + + with wave.open(path, "rb") as wav: + if wav.getframerate() != 16000: + print(f"{path} is {wav.getframerate()} Hz; the Agent STT service needs 16 kHz audio") + return + + next_turn_end = TURN_SECONDS + while True: + frame = wav.readframes(CHUNK_SIZE // wav.getsampwidth()) + if not frame: + break + await client.send_audio(frame) + + if client.audio_seconds_sent >= next_turn_end: + print(f"[client vad] end of turn at {client.audio_seconds_sent:.2f}s") + await client.force_end_of_utterance() + next_turn_end += TURN_SECONDS + + print(f"\nTranscript: {client.transcript}") + + +asyncio.run(main(sys.argv[1] if len(sys.argv) > 1 else DEFAULT_AUDIO_FILE)) diff --git a/examples/agent_stt/microphone/main.py b/examples/agent_stt/microphone/main.py new file mode 100644 index 00000000..98c95ac7 --- /dev/null +++ b/examples/agent_stt/microphone/main.py @@ -0,0 +1,60 @@ +"""Transcribe the microphone with the Agent STT service. + +The service's VAD decides where turns begin and end, so this prints segments as they close +plus the speech and turn events around them. + +Run with: python examples/agent_stt/microphone/main.py +""" + +import asyncio + +from speechmatics.agent_stt import AgentSttAsyncClient +from speechmatics.agent_stt import Microphone +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig + +SAMPLE_RATE = 16000 +CHUNK_SIZE = 1024 + + +async def main() -> None: + mic = Microphone(sample_rate=SAMPLE_RATE, chunk_size=CHUNK_SIZE) + if not mic.start(): + print("PyAudio not installed - install with: pip install pyaudio") + return + + config = TranscriptionConfig(language="en", enable_partials=True, diarization="speaker") + + # Uses SPEECHMATICS_API_KEY from the environment + async with AgentSttAsyncClient(config=config) as client: + + @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) + def handle_partial_segment(message): + print(f"[partial] {message['segment']['transcript']}") + + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + speaker = message["segment"].get("speaker", "?") + print(f"[final] {speaker}: {message['segment']['transcript']}") + + @client.on(ServerMessageType.END_OF_TURN) + def handle_end_of_turn(message): + print(f"[turn] end at {message['metadata']['end_time']}s") + + print("\nMicrophone ready - speak now (Ctrl+C to stop)\n") + + try: + while True: + await client.send_audio(await mic.read(CHUNK_SIZE)) + except (asyncio.CancelledError, KeyboardInterrupt): + pass + finally: + mic.stop() + + print(f"\nTranscript: {client.transcript_text(speaker_labels=True)}") + + +try: + asyncio.run(main()) +except KeyboardInterrupt: + pass diff --git a/examples/agent_stt/realtime_file/main.py b/examples/agent_stt/realtime_file/main.py new file mode 100644 index 00000000..379dfb1a --- /dev/null +++ b/examples/agent_stt/realtime_file/main.py @@ -0,0 +1,169 @@ +"""Stream a 16 kHz WAV file to the Agent STT service at wall-clock speed. + +A file read at full speed reaches the service far ahead of real time, so VAD windows, turn +boundaries and latency all read wrong. This paces the send loop so each frame leaves at the +moment its audio would have been captured live, and prints how far behind the audio position +every message arrives - the number worth watching when testing locally. + +Run with: python examples/agent_stt/realtime_file/main.py [path/to/16kHz.wav] +""" + +import argparse +import asyncio +import time +import wave +from typing import Optional + +from speechmatics.agent_stt import AgentSttAsyncClient +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import TurnDetectionMode +from speechmatics.agent_stt import VADConfig + +DEFAULT_AUDIO_FILE = "./tests/voice/assets/audio_01_16kHz.wav" +SAMPLE_RATE = 16000 +BYTES_PER_SAMPLE = 2 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("audio", nargs="?", default=DEFAULT_AUDIO_FILE, help="16 kHz mono WAV file") + parser.add_argument("--language", default="en") + parser.add_argument("--chunk-ms", type=float, default=20.0, help="audio frame size in milliseconds") + parser.add_argument( + "--turn-detection", + choices=[TurnDetectionMode.VAD.value, TurnDetectionMode.EXTERNAL.value], + default=TurnDetectionMode.VAD.value, + help="which mechanism closes turns: the service's VAD, or this script calling finalize()", + ) + parser.add_argument("--vad-window", type=float, help="silence in seconds before the service closes a turn") + parser.add_argument( + "--turn-seconds", type=float, default=5.0, help="fake turn length, --turn-detection external only" + ) + parser.add_argument("--no-partials", action="store_true") + parser.add_argument("--emit-sentences", action="store_true", help="close a segment on every sentence boundary") + return parser.parse_args() + + +class Clock: + """Wall clock for the streaming run, and the lag of each message behind the audio.""" + + def __init__(self) -> None: + self._started_at = time.monotonic() + self.segment_lags = [] + + def start(self) -> None: + """Reset to zero, so the connection handshake does not count as streaming time.""" + self._started_at = time.monotonic() + + @property + def elapsed(self) -> float: + return time.monotonic() - self._started_at + + def log(self, tag: str, text: str, *, audio_time: Optional[float] = None, record: bool = False) -> None: + lag = "" + if audio_time is not None: + behind = self.elapsed - audio_time + if record: + self.segment_lags.append(behind) + lag = f"+{behind * 1000:>5.0f}ms" + print(f"[{self.elapsed:6.2f}s] {tag:<17}{lag:<10} {text}") + + +def build_client(args: argparse.Namespace, clock: Clock) -> AgentSttAsyncClient: + config = TranscriptionConfig( + language=args.language, + enable_partials=not args.no_partials, + turn_detection_mode=TurnDetectionMode(args.turn_detection), + vad_config=VADConfig(window=args.vad_window) if args.vad_window is not None else None, + emit_sentences=args.emit_sentences, + ) + + # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_RT_URL to point at a local service + client = AgentSttAsyncClient(config=config) + + @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) + def handle_partial_segment(message): + clock.log("[partial]", message["segment"]["transcript"], audio_time=message["metadata"]["end_time"]) + + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + clock.log( + "[final]", + message["segment"]["transcript"], + audio_time=message["metadata"]["end_time"], + record=True, + ) + + @client.on(ServerMessageType.SPEECH_STARTED) + def handle_speech_started(message): + clock.log("[speech started]", f"{message['metadata']['start_time']:.2f}s") + + @client.on(ServerMessageType.SPEECH_ENDED) + def handle_speech_ended(message): + clock.log("[speech ended]", f"{message['metadata']['end_time']:.2f}s") + + @client.on(ServerMessageType.START_OF_TURN) + def handle_start_of_turn(message): + clock.log("[turn started]", f"{message['metadata']['start_time']:.2f}s") + + @client.on(ServerMessageType.END_OF_TURN) + def handle_end_of_turn(message): + end_time = message["metadata"]["end_time"] + clock.log("[turn ended]", f"{end_time:.2f}s", audio_time=end_time) + + @client.on(ServerMessageType.ERROR) + def handle_error(message): + clock.log("[error]", str(message)) + + return client + + +async def stream(client: AgentSttAsyncClient, wav: wave.Wave_read, args: argparse.Namespace, clock: Clock) -> None: + """Send the file frame by frame, releasing each frame no earlier than its capture time.""" + frames_per_chunk = int(SAMPLE_RATE * args.chunk_ms / 1000) + next_turn_end = args.turn_seconds + clock.start() + + while frame := wav.readframes(frames_per_chunk): + # Hold each frame until the wall clock reaches the end of the audio it carries, so a + # frame leaves exactly when a live capture would have finished recording it. Paced off + # the session clock rather than per-frame sleeps, so the send does not drift. + frame_end = client.audio_seconds_sent + len(frame) / (SAMPLE_RATE * BYTES_PER_SAMPLE) + early = frame_end - clock.elapsed + if early > 0: + await asyncio.sleep(early) + + await client.send_audio(frame) + + if args.turn_detection == TurnDetectionMode.EXTERNAL.value and client.audio_seconds_sent >= next_turn_end: + clock.log("[external]", f"end of turn at {client.audio_seconds_sent:.2f}s") + await client.force_end_of_utterance() + next_turn_end += args.turn_seconds + + +async def main() -> None: + args = parse_args() + clock = Clock() + client = build_client(args, clock) + + with wave.open(args.audio, "rb") as wav: + if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (SAMPLE_RATE, 1, BYTES_PER_SAMPLE): + print(f"{args.audio} must be 16 kHz mono 16-bit PCM for the Agent STT service") + return + duration = wav.getnframes() / SAMPLE_RATE + + async with client: + await stream(client, wav, args, clock) + + print(f"\nTranscript: {client.transcript}") + print(f"Streamed {duration:.2f}s of audio in {clock.elapsed:.2f}s") + if clock.segment_lags: + mean = sum(clock.segment_lags) / len(clock.segment_lags) + print(f"Final segment lag behind audio: mean {mean * 1000:.0f}ms, max {max(clock.segment_lags) * 1000:.0f}ms") + + +asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 018d6c1a..bf8e6ed8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.pyright] venvPath = "." venv = ".venv" -extraPaths = ["sdk/batch", "sdk/flow", "sdk/rt", "sdk/tts", "sdk/voice"] +extraPaths = ["sdk/agent_stt", "sdk/batch", "sdk/flow", "sdk/rt", "sdk/tts", "sdk/voice"] [tool.black] line-length = 120 diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md new file mode 100644 index 00000000..625b27bf --- /dev/null +++ b/sdk/agent_stt/PLAN.md @@ -0,0 +1,168 @@ +# Agent STT SDK - plan + +## What this is + +`speechmatics-agent-stt` is a thin extension of `speechmatics-rt` that talks to the +**Voice Agent Service** (`voice-agent-service`) instead of the raw RT engine. The service sits +between the RT SaaS proxy and the transcriber and turns word-level RT output into +**segments**, plus VAD/turn signals. + +The SDK does **no VAD and no turn detection of its own**. Boundaries come from one of two places: + +| Mode | Who decides the turn boundary | Wire behaviour | +| --- | --- | --- | +| `TurnDetectionMode.VAD` | the service's own (Silero) VAD | `transcription_config.vad_config.enabled = true`; service emits `SpeechStarted`/`SpeechEnded`/`StartOfTurn`/`EndOfTurn` and forces end-of-utterance internally | +| `TurnDetectionMode.EXTERNAL` | the host framework (Pipecat, LiveKit, ...) | `transcription_config.vad_config.enabled = false`; the host calls `client.force_end_of_utterance()` on its own VAD's stop-speaking event | + +This is the whole reason the SDK exists as a separate package: the `voice` SDK bundles VAD + +smart-turn models in-process, which duplicates what Pipecat/LiveKit already run and what the +service now does server-side. + +## Protocol delta vs the RT SDK + +Endpoint: RT URL + `/agent` (`wss://eu2.rt.speechmatics.com/v2/agent`). + +Client -> server: unchanged (`StartRecognition`, binary audio, `EndOfStream`, +`ForceEndOfUtterance`). No new client messages. + +`StartRecognition.transcription_config` gains two service-only fields, stripped by the service +before it forwards to the RT engine: + +- `vad_config`: `{enabled, window, onset_threshold, offset_threshold}` +- `emit_sentences`: `bool` - close a segment on sentence boundaries mid-turn + +Server -> client, new messages (`voice_agent_api/_service_messages.py`): + +- `AddSegment` - `{segment: {transcript, speaker?}, metadata: {start_time, end_time}}` +- `AddPartialSegment` - same shape, interim +- `SpeechStarted` / `SpeechEnded` - `{metadata: {start_time|end_time}}` (VAD) +- `StartOfTurn` / `EndOfTurn` - `{metadata: {start_time|end_time}}` (turn detection) +- `Warning` - RT shape, also emitted by the service for config adjustments + +Server -> client, RT passthrough: `RecognitionStarted`, `AudioAdded`, `AddTranscript`, +`AddPartialTranscript`, `EndOfTranscript`, `Info`, `Warning`, `Error`, audio events. + +Note: the service still forwards `AddTranscript`/`AddPartialTranscript` verbatim today. The SDK +accumulates its transcript from **segments only** and models neither those nor audio events; +they still reach the event log and any handler registered under their name, so nothing is lost +if the service stops sending them. + +Constraints the service imposes: `audio_format.type` must be `raw`, sample rate `16000` +(Silero), encoding `pcm_s16le` or `pcm_f32le` (no mulaw). The SDK defaults to exactly that. + +## Milestone 1 - the SDK (done) + +``` +sdk/agent_stt/ + pyproject.toml speechmatics-agent-stt, depends on speechmatics-rt + README.md + PLAN.md this file + speechmatics/agent_stt/ + __init__.py public API + _client.py AgentSttAsyncClient (subclasses rt.AsyncClient) + _models.py message enums, TranscriptionConfig, VADConfig, Segment, TimedEvent + _transcript.py Transcript - final segments, live partial, timeline, raw event log + _transport.py AgentTransport - stamps sm-sdk=python-agent-stt-sdk-vX + _url.py URL resolution +tests/agent_stt/ offline unit tests (no API key needed) +examples/agent_stt/ server-VAD and client-VAD (BYO) examples +``` + +Everything inherited from `rt.AsyncClient` stays: auth (`StaticKeyAuth`/`JWTAuth`), transport, +reconnect-free lifecycle, `send_audio`, `transcribe`, `stop_session`, +`force_end_of_utterance`, the `EventEmitter` decorator API. + +What the subclass adds: + +1. URL resolution (`/agent` on top of the `SPEECHMATICS_RT_URL` endpoint). +2. `TranscriptionConfig` with `turn_detection_mode`, `vad_config`, `emit_sentences`, and an Agent STT + `Model` enum defaulting to `linden-1`. The request goes to the proxy rather than the service + websocket directly, and the proxy resolves the Agent STT model name onto the engine's + operating point, so the transcriber never sees a name it has no notion of. The deprecated + `operating_point` suppresses the `model` default, so the merged `StartRecognition` never + carries both keys. `linden-2` lands as one more enum member. +3. Segment-level transcript accumulation, so `client.transcript` reads like the RT flow does. +4. A raw event log (`client.events`) capturing every server message, including ones this SDK + version doesn't model yet - that is the "accept and save the other messages" requirement. +5. 16 kHz raw PCM defaults. + +Plumbing carried over from the `voice` SDK (rewritten, not imported, since that SDK goes away): + +- `connect()` / `disconnect()` and a context manager that connects on entry, the shape Pipecat + already calls +- an audio gate: frames before RecognitionStarted or after close are dropped, not raised on, and + a transport error closes the gate instead of propagating into an audio callback +- `finalize()` callable from a sync handler, stamping ForceEndOfUtterance with the audio position + **at the moment of the call** rather than when the send lands +- `app` reported as `sm-app` on the URL, alongside the SDK's own `sm-sdk` identifier +- the language pack's `word_delimiter` from RecognitionStarted driving how text is joined + +Not carried over: VAD, smart turn, the audio ring buffer, metrics/diagnostics messages, +speaker focus, fragment-level segment assembly (the service does that now). + +### Verified end to end + +Run against the real `voice-agent-service` with a stub RT transcriber behind it, in client-VAD +mode: `/v2/agent/default` routing, three `ForceEndOfUtterance` turns each flushing one +`AddSegment`, speaker labels, the event log, and `client.transcript` coming out as +`"Hello there. How are you today? Goodbye."`. + +The StartRecognition the service forwarded downstream confirmed that `vad_config` and +`emit_sentences` are stripped before it reaches the RT engine. + +That run predates the `linden-1` default, so it sent no `model` at all and the message carried +only the service's locked `operating_point`. Model resolution happens in the proxy ahead of the +service, which this stub setup does not exercise, so it still needs verifying against the real +proxy. + +## Milestone 2 - Pipecat (not in this change) + +`pipecat/src/pipecat/services/speechmatics/stt.py` (~1266 lines) currently drives +`speechmatics.voice.VoiceAgentClient` and its in-process VAD/smart-turn. Nothing in Pipecat is +touched by this change. The migration: + +- swap the import block to `speechmatics.agent_stt` +- collapse Pipecat's own `TurnDetectionMode` onto the two modes that exist: + - its `EXTERNAL` -> `TurnDetectionMode.EXTERNAL`, with `finalize()` on + `VADUserStoppedSpeakingFrame` + - its `ADAPTIVE` -> `TurnDetectionMode.VAD`, with `StartOfTurn`/`EndOfTurn` driving + `ProposedUserStartedSpeakingFrame`/`ProposedUserStoppedSpeakingFrame` + - `FIXED` and `SMART_TURN` -> removed (see below) +- `AddPartialSegment` -> `InterimTranscriptionFrame`, `AddSegment` -> `TranscriptionFrame` +- drop the `pipecat-ai[speechmatics]` onnxruntime/transformers extras that only existed for the + bundled VAD and smart-turn models +- drop `end_of_utterance_silence_trigger` and `end_of_utterance_max_delay` from + `SpeechmaticsSTTSettings` and `InputParams`, along with the passthrough at + `stt.py:788` and `stt.py:1252` - engine silence-based end of utterance is off for this + service, so both are no-ops +- `_enable_vad` (`stt.py:530`) becomes `turn_detection_mode is TurnDetectionMode.EXTERNAL`, since that is now the only + mode where Pipecat's own VAD drives the boundary +- otherwise keep `SpeechmaticsSTTSettings` as the public surface so user code doesn't change + +### `FIXED` and `SMART_TURN` are removed + +`end_of_utterance_silence_trigger` is off for this service: the service pins it to `0.0`, and a +non-forced end of utterance does not close a segment. So there is nothing for `TurnDetectionMode.FIXED` to mean here and it +goes away rather than being aliased to another mode. + +`SMART_TURN` goes for the same reason: the service has no smart-turn endpoint yet (planned for a +later release), and this SDK loads no models, so the mode cannot be honoured as specified. It +comes back when the service does, as a `TurnDetectionMode.SMART_TURN` member alongside `TurnDetectionMode.VAD`. + +Removing it costs Pipecat users nothing, because Pipecat's own turn analyzer still works: any +host-side endpointing - VAD, ML turn model, push-to-talk - reaches the service the same way, +through `finalize()`. `TurnDetectionMode.EXTERNAL` is agnostic about what produced the signal. + +Turns therefore end in exactly two ways, which is what `TurnDetectionMode` models: the service's VAD, or +the application calling `finalize()`. + +Open questions to settle before starting milestone 2: + +- ~~speaker-focus / known-speaker features in the Pipecat service have no service-side + equivalent yet~~ - dropped for now, to be added in a later service release. `known_speakers` + still works, since `speaker_diarization_config.speakers` passes straight through. +- ~~engine-silence endpointing / `FIXED` mode~~ - removed, see above. +- ~~`SMART_TURN`~~ - removed until the service implements it. Host-side turn models keep working + through `TurnDetectionMode.EXTERNAL`. + +Both removals are user-visible, so they need a changelog entry when the Pipecat change lands. diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md new file mode 100644 index 00000000..32bb5bc8 --- /dev/null +++ b/sdk/agent_stt/README.md @@ -0,0 +1,149 @@ +# Speechmatics Agent STT SDK + +Python client for the Speechmatics **Agent STT** service, built on +[`speechmatics-rt`](https://pypi.org/project/speechmatics-rt/). + +The Agent STT service works in **segments** rather than word groups, and reports the speech and +turn events a voice agent needs. This SDK runs **no VAD and no turn detection of its own** - +either the service's VAD closes turns, or your application's does. + +```bash +pip install speechmatics-agent-stt +``` + +## Quick start + +```python +import asyncio +from speechmatics.agent_stt import AgentSttAsyncClient, ServerMessageType, TranscriptionConfig + +async def main(): + # Uses SPEECHMATICS_API_KEY from the environment + async with AgentSttAsyncClient(config=TranscriptionConfig(language="en", enable_partials=True)) as client: + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + print(message["segment"]["transcript"]) + + while chunk := next_audio_chunk(): + await client.send_audio(chunk) + + print(client.transcript) + +asyncio.run(main()) +``` + +## Who closes the turn + +The service needs a boundary to close a segment on. Pick where it comes from: + +```python +from speechmatics.agent_stt import TranscriptionConfig, TurnDetectionMode, VADConfig + +# The service's VAD (default). It emits SpeechStarted/SpeechEnded and StartOfTurn/EndOfTurn. +config = TranscriptionConfig( + turn_detection_mode=TurnDetectionMode.VAD, + vad_config=VADConfig(window=0.2, onset_threshold=0.5, offset_threshold=0.35), +) + +# Your endpointing - Pipecat, LiveKit, or your own. The service's VAD stays off. +config = TranscriptionConfig(turn_detection_mode=TurnDetectionMode.EXTERNAL) +``` + +With `TurnDetectionMode.EXTERNAL`, close each turn when your side decides speech has ended: + +```python +client.finalize() # from a sync callback +await client.force_end_of_utterance() # from async code +``` + +Either sends `ForceEndOfUtterance` stamped with the audio position at the moment of the call, so +the service cuts the turn where you heard the end of speech rather than wherever the send lands. +The flushed segment comes back as a normal `AddSegment`. + +What decides that is entirely yours - a VAD, an ML turn model, or a push-to-talk button. The SDK +only cares that something calls `finalize()`. + +## Session output + +Every server message is dispatched to your handlers and also kept on the client: + +```python +client.transcript # final segments joined by the language's word delimiter +client.segments # list[Segment] - transcript, timing, speaker, is_final +client.partial_segment # the segment currently in flight, or None +client.timeline # list[TimedEvent] - the speech and turn events, in order +client.events # every raw message, including ones this SDK does not model +client.session_info # session id and the language pack the service reported + +client.transcript_text(speaker_labels=True, include_partial=False) +``` + +Pass `record_events=False` to `AgentSttAsyncClient` for long-running sessions where the raw log is not +wanted. + +## Messages + +Emitted by the service: + +| Message | Payload | +| --- | --- | +| `AddSegment` | `segment.transcript`, optional `segment.speaker`, `metadata.start_time`, `metadata.end_time` | +| `AddPartialSegment` | interim preview of the segment being built | +| `SpeechStarted` / `SpeechEnded` | `metadata.start_time` / `metadata.end_time` (service VAD) | +| `StartOfTurn` / `EndOfTurn` | `metadata.start_time` / `metadata.end_time` (service turn detection) | + +Passed through from the RT engine: `RecognitionStarted`, `AudioAdded`, `EndOfTranscript`, +`SpeakersResult`, `Info`, `Warning`, `Error`. + +Anything else the engine sends - the word-level `AddTranscript`/`AddPartialTranscript`, audio +events - is not modelled here, but still reaches `client.events` and any handler registered +under its name. + +## Configuration + +`TranscriptionConfig` is the RT transcription config plus the service-only fields: + +| Field | Meaning | +| --- | --- | +| `turn_detection_mode` | `TurnDetectionMode.VAD` (default) or `TurnDetectionMode.EXTERNAL` | +| `vad_config` | `window`, `onset_threshold`, `offset_threshold` for the service's VAD | +| `emit_sentences` | Close a segment on every sentence boundary, not only at the turn boundary | + +`model` takes an Agent STT `Model` and defaults to `DEFAULT_MODEL` (`Model.LINDEN_1`): + +```python +from speechmatics.agent_stt import Model, TranscriptionConfig + +config = TranscriptionConfig(model=Model.LINDEN_1) +``` + +The proxy in front of the service resolves the Agent STT model name onto the engine's operating +point, so the transcriber never sees a name it has no notion of. The RT models (`enhanced`, +`standard`) are not Agent STT models and are not accepted here; the deprecated `operating_point` +still passes through, and suppresses the `model` default so the two never arrive together. + +Engine silence-based end of utterance is not offered here. A turn ends either because the +service's VAD said so, or because you called `finalize()`. + +## Endpoint + +The Agent STT endpoint is the RT endpoint plus `/agent`: + +```python +AgentSttAsyncClient(url="wss://eu2.rt.speechmatics.com/v2") # -> /v2/agent +AgentSttAsyncClient(url="ws://localhost:8000/v2") # -> /v2/agent +AgentSttAsyncClient(app="pipecat/1.0") # reported as sm-app +``` + +Resolution order: the `url` argument, `SPEECHMATICS_RT_URL`, then the EU endpoint. The `/agent` +segment is appended when it is missing. + +## Audio + +The service requires **16 kHz raw PCM**, `pcm_s16le` or `pcm_f32le`, which is what the client +defaults to. Audio sent before the session is ready, or after it closes, is dropped rather than +raising, so an audio callback does not have to track session state. + +## Examples + +See [examples/agent_stt](../../examples/agent_stt). diff --git a/sdk/agent_stt/pyproject.toml b/sdk/agent_stt/pyproject.toml new file mode 100644 index 00000000..974188f2 --- /dev/null +++ b/sdk/agent_stt/pyproject.toml @@ -0,0 +1,69 @@ +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "speechmatics-agent-stt" +dynamic = ["version"] +description = "Speechmatics Agent STT Python client for voice agents" +readme = "README.md" +authors = [{ name = "Speechmatics", email = "support@speechmatics.com" }] +license = "MIT" +requires-python = ">=3.9" +dependencies = ["speechmatics-rt>=0.5.3"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Multimedia :: Sound/Audio :: Speech", + "Topic :: Software Development :: Libraries :: Python Modules", +] +keywords = [ + "speechmatics", + "speech-to-text", + "conversational-ai", + "voice", + "agents", + "real-time", + "websocket", + "pipecat", + "livekit", +] + +[project.optional-dependencies] +jwt = ["aiohttp"] +dev = [ + "black", + "ruff", + "mypy", + "pre-commit", + "pytest", + "pytest-asyncio", + "pytest-cov", + "pytest-mock", + "build", +] + +[project.urls] +homepage = "https://github.com/speechmatics/speechmatics-python-sdk" +documentation = "https://docs.speechmatics.com/" +repository = "https://github.com/speechmatics/speechmatics-python-sdk" +issues = "https://github.com/speechmatics/speechmatics-python-sdk/issues" + +[tool.setuptools.dynamic] +version = { attr = "speechmatics.agent_stt.__version__" } + +[tool.setuptools.package-data] +"speechmatics.agent_stt" = ["py.typed"] + +[tool.setuptools.packages.find] +where = ["."] + +[[tool.mypy.overrides]] +module = ["speechmatics.rt.*"] +ignore_missing_imports = true diff --git a/sdk/agent_stt/speechmatics/__init__.py b/sdk/agent_stt/speechmatics/__init__.py new file mode 100644 index 00000000..8db66d3d --- /dev/null +++ b/sdk/agent_stt/speechmatics/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/sdk/agent_stt/speechmatics/agent_stt/__init__.py b/sdk/agent_stt/speechmatics/agent_stt/__init__.py new file mode 100644 index 00000000..82dfaeb9 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/__init__.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2026, Speechmatics / Cantab Research Ltd +# + +"""Speechmatics Agent STT SDK. + +A client for the Speechmatics Agent STT service, built on the Speechmatics Python Real-Time +SDK. The service works in segments rather than word groups and reports speech and turn events. + +This SDK runs no VAD and no turn detection of its own: either the service's VAD closes turns, +or the application's does (Pipecat, LiveKit, ...) by calling `finalize()`. +""" + +__version__ = "0.0.0" + +from speechmatics.rt import AudioEncoding +from speechmatics.rt import AudioError +from speechmatics.rt import AudioFormat +from speechmatics.rt import AuthBase +from speechmatics.rt import AuthenticationError +from speechmatics.rt import ConfigurationError +from speechmatics.rt import ConnectionConfig +from speechmatics.rt import ConnectionError +from speechmatics.rt import EventEmitter +from speechmatics.rt import JWTAuth +from speechmatics.rt import Microphone +from speechmatics.rt import SessionError +from speechmatics.rt import SpeakerDiarizationConfig +from speechmatics.rt import SpeakerIdentifier +from speechmatics.rt import StaticKeyAuth +from speechmatics.rt import TimeoutError +from speechmatics.rt import TranscriptionError +from speechmatics.rt import TransportError + +from ._client import AgentSttAsyncClient +from ._client import AgentSTTClient +from ._models import DEFAULT_CHUNK_SIZE +from ._models import DEFAULT_MODEL +from ._models import DEFAULT_SAMPLE_RATE +from ._models import DEFAULT_WORD_DELIMITER +from ._models import SEGMENT_MESSAGES +from ._models import TIMED_MESSAGES +from ._models import AdditionalVocabEntry +from ._models import ClientMessageType +from ._models import LanguagePackInfo +from ._models import Model +from ._models import Segment +from ._models import ServerMessageType +from ._models import SessionInfo +from ._models import TimedEvent +from ._models import TranscriptionConfig +from ._models import TurnDetectionMode +from ._models import VADConfig +from ._transcript import Transcript +from ._url import resolve_url + +__all__ = [ + "DEFAULT_CHUNK_SIZE", + "DEFAULT_MODEL", + "DEFAULT_SAMPLE_RATE", + "DEFAULT_WORD_DELIMITER", + "SEGMENT_MESSAGES", + "TIMED_MESSAGES", + "__version__", + # Client + "AgentSTTClient", + "AgentSttAsyncClient", + # Config + "AdditionalVocabEntry", + "AudioEncoding", + "AudioFormat", + "ConnectionConfig", + "SpeakerDiarizationConfig", + "SpeakerIdentifier", + "TranscriptionConfig", + "TurnDetectionMode", + "Model", + "VADConfig", + # Auth + "AuthBase", + "JWTAuth", + "StaticKeyAuth", + # Messages + "ClientMessageType", + "ServerMessageType", + "Segment", + "TimedEvent", + # Session + "LanguagePackInfo", + "SessionInfo", + "Transcript", + "resolve_url", + # Utilities + "EventEmitter", + "Microphone", + # Exceptions + "AudioError", + "AuthenticationError", + "ConfigurationError", + "ConnectionError", + "SessionError", + "TimeoutError", + "TranscriptionError", + "TransportError", +] diff --git a/sdk/agent_stt/speechmatics/agent_stt/_client.py b/sdk/agent_stt/speechmatics/agent_stt/_client.py new file mode 100644 index 00000000..90e2d383 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_client.py @@ -0,0 +1,474 @@ +from __future__ import annotations + +import asyncio +import time +import uuid +from typing import Any +from typing import BinaryIO +from typing import Optional + +from speechmatics.rt import AsyncClient as RTAsyncClient +from speechmatics.rt import AudioEncoding +from speechmatics.rt import AudioFormat +from speechmatics.rt import AuthBase +from speechmatics.rt import ConnectionConfig +from speechmatics.rt import StaticKeyAuth +from speechmatics.rt import TimeoutError as RTTimeoutError +from speechmatics.rt import TranscriptionConfig as RTTranscriptionConfig +from speechmatics.rt import TransportError + +from ._logging import get_logger +from ._models import DEFAULT_CHUNK_SIZE +from ._models import DEFAULT_SAMPLE_RATE +from ._models import TIMED_MESSAGES +from ._models import ClientMessageType +from ._models import LanguagePackInfo +from ._models import Segment +from ._models import ServerMessageType +from ._models import SessionInfo +from ._models import TimedEvent +from ._models import TranscriptionConfig +from ._transcript import Transcript +from ._transport import AgentTransport +from ._url import resolve_url + +_UNSET = object() + +DISCONNECT_TIMEOUT_S = 5.0 + + +class AgentSttAsyncClient(RTAsyncClient): + """ + Asynchronous client for the Speechmatics Agent STT service. + + Extends the RT client to talk to the Agent STT endpoint (`/agent`), which works in + segments rather than word groups and reports speech and turn events. The client runs no + VAD and no turn detection of its own: either the service's VAD closes turns + (`TurnDetectionMode.VAD`) or the application's does, by calling `finalize()` + (`TurnDetectionMode.EXTERNAL`). + + Args: + auth: Authentication instance. Defaults to `StaticKeyAuth` built from `api_key` or the + `SPEECHMATICS_API_KEY` environment variable. + api_key: Speechmatics API key, used when `auth` is not given. + url: WebSocket endpoint. Defaults to `SPEECHMATICS_RT_URL`, then the EU endpoint. + An `/agent` segment is appended if absent. + app: Application name reported to the service as `sm-app`. + config: Transcription config for the session, normally an + `agent_stt.TranscriptionConfig`. + audio_format: Audio format. Defaults to 16 kHz signed 16-bit PCM, which is what the + service requires. + conn_config: WebSocket connection configuration. + record_events: Whether to keep every raw server message in `events`. + + Examples: + Service VAD, transcript at the end: + >>> async with AgentSttAsyncClient(api_key="your-key") as client: + ... @client.on(ServerMessageType.ADD_SEGMENT) + ... def handle_segment(message): + ... print(message["segment"]["transcript"]) + ... await client.send_audio(frame) + >>> print(client.transcript) + + External endpointing (Pipecat, LiveKit): + >>> config = TranscriptionConfig(turn_detection_mode=TurnDetectionMode.EXTERNAL) + >>> client = AgentSttAsyncClient(api_key="your-key", config=config) + >>> await client.connect() + >>> await client.send_audio(frame) + >>> client.finalize() # on the application's own end-of-speech signal + """ + + def __init__( + self, + auth: Optional[AuthBase] = None, + *, + api_key: Optional[str] = None, + url: Optional[str] = None, + app: Optional[str] = None, + config: Optional[RTTranscriptionConfig] = None, + audio_format: Optional[AudioFormat] = None, + conn_config: Optional[ConnectionConfig] = None, + record_events: bool = True, + ) -> None: + super().__init__( + auth, + api_key=api_key, + url=resolve_url(url, app=app), + conn_config=conn_config, + ) + + self._logger = get_logger("speechmatics.agent_stt.client") + + self._config: RTTranscriptionConfig = config or TranscriptionConfig() + self._audio_format = audio_format or AudioFormat( + encoding=AudioEncoding.PCM_S16LE, + sample_rate=DEFAULT_SAMPLE_RATE, + chunk_size=DEFAULT_CHUNK_SIZE, + ) + + self._session_info = SessionInfo(request_id=self._session.request_id) + self._transcript = Transcript(record_events=record_events) + + self._is_connected = False + self._is_ready_for_audio = False + self._finalize_sent_at: Optional[float] = None + self._last_finalize_latency = 0.0 + + self._register_handlers() + + @classmethod + def _create_transport_from_config( + cls, + auth: Optional[AuthBase] = None, + *, + api_key: Optional[str] = None, + url: Optional[str] = None, + conn_config: Optional[ConnectionConfig] = None, + request_id: Optional[str] = None, + ) -> AgentTransport: + """Build the Agent STT transport, so the service sees this SDK's identifier.""" + return AgentTransport( + url or resolve_url(), + conn_config or ConnectionConfig(), + auth or StaticKeyAuth(api_key), + request_id or str(uuid.uuid4()), + ) + + def _register_handlers(self) -> None: + """Track session state and accumulate segments, leaving all messages for the application.""" + self.on(ServerMessageType.RECOGNITION_STARTED, self._on_session_started) + self.on(ServerMessageType.ADD_SEGMENT, self._on_segment) + self.on(ServerMessageType.ADD_PARTIAL_SEGMENT, self._on_segment) + for message_type in TIMED_MESSAGES: + self.on(message_type, self._on_timed_event) + + # ========================================================================== + # Session lifecycle + # ========================================================================== + + async def connect(self, ws_headers: Optional[dict] = None) -> None: + """ + Open the session and wait until the service is ready for audio. + + Audio sent before this returns is dropped, so callers do not have to sequence the + handshake themselves. + + Args: + ws_headers: Additional WebSocket handshake headers. + + Raises: + ConnectionError: If the WebSocket connection fails. + TimeoutError: If the service does not accept the session in time. + + Examples: + >>> client = AgentSttAsyncClient(api_key="your-key") + >>> await client.connect() + """ + if self._is_connected: + return + + await self.start_session( + transcription_config=self._config, + audio_format=self._audio_format, + ws_headers=ws_headers, + ) + self._is_connected = True + + async def disconnect(self) -> None: + """ + Close the session, flushing whatever the service still holds. + + Examples: + >>> await client.disconnect() + >>> print(client.transcript) + """ + if not self._is_connected: + await self.close() + return + + self._is_ready_for_audio = False + try: + await asyncio.wait_for(self.stop_session(), timeout=DISCONNECT_TIMEOUT_S) + except Exception as e: + self._logger.warning("Error closing session: %s", e) + await self.close() + finally: + self._is_connected = False + + async def __aenter__(self) -> AgentSttAsyncClient: + """Open the session on entry.""" + await self.connect() + return self + + async def __aexit__(self, *args: Any) -> None: + """Close the session on exit.""" + await self.disconnect() + + async def start_session( + self, + *, + transcription_config: Optional[RTTranscriptionConfig] = None, + audio_format: Optional[AudioFormat] = None, + ws_headers: Optional[dict] = None, + ) -> None: + """ + Start the session, defaulting to the config this client was built with. + + Args: + transcription_config: Transcription config for the session. + audio_format: Audio format. Must be 16 kHz raw PCM for the Agent STT service. + ws_headers: Additional WebSocket handshake headers. + + Raises: + ConnectionError: If the WebSocket connection fails. + TimeoutError: If the service does not accept the session in time. + """ + await super().start_session( + transcription_config=transcription_config or self._config, + audio_format=audio_format or self._audio_format, + ws_headers=ws_headers, + ) + + # ========================================================================== + # Audio and turn control + # ========================================================================== + + async def send_audio(self, payload: bytes) -> None: + """ + Send an audio frame. + + Frames sent before the session is ready, or after it has closed, are dropped rather + than raising, so an audio callback does not have to track session state. + + Args: + payload: Raw audio bytes in the session's audio format. + + Examples: + >>> await client.send_audio(frame) + """ + if not self._is_ready_for_audio: + return + + try: + await super().send_audio(payload) + except TransportError as e: + self._logger.warning("Error sending audio: %s", e) + self._is_ready_for_audio = False + + def finalize(self, *, timestamp: Optional[float] | object = _UNSET) -> None: + """ + Close the current turn now, from a synchronous context. + + Sends ForceEndOfUtterance stamped with the audio timestamp at the moment of the call, + so the service closes the turn where the application's VAD detected the end of speech + rather than wherever the send happens to land. The flushed segment arrives as a normal + AddSegment message. + + Use this when the application brings its own VAD (`TurnDetectionMode.EXTERNAL`); with + `TurnDetectionMode.VAD` the service closes turns itself. + + Args: + timestamp: Audio timestamp in seconds for the end of the utterance. Defaults to + the amount of audio sent so far. Pass None to send no timestamp. + + Examples: + >>> client.finalize() + """ + resolved = self.audio_seconds_sent if timestamp is _UNSET else timestamp + try: + loop = asyncio.get_running_loop() + except RuntimeError: + self._logger.warning("finalize() needs a running event loop; use force_end_of_utterance()") + return + loop.create_task(self._send_force_end_of_utterance(resolved)) + + async def force_end_of_utterance(self, *, timestamp: Optional[float] | object = _UNSET) -> None: + """ + Close the current turn now, awaiting the send. + + The awaitable form of `finalize()`. + + Args: + timestamp: Audio timestamp in seconds for the end of the utterance. Defaults to + the amount of audio sent so far. Pass None to send no timestamp. + + Examples: + >>> await client.force_end_of_utterance() + """ + resolved = self.audio_seconds_sent if timestamp is _UNSET else timestamp + await self._send_force_end_of_utterance(resolved) + + async def _send_force_end_of_utterance(self, timestamp: Optional[float] | object) -> None: + """Send ForceEndOfUtterance, including the timestamp unless it is None.""" + message: dict[str, Any] = {"message": ClientMessageType.FORCE_END_OF_UTTERANCE} + if timestamp is not None: + message["timestamp"] = timestamp + + try: + await self.send_message(message) + except TransportError as e: + self._logger.warning("Error sending %s: %s", ClientMessageType.FORCE_END_OF_UTTERANCE, e) + return + self._finalize_sent_at = time.perf_counter() + + async def transcribe( + self, + source: BinaryIO, + *, + transcription_config: Optional[RTTranscriptionConfig] = None, + audio_format: Optional[AudioFormat] = None, + ws_headers: Optional[dict] = None, + timeout: Optional[float] = None, + ) -> None: + """ + Stream an audio source to the end, then close the session. + + Args: + source: Audio source with a `read()` method, holding raw PCM in the session's + audio format. + transcription_config: Transcription config for the session. + audio_format: Audio format. Must be 16 kHz raw PCM for the Agent STT service. + ws_headers: Additional WebSocket handshake headers. + timeout: Maximum time in seconds to wait for the stream to finish. + + Raises: + TimeoutError: If streaming exceeds the timeout. + TranscriptionError: If the service reports an error. + + Examples: + >>> with open("speech.raw", "rb") as audio: + ... await client.transcribe(audio) + >>> print(client.transcript) + """ + if transcription_config is not None: + self._config = transcription_config + if audio_format is not None: + self._audio_format = audio_format + + if not self._is_connected: + await self.start_session( + transcription_config=self._config, + audio_format=self._audio_format, + ws_headers=ws_headers, + ) + self._is_connected = True + + try: + await asyncio.wait_for( + self._audio_producer(source, self._audio_format.chunk_size), + timeout=timeout, + ) + except asyncio.TimeoutError as exc: + raise RTTimeoutError("Agent STT session timed out") from exc + finally: + self._is_connected = False + self._is_ready_for_audio = False + + # ========================================================================== + # Session output + # ========================================================================== + + @property + def transcript(self) -> str: + """The final segments received so far, joined by the language's word delimiter.""" + return self._transcript.text() + + @property + def segments(self) -> list[Segment]: + """The final segments received so far, in order.""" + return self._transcript.segments + + @property + def partial_segment(self) -> Optional[Segment]: + """The segment currently being built, or None when there is nothing in flight.""" + return self._transcript.partial + + @property + def timeline(self) -> list[TimedEvent]: + """The speech and turn events received so far, in order.""" + return self._transcript.timeline + + @property + def events(self) -> list[dict[str, Any]]: + """Every raw server message received, in order, including unmodelled ones.""" + return self._transcript.events + + @property + def session_info(self) -> SessionInfo: + """Session identity and the language pack reported by the service.""" + return self._session_info + + @property + def is_connected(self) -> bool: + """Whether the session is open.""" + return self._is_connected + + @property + def is_ready_for_audio(self) -> bool: + """Whether the service has accepted the session, so audio will be sent.""" + return self._is_ready_for_audio + + @property + def last_finalize_latency(self) -> float: + """Seconds between the last `finalize()` and the segment it flushed.""" + return self._last_finalize_latency + + def transcript_text(self, *, include_partial: bool = False, speaker_labels: bool = False) -> str: + """ + Render the transcript. + + Args: + include_partial: Append the segment currently in flight. + speaker_labels: Prefix each segment with its speaker label, where one is attributed. + + Returns: + The transcript as text. + + Examples: + >>> print(client.transcript_text(speaker_labels=True)) + """ + return self._transcript.text(include_partial=include_partial, speaker_labels=speaker_labels) + + def reset_transcript(self) -> None: + """Drop the accumulated segments, timeline and event log.""" + self._transcript.reset() + + # ========================================================================== + # Message handling + # ========================================================================== + + def emit(self, event: Any, message: dict[str, Any]) -> None: + """Record every server message before dispatching it to the application's handlers.""" + self._transcript.record(message) + super().emit(event, message) + + def _on_session_started(self, message: dict[str, Any]) -> None: + """Capture the session id and language pack, and open the audio gate.""" + self._session_info.session_id = message.get("id") + self._session_info.language_pack_info = LanguagePackInfo.from_dict(message.get("language_pack_info") or {}) + self._transcript.set_delimiter(self._session_info.language_pack_info.word_delimiter) + self._is_ready_for_audio = True + + def _on_segment(self, message: dict[str, Any]) -> None: + """Accumulate a final segment, or refresh the live partial.""" + if message.get("message") != ServerMessageType.ADD_SEGMENT: + self._transcript.add_partial_segment(message) + return + + self._transcript.add_segment(message) + if self._finalize_sent_at is not None: + self._last_finalize_latency = time.perf_counter() - self._finalize_sent_at + self._finalize_sent_at = None + + def _on_timed_event(self, message: dict[str, Any]) -> None: + """Append a speech or turn event to the timeline.""" + self._transcript.add_timed_event(message) + + async def close(self) -> None: + """Close the connection without waiting for outstanding messages.""" + self._is_ready_for_audio = False + self._is_connected = False + await super().close() + + +AgentSTTClient = AgentSttAsyncClient diff --git a/sdk/agent_stt/speechmatics/agent_stt/_logging.py b/sdk/agent_stt/speechmatics/agent_stt/_logging.py new file mode 100644 index 00000000..38178dbc --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_logging.py @@ -0,0 +1,23 @@ +import logging + + +def get_logger(name: str) -> logging.Logger: + """ + Get a logger that stays silent unless the application configures logging. + + Args: + name: Logger name, typically __name__ from the calling module. + + Returns: + Logger with a NullHandler attached. + + Examples: + >>> import logging + >>> logging.getLogger("speechmatics.agent_stt").setLevel(logging.DEBUG) + """ + logger = logging.getLogger(name) + logger.addHandler(logging.NullHandler()) + return logger + + +__all__ = ["get_logger"] diff --git a/sdk/agent_stt/speechmatics/agent_stt/_models.py b/sdk/agent_stt/speechmatics/agent_stt/_models.py new file mode 100644 index 00000000..03551991 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_models.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field +from enum import Enum +from typing import Any +from typing import Optional +from typing import Union +from typing import cast +from warnings import warn + +from speechmatics.rt import TranscriptionConfig as RTTranscriptionConfig + +DEFAULT_SAMPLE_RATE = 16000 +DEFAULT_CHUNK_SIZE = 1024 +DEFAULT_WORD_DELIMITER = " " + + +class Model(str, Enum): + """ + Models available on the Agent STT service. + + Attributes: + LINDEN_1: The first Agent STT model. + """ + + LINDEN_1 = "linden-1" + + +DEFAULT_MODEL = Model.LINDEN_1 + +_UNSET = cast(Model, object()) + + +class ClientMessageType(str, Enum): + """ + Message types that can be sent from client to the Agent STT service. + + Attributes: + START_RECOGNITION: Starts the session, carrying the audio and transcription config. + END_OF_STREAM: Signals that no more audio will be sent. + FORCE_END_OF_UTTERANCE: Closes the current turn immediately, flushing the buffered + segment. Sent by clients that run their own VAD. + GET_SPEAKERS: Requests the session's speaker data. + """ + + START_RECOGNITION = "StartRecognition" + END_OF_STREAM = "EndOfStream" + FORCE_END_OF_UTTERANCE = "ForceEndOfUtterance" + GET_SPEAKERS = "GetSpeakers" + + +class ServerMessageType(str, Enum): + """ + Message types that can be received from the Agent STT service. + + The service emits its own segment, speech and turn messages, and passes the RT engine's + messages through unchanged. + + Attributes: + RECOGNITION_STARTED: Session accepted; carries the session id and language pack info. + AUDIO_ADDED: Audio frame acknowledged. + ADD_SEGMENT: A finalized segment, closed at a turn or content boundary. + ADD_PARTIAL_SEGMENT: Interim preview of the segment being built. + SPEECH_STARTED: The service's VAD detected speech onset. + SPEECH_ENDED: The service's VAD detected speech offset. + START_OF_TURN: The service's turn detection opened a turn. + END_OF_TURN: The service's turn detection closed a turn. + END_OF_TRANSCRIPT: The service has finished sending messages. + SPEAKERS_RESULT: Response to GetSpeakers. + INFO: Informational message. + WARNING: Warning; the session continues, possibly with adjusted config. + ERROR: Error; the session is over. + + Examples: + >>> @client.on(ServerMessageType.ADD_SEGMENT) + >>> def handle_segment(message): + ... print(message["segment"]["transcript"]) + """ + + RECOGNITION_STARTED = "RecognitionStarted" + AUDIO_ADDED = "AudioAdded" + ADD_SEGMENT = "AddSegment" + ADD_PARTIAL_SEGMENT = "AddPartialSegment" + SPEECH_STARTED = "SpeechStarted" + SPEECH_ENDED = "SpeechEnded" + START_OF_TURN = "StartOfTurn" + END_OF_TURN = "EndOfTurn" + END_OF_TRANSCRIPT = "EndOfTranscript" + SPEAKERS_RESULT = "SpeakersResult" + INFO = "Info" + WARNING = "Warning" + ERROR = "Error" + + +SEGMENT_MESSAGES = (ServerMessageType.ADD_SEGMENT, ServerMessageType.ADD_PARTIAL_SEGMENT) + +TIMED_MESSAGES = ( + ServerMessageType.SPEECH_STARTED, + ServerMessageType.SPEECH_ENDED, + ServerMessageType.START_OF_TURN, + ServerMessageType.END_OF_TURN, +) + + +class TurnDetectionMode(str, Enum): + """ + Which mechanism closes a turn. This SDK never detects turn boundaries itself. + + The mode names the mechanism rather than the side it runs on, so a second service-side + mechanism - smart turn, once the service implements it - joins as another member without + changing what the existing ones mean. + + Attributes: + VAD: The service's own VAD closes turns, emitting SpeechStarted, SpeechEnded, + StartOfTurn and EndOfTurn, and closing segments itself. + EXTERNAL: The application closes each turn by calling `finalize()`, which sends + ForceEndOfUtterance with an audio timestamp. Whatever produced that signal is up to + the application - a VAD, a turn model, or a push-to-talk button - so a host + framework's own endpointing (Pipecat, LiveKit, ...) works unchanged. + """ + + VAD = "vad" + EXTERNAL = "external" + + +@dataclass +class VADConfig: + """ + Tuning for the service's VAD. Only applied when `TurnDetectionMode.VAD` is in use. + + Attributes: + window: Silence in seconds before the service closes the turn. + onset_threshold: Speech probability above which speech starts. + offset_threshold: Speech probability below which speech ends. + """ + + window: Optional[float] = None + onset_threshold: Optional[float] = None + offset_threshold: Optional[float] = None + + +@dataclass +class AdditionalVocabEntry: + """ + A word to bias the engine towards, optionally with pronunciation hints. + + Attributes: + content: The word or phrase. + sounds_like: Alternative pronunciations, written as they sound. + + Examples: + >>> AdditionalVocabEntry(content="Speechmatics", sounds_like=["speech matics"]) + """ + + content: str + sounds_like: Optional[list[str]] = None + + +@dataclass +class TranscriptionConfig(RTTranscriptionConfig): + """ + Transcription config for the Agent STT service. + + Extends the RT transcription config with the service-only fields (`vad_config`, + `emit_sentences`). See `speechmatics.rt.TranscriptionConfig` for the inherited fields. + + Attributes: + turn_detection_mode: Which mechanism closes a turn: the service's VAD, or the application. + vad_config: Tuning for the service's VAD, used when `turn_detection_mode` is `VAD`. + emit_sentences: Close a segment on every sentence boundary, not just at the turn + boundary. + additional_vocab: Words to bias the engine towards, as `AdditionalVocabEntry` objects + or raw dicts. + model: Agent STT model, defaulting to `DEFAULT_MODEL`. The proxy in front of the + service resolves the name to the engine's operating point, so a model the + transcriber has no notion of still routes correctly. + + Examples: + Service VAD, sentence-level segments: + >>> config = TranscriptionConfig(language="en", emit_sentences=True) + + External endpointing (Pipecat, LiveKit): + >>> config = TranscriptionConfig(language="en", turn_detection_mode=TurnDetectionMode.EXTERNAL) + """ + + model: Model = _UNSET + additional_vocab: Optional[list[Union[AdditionalVocabEntry, dict[str, Any]]]] = None + turn_detection_mode: TurnDetectionMode = TurnDetectionMode.VAD + vad_config: VADConfig = field(default_factory=VADConfig) + emit_sentences: Optional[bool] = None + + def __post_init__(self) -> None: + if self.model is not _UNSET and self.operating_point is not None: + raise ValueError("Cannot specify both 'model' and 'operating_point'. Use 'model' instead.") + if self.model is _UNSET and self.operating_point is None: + self.model = DEFAULT_MODEL + if self.operating_point is not None: + warn("'operating_point' is deprecated, use 'model' instead.", DeprecationWarning, stacklevel=2) + + def to_dict(self) -> dict[str, Any]: + """ + Convert to the wire form of `StartRecognition.transcription_config`. + + Returns: + The config as a dict, excluding None values. `vad_config.enabled` is derived + from `turn_detection_mode` - on unless the application closes turns itself - and + the mode itself is dropped. + """ + result = super().to_dict() + if self.model is _UNSET: + result.pop("model", None) + result.pop("turn_detection_mode", None) + vad_config = result.pop("vad_config", None) or {} + vad_config["enabled"] = self.turn_detection_mode is not TurnDetectionMode.EXTERNAL + result["vad_config"] = vad_config + return result + + +@dataclass +class Segment: + """ + A segment of transcript, the unit the Agent STT service works in. + + Attributes: + transcript: The rendered segment text. + start_time: Segment start in seconds from the start of the session. + end_time: Segment end in seconds from the start of the session. + speaker: Speaker label (e.g. "S1"), when diarization attributed one. + is_final: True for AddSegment, False for AddPartialSegment. + """ + + transcript: str + start_time: float = 0.0 + end_time: float = 0.0 + speaker: Optional[str] = None + is_final: bool = True + + @classmethod + def from_message(cls, message: dict[str, Any]) -> Segment: + """Create a Segment from an AddSegment or AddPartialSegment message.""" + segment = message.get("segment") or {} + metadata = message.get("metadata") or {} + return cls( + transcript=segment.get("transcript", ""), + start_time=metadata.get("start_time", 0.0), + end_time=metadata.get("end_time", 0.0), + speaker=segment.get("speaker"), + is_final=message.get("message") == ServerMessageType.ADD_SEGMENT, + ) + + +@dataclass +class TimedEvent: + """ + A speech or turn event from the service, reduced to its single timestamp. + + Attributes: + message: The message type (SpeechStarted, SpeechEnded, StartOfTurn, EndOfTurn). + time: The event time in seconds from the start of the session. + """ + + message: str + time: float = 0.0 + + @classmethod + def from_message(cls, message: dict[str, Any]) -> TimedEvent: + """Create a TimedEvent from a speech or turn message.""" + metadata = message.get("metadata") or {} + time = metadata.get("start_time", metadata.get("end_time", 0.0)) + return cls(message=message.get("message", ""), time=time) + + +@dataclass +class LanguagePackInfo: + """ + Language pack details reported in RecognitionStarted. + + Attributes: + language_description: Human-readable language name. + word_delimiter: The separator between words for this language, used when joining + segments into a transcript. + writing_direction: "ltr" or "rtl". + itn: Whether inverse text normalization is applied. + adapted: Whether the language pack is adapted. + """ + + language_description: str = "" + word_delimiter: str = DEFAULT_WORD_DELIMITER + writing_direction: str = "ltr" + itn: bool = True + adapted: bool = False + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> LanguagePackInfo: + """Create LanguagePackInfo from the RecognitionStarted language_pack_info block.""" + return cls( + language_description=data.get("language_description", ""), + word_delimiter=data.get("word_delimiter", DEFAULT_WORD_DELIMITER), + writing_direction=data.get("writing_direction", "ltr"), + itn=data.get("itn", True), + adapted=data.get("adapted", False), + ) + + +@dataclass +class SessionInfo: + """ + State of the current Agent STT session. + + Attributes: + request_id: Client-generated id for this session. + session_id: Service-assigned session id, set once RecognitionStarted arrives. + language_pack_info: Language pack reported in RecognitionStarted. + """ + + request_id: str + session_id: Optional[str] = None + language_pack_info: LanguagePackInfo = field(default_factory=LanguagePackInfo) diff --git a/sdk/agent_stt/speechmatics/agent_stt/_transcript.py b/sdk/agent_stt/speechmatics/agent_stt/_transcript.py new file mode 100644 index 00000000..d869c715 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_transcript.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from typing import Any +from typing import Optional + +from ._models import DEFAULT_WORD_DELIMITER +from ._models import Segment +from ._models import TimedEvent + + +class Transcript: + """ + Accumulates a session's output: final segments, the live partial, speech/turn events + and the raw message log. + + A session's transcript is the concatenation of its final segments, so the text reads the + same way an RT session's does, one segment at a time instead of one word group at a time. + + Args: + record_events: Whether to keep every raw server message in `events`. Messages this SDK + version does not model are still captured there. + delimiter: Separator used between segments. Replaced by the language pack's word + delimiter once RecognitionStarted arrives. + + Examples: + >>> transcript = Transcript() + >>> transcript.add_segment({"message": "AddSegment", + ... "segment": {"transcript": "Hello world"}, + ... "metadata": {"start_time": 0.0, "end_time": 1.0}}) + >>> transcript.text() + 'Hello world' + """ + + def __init__(self, *, record_events: bool = True, delimiter: str = DEFAULT_WORD_DELIMITER) -> None: + self._record_events = record_events + self._delimiter = delimiter + self._segments: list[Segment] = [] + self._partial: Optional[Segment] = None + self._events: list[dict[str, Any]] = [] + self._timeline: list[TimedEvent] = [] + + @property + def segments(self) -> list[Segment]: + """The final segments received so far, in order.""" + return list(self._segments) + + @property + def partial(self) -> Optional[Segment]: + """The segment currently being built, or None when there is nothing in flight.""" + return self._partial + + @property + def events(self) -> list[dict[str, Any]]: + """Every raw server message received, in order, when recording is enabled.""" + return list(self._events) + + @property + def timeline(self) -> list[TimedEvent]: + """The speech and turn events received so far, in order.""" + return list(self._timeline) + + @property + def delimiter(self) -> str: + """The separator used between segments.""" + return self._delimiter + + def set_delimiter(self, delimiter: str) -> None: + """Set the separator between segments, from the language pack's word delimiter.""" + self._delimiter = delimiter + + def record(self, message: dict[str, Any]) -> None: + """Append a raw server message to the event log.""" + if self._record_events: + self._events.append(message) + + def add_segment(self, message: dict[str, Any]) -> Segment: + """Add a final segment from an AddSegment message and clear the live partial.""" + segment = Segment.from_message(message) + self._segments.append(segment) + self._partial = None + return segment + + def add_partial_segment(self, message: dict[str, Any]) -> Segment: + """Replace the live partial with the one in an AddPartialSegment message.""" + self._partial = Segment.from_message(message) + return self._partial + + def add_timed_event(self, message: dict[str, Any]) -> TimedEvent: + """Add a speech or turn event to the timeline.""" + event = TimedEvent.from_message(message) + self._timeline.append(event) + return event + + def text(self, *, include_partial: bool = False, speaker_labels: bool = False) -> str: + """ + Render the transcript. + + Args: + include_partial: Append the segment currently in flight. + speaker_labels: Prefix each segment with its speaker label, where one is attributed. + + Returns: + The final segments joined by the session's delimiter. + """ + segments = list(self._segments) + if include_partial and self._partial is not None: + segments.append(self._partial) + + parts = [] + for segment in segments: + if not segment.transcript: + continue + if speaker_labels and segment.speaker: + parts.append(f"{segment.speaker}: {segment.transcript}") + else: + parts.append(segment.transcript) + return self._delimiter.join(parts) + + def reset(self) -> None: + """Drop all accumulated state, keeping the delimiter.""" + self._segments.clear() + self._partial = None + self._events.clear() + self._timeline.clear() + + def __str__(self) -> str: + return self.text() diff --git a/sdk/agent_stt/speechmatics/agent_stt/_transport.py b/sdk/agent_stt/speechmatics/agent_stt/_transport.py new file mode 100644 index 00000000..34bad960 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_transport.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from urllib.parse import parse_qsl +from urllib.parse import urlencode +from urllib.parse import urlparse +from urllib.parse import urlunparse + +from speechmatics.rt._transport import Transport + +from ._version import get_version + + +class AgentTransport(Transport): + """ + RT transport that identifies itself as the Agent STT SDK. + + Only the SDK identifier on the connection URL differs; the WebSocket handling, + authentication and message framing are the RT transport's. + """ + + def _prepare_url(self) -> str: + """Return the connection URL with the Agent STT SDK version as `sm-sdk`.""" + parsed = urlparse(self._url) + params = dict(parse_qsl(parsed.query, keep_blank_values=True)) + params["sm-sdk"] = f"python-agent-stt-sdk-v{get_version()}" + return urlunparse(parsed._replace(query=urlencode(params))) diff --git a/sdk/agent_stt/speechmatics/agent_stt/_url.py b/sdk/agent_stt/speechmatics/agent_stt/_url.py new file mode 100644 index 00000000..572d7086 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_url.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import os +from typing import Optional +from urllib.parse import parse_qs +from urllib.parse import urlencode +from urllib.parse import urlparse +from urllib.parse import urlunparse + +from ._version import get_version + +DEFAULT_RT_URL = "wss://eu2.rt.speechmatics.com/v2" +AGENT_PATH_SEGMENT = "agent" + + +def resolve_url( + url: Optional[str] = None, + app: Optional[str] = None, +) -> str: + """ + Resolve the Agent STT WebSocket URL. + + The Agent STT endpoint is the RT endpoint plus an `/agent` path segment. + + Args: + url: Explicit endpoint. Falls back to the `SPEECHMATICS_RT_URL` environment + variable, then the EU endpoint. + app: Optional application name reported to the service as `sm-app`. + + Returns: + The complete WebSocket URL. + + Examples: + >>> resolve_url() + 'wss://eu2.rt.speechmatics.com/v2/agent?sm-app=agent-stt-sdk%2F0.0.0' + >>> resolve_url("wss://host/v2", app="pipecat/1.0") + 'wss://host/v2/agent?sm-app=pipecat%2F1.0' + """ + base = url or os.getenv("SPEECHMATICS_RT_URL") or DEFAULT_RT_URL + parsed = urlparse(base) + return urlunparse( + parsed._replace( + path=_resolve_path(parsed.path), + query=_resolve_query(parsed.query, app), + ) + ) + + +def _resolve_path(path: str) -> str: + """Append the /agent segment, skipping any already present.""" + segments = [segment for segment in path.split("/") if segment] + + if AGENT_PATH_SEGMENT not in segments: + segments.append(AGENT_PATH_SEGMENT) + return "/" + "/".join(segments) + + +def _resolve_query(query: str, app: Optional[str]) -> str: + """Set the sm-app parameter, keeping any parameters already on the URL.""" + params = parse_qs(query, keep_blank_values=True) + existing_app = params.get("sm-app", [None])[0] + params["sm-app"] = [app or existing_app or f"agent-stt-sdk/{get_version()}"] + return urlencode(params, doseq=True) diff --git a/sdk/agent_stt/speechmatics/agent_stt/_version.py b/sdk/agent_stt/speechmatics/agent_stt/_version.py new file mode 100644 index 00000000..dca9855c --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_version.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import importlib.metadata + + +def get_version() -> str: + """ + Get the installed version of the speechmatics-agent-stt package. + + This function attempts to retrieve the package version using multiple + fallback strategies to ensure it works in various deployment scenarios. + + Returns: + str: The package version string (e.g., "1.2.3"), or "0.0.0" if + version cannot be determined. + """ + try: + return importlib.metadata.version("speechmatics-agent-stt") + except importlib.metadata.PackageNotFoundError: + try: + from . import __version__ + + return __version__ + except ImportError: + return "0.0.0" diff --git a/sdk/agent_stt/speechmatics/agent_stt/py.typed b/sdk/agent_stt/speechmatics/agent_stt/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/tests/agent_stt/test_client.py b/tests/agent_stt/test_client.py new file mode 100644 index 00000000..4f8d91be --- /dev/null +++ b/tests/agent_stt/test_client.py @@ -0,0 +1,269 @@ +import json + +import pytest + +from speechmatics.agent_stt import AgentSttAsyncClient +from speechmatics.agent_stt import AudioEncoding +from speechmatics.agent_stt import ClientMessageType +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import TurnDetectionMode + +API_KEY = "test-key" + + +class StubTransport: + """Captures what the client sends instead of opening a WebSocket.""" + + def __init__(self): + self.sent = [] + self.closed = False + + async def send_message(self, payload): + self.sent.append(payload) + + async def close(self): + self.closed = True + + @property + def messages(self): + return [json.loads(payload) for payload in self.sent if isinstance(payload, str)] + + @property + def audio(self): + return [payload for payload in self.sent if isinstance(payload, bytes)] + + +@pytest.fixture +def client(monkeypatch): + monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) + return AgentSttAsyncClient(api_key=API_KEY) + + +def recognition_started(word_delimiter=" "): + return { + "message": ServerMessageType.RECOGNITION_STARTED, + "id": "session-1", + "language_pack_info": {"language_description": "English", "word_delimiter": word_delimiter}, + } + + +def segment_message(transcript, is_final=True, speaker=None, start=0.0, end=1.0): + segment = {"transcript": transcript} + if speaker is not None: + segment["speaker"] = speaker + return { + "message": ServerMessageType.ADD_SEGMENT if is_final else ServerMessageType.ADD_PARTIAL_SEGMENT, + "segment": segment, + "metadata": {"start_time": start, "end_time": end}, + } + + +def start_session(client): + """Put the client in the state it reaches after RecognitionStarted.""" + transport = StubTransport() + client._transport = transport + client.emit(ServerMessageType.RECOGNITION_STARTED, recognition_started()) + return transport + + +@pytest.mark.asyncio +async def test_endpoint_is_the_agent_path(client): + assert client._transport._url.startswith("wss://eu2.rt.speechmatics.com/v2/agent") + + +@pytest.mark.asyncio +async def test_app_reaches_the_url(monkeypatch): + monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) + client = AgentSttAsyncClient(api_key=API_KEY, app="pipecat/1.0") + assert "/v2/agent" in client._transport._url + assert "sm-app=pipecat%2F1.0" in client._transport._url + + +@pytest.mark.asyncio +async def test_sdk_identifier(client): + assert "sm-sdk=python-agent-stt-sdk-v" in client._transport._prepare_url() + + +@pytest.mark.asyncio +async def test_audio_format_defaults_to_16k_pcm(client): + assert client._audio_format.encoding == AudioEncoding.PCM_S16LE + assert client._audio_format.sample_rate == 16000 + + +@pytest.mark.asyncio +async def test_recognition_started_sets_session_state(client): + start_session(client) + assert client.is_ready_for_audio + assert client.session_info.session_id == "session-1" + assert client.session_info.language_pack_info.language_description == "English" + + +@pytest.mark.asyncio +async def test_transcript_accumulates_from_segments(client): + start_session(client) + client.emit(ServerMessageType.ADD_PARTIAL_SEGMENT, segment_message("Hello th", is_final=False)) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hello there.")) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("How are you?")) + + assert client.transcript == "Hello there. How are you?" + assert client.partial_segment is None + assert [segment.transcript for segment in client.segments] == ["Hello there.", "How are you?"] + + +@pytest.mark.asyncio +async def test_transcript_uses_language_pack_delimiter(client): + client._transport = StubTransport() + client.emit(ServerMessageType.RECOGNITION_STARTED, recognition_started(word_delimiter="")) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("你好")) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("世界")) + assert client.transcript == "你好世界" + + +@pytest.mark.asyncio +async def test_transcript_with_speaker_labels(client): + start_session(client) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hello.", speaker="S1")) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hi.", speaker="S2")) + assert client.transcript_text(speaker_labels=True) == "S1: Hello. S2: Hi." + + +@pytest.mark.asyncio +async def test_timeline_records_speech_and_turn_events(client): + start_session(client) + client.emit(ServerMessageType.SPEECH_STARTED, {"message": "SpeechStarted", "metadata": {"start_time": 0.5}}) + client.emit(ServerMessageType.START_OF_TURN, {"message": "StartOfTurn", "metadata": {"start_time": 0.5}}) + client.emit(ServerMessageType.SPEECH_ENDED, {"message": "SpeechEnded", "metadata": {"end_time": 2.0}}) + client.emit(ServerMessageType.END_OF_TURN, {"message": "EndOfTurn", "metadata": {"end_time": 2.2}}) + + assert [(event.message, event.time) for event in client.timeline] == [ + ("SpeechStarted", 0.5), + ("StartOfTurn", 0.5), + ("SpeechEnded", 2.0), + ("EndOfTurn", 2.2), + ] + + +@pytest.mark.asyncio +async def test_every_message_is_recorded(client): + start_session(client) + unmodelled = {"message": "SomeFutureMessage", "metadata": {"value": 1}} + client.emit("SomeFutureMessage", unmodelled) + + recorded = [event["message"] for event in client.events] + assert recorded == [ServerMessageType.RECOGNITION_STARTED, "SomeFutureMessage"] + assert client.events[-1] == unmodelled + + +@pytest.mark.asyncio +async def test_event_recording_can_be_disabled(monkeypatch): + monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) + client = AgentSttAsyncClient(api_key=API_KEY, record_events=False) + start_session(client) + assert client.events == [] + assert client.session_info.session_id == "session-1" + + +@pytest.mark.asyncio +async def test_audio_dropped_until_session_is_ready(client): + transport = StubTransport() + client._transport = transport + + await client.send_audio(b"\x00" * 32) + assert transport.audio == [] + + client.emit(ServerMessageType.RECOGNITION_STARTED, recognition_started()) + await client.send_audio(b"\x00" * 32) + assert transport.audio == [b"\x00" * 32] + + +@pytest.mark.asyncio +async def test_finalize_sends_force_end_of_utterance_with_timestamp(client): + transport = start_session(client) + await client.send_audio(b"\x00" * 32000) # 1 second of 16 kHz signed 16-bit PCM + + await client.force_end_of_utterance() + + assert transport.messages == [ + {"message": ClientMessageType.FORCE_END_OF_UTTERANCE.value, "timestamp": 1.0}, + ] + + +@pytest.mark.asyncio +async def test_finalize_accepts_an_explicit_timestamp(client): + transport = start_session(client) + await client.force_end_of_utterance(timestamp=4.2) + assert transport.messages[0]["timestamp"] == 4.2 + + +@pytest.mark.asyncio +async def test_finalize_can_omit_the_timestamp(client): + transport = start_session(client) + await client.force_end_of_utterance(timestamp=None) + assert "timestamp" not in transport.messages[0] + + +@pytest.mark.asyncio +async def test_sync_finalize_schedules_the_send(client): + transport = start_session(client) + await client.send_audio(b"\x00" * 16000) # 0.5 seconds + + client.finalize() + assert transport.messages == [] # scheduled, not yet sent + + await _drain() + assert transport.messages == [ + {"message": ClientMessageType.FORCE_END_OF_UTTERANCE.value, "timestamp": 0.5}, + ] + + +@pytest.mark.asyncio +async def test_finalize_latency_measured_against_the_flushed_segment(client): + start_session(client) + assert client.last_finalize_latency == 0.0 + + await client.force_end_of_utterance() + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hello there.")) + assert client.last_finalize_latency > 0.0 + + +@pytest.mark.asyncio +async def test_external_turn_detection_reaches_start_recognition(client): + transport = StubTransport() + client._transport = transport + client._config = TranscriptionConfig(language="en", turn_detection_mode=TurnDetectionMode.EXTERNAL) + + await client.send_message( + { + "message": ClientMessageType.START_RECOGNITION.value, + "transcription_config": client._config.to_dict(), + } + ) + + assert transport.messages[0]["transcription_config"]["vad_config"] == {"enabled": False} + + +@pytest.mark.asyncio +async def test_reset_transcript(client): + start_session(client) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hello.")) + client.reset_transcript() + assert client.transcript == "" + assert client.events == [] + + +@pytest.mark.asyncio +async def test_close_clears_session_state(client): + transport = start_session(client) + await client.close() + assert transport.closed + assert not client.is_connected + assert not client.is_ready_for_audio + + +async def _drain(): + """Let scheduled tasks run.""" + import asyncio + + await asyncio.sleep(0) + await asyncio.sleep(0) diff --git a/tests/agent_stt/test_config.py b/tests/agent_stt/test_config.py new file mode 100644 index 00000000..99ecf756 --- /dev/null +++ b/tests/agent_stt/test_config.py @@ -0,0 +1,70 @@ +import pytest + +from speechmatics.agent_stt import Model +from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import TurnDetectionMode +from speechmatics.agent_stt import VADConfig + + +def test_service_vad_is_the_default(): + assert TranscriptionConfig().to_dict()["vad_config"] == {"enabled": True} + + +def test_external_mode_disables_service_vad(): + config = TranscriptionConfig(turn_detection_mode=TurnDetectionMode.EXTERNAL) + assert config.to_dict()["vad_config"] == {"enabled": False} + + +def test_turn_detection_mode_is_not_sent(): + assert "turn_detection_mode" not in TranscriptionConfig().to_dict() + + +def test_vad_tuning_passed_through(): + config = TranscriptionConfig(vad_config=VADConfig(window=0.3, onset_threshold=0.6, offset_threshold=0.4)) + assert config.to_dict()["vad_config"] == { + "window": 0.3, + "onset_threshold": 0.6, + "offset_threshold": 0.4, + "enabled": True, + } + + +def test_model_defaults_to_linden_1(): + assert TranscriptionConfig().to_dict()["model"] == "linden-1" + + +def test_model_sent_when_given(): + assert TranscriptionConfig(model=Model.LINDEN_1).to_dict()["model"] == Model.LINDEN_1 + + +def test_model_omitted_when_operating_point_is_used(): + """The deprecated operating_point must not arrive alongside a defaulted model.""" + with pytest.warns(DeprecationWarning): + config = TranscriptionConfig(operating_point="enhanced") + assert "model" not in config.to_dict() + assert config.to_dict()["operating_point"] == "enhanced" + + +def test_emit_sentences_omitted_unless_set(): + assert "emit_sentences" not in TranscriptionConfig().to_dict() + assert TranscriptionConfig(emit_sentences=True).to_dict()["emit_sentences"] is True + assert TranscriptionConfig(emit_sentences=False).to_dict()["emit_sentences"] is False + + +def test_rt_fields_still_work(): + config = TranscriptionConfig(language="es", diarization="speaker", enable_partials=True, max_delay=1.5) + result = config.to_dict() + assert result["language"] == "es" + assert result["diarization"] == "speaker" + assert result["enable_partials"] is True + assert result["max_delay"] == 1.5 + + +def test_model_and_operating_point_conflict(): + with pytest.raises(ValueError): + TranscriptionConfig(model=Model.LINDEN_1, operating_point="enhanced") + + +def test_operating_point_deprecated(): + with pytest.warns(DeprecationWarning): + TranscriptionConfig(operating_point="enhanced") diff --git a/tests/agent_stt/test_transcript.py b/tests/agent_stt/test_transcript.py new file mode 100644 index 00000000..bbbb347f --- /dev/null +++ b/tests/agent_stt/test_transcript.py @@ -0,0 +1,122 @@ +from speechmatics.agent_stt import Segment +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TimedEvent +from speechmatics.agent_stt import Transcript + + +def segment_message(transcript, start=0.0, end=1.0, speaker=None, is_final=True): + segment = {"transcript": transcript} + if speaker is not None: + segment["speaker"] = speaker + return { + "message": ServerMessageType.ADD_SEGMENT if is_final else ServerMessageType.ADD_PARTIAL_SEGMENT, + "segment": segment, + "metadata": {"start_time": start, "end_time": end}, + } + + +def test_segment_from_message(): + segment = Segment.from_message(segment_message("Hello world", 1.0, 2.5, speaker="S1")) + assert segment == Segment(transcript="Hello world", start_time=1.0, end_time=2.5, speaker="S1", is_final=True) + + +def test_partial_segment_is_not_final(): + assert Segment.from_message(segment_message("Hello", is_final=False)).is_final is False + + +def test_segment_without_speaker(): + assert Segment.from_message(segment_message("Hello")).speaker is None + + +def test_timed_event_from_start_time(): + event = TimedEvent.from_message({"message": "StartOfTurn", "metadata": {"start_time": 1.25}}) + assert event == TimedEvent(message="StartOfTurn", time=1.25) + + +def test_timed_event_from_end_time(): + event = TimedEvent.from_message({"message": "EndOfTurn", "metadata": {"end_time": 3.5}}) + assert event == TimedEvent(message="EndOfTurn", time=3.5) + + +def test_finals_accumulate(): + transcript = Transcript() + transcript.add_segment(segment_message("Hello there.")) + transcript.add_segment(segment_message("How are you?")) + assert transcript.text() == "Hello there. How are you?" + assert len(transcript.segments) == 2 + + +def test_partial_replaced_and_cleared_by_final(): + transcript = Transcript() + transcript.add_partial_segment(segment_message("Hel", is_final=False)) + transcript.add_partial_segment(segment_message("Hello wor", is_final=False)) + assert transcript.partial is not None + assert transcript.partial.transcript == "Hello wor" + assert transcript.text() == "" + + transcript.add_segment(segment_message("Hello world.")) + assert transcript.partial is None + assert transcript.text() == "Hello world." + + +def test_text_including_partial(): + transcript = Transcript() + transcript.add_segment(segment_message("Hello.")) + transcript.add_partial_segment(segment_message("How are", is_final=False)) + assert transcript.text() == "Hello." + assert transcript.text(include_partial=True) == "Hello. How are" + + +def test_text_with_speaker_labels(): + transcript = Transcript() + transcript.add_segment(segment_message("Hello.", speaker="S1")) + transcript.add_segment(segment_message("Hi.", speaker="S2")) + transcript.add_segment(segment_message("No speaker.")) + assert transcript.text(speaker_labels=True) == "S1: Hello. S2: Hi. No speaker." + + +def test_delimiter_from_language_pack(): + transcript = Transcript() + transcript.set_delimiter("") + transcript.add_segment(segment_message("你好")) + transcript.add_segment(segment_message("世界")) + assert transcript.text() == "你好世界" + + +def test_empty_segments_skipped(): + transcript = Transcript() + transcript.add_segment(segment_message("Hello.")) + transcript.add_segment(segment_message("")) + assert transcript.text() == "Hello." + + +def test_events_recorded(): + transcript = Transcript() + message = segment_message("Hello.") + transcript.record(message) + transcript.record({"message": "SomeFutureMessage", "payload": 1}) + assert transcript.events == [message, {"message": "SomeFutureMessage", "payload": 1}] + + +def test_event_recording_can_be_disabled(): + transcript = Transcript(record_events=False) + transcript.record(segment_message("Hello.")) + assert transcript.events == [] + + +def test_timeline(): + transcript = Transcript() + transcript.add_timed_event({"message": "SpeechStarted", "metadata": {"start_time": 0.5}}) + transcript.add_timed_event({"message": "EndOfTurn", "metadata": {"end_time": 2.0}}) + assert [event.message for event in transcript.timeline] == ["SpeechStarted", "EndOfTurn"] + + +def test_reset_keeps_delimiter(): + transcript = Transcript() + transcript.set_delimiter("") + transcript.add_segment(segment_message("Hello.")) + transcript.record(segment_message("Hello.")) + transcript.reset() + assert transcript.segments == [] + assert transcript.events == [] + assert transcript.delimiter == "" diff --git a/tests/agent_stt/test_url.py b/tests/agent_stt/test_url.py new file mode 100644 index 00000000..ef8b1d0d --- /dev/null +++ b/tests/agent_stt/test_url.py @@ -0,0 +1,60 @@ +from urllib.parse import parse_qs +from urllib.parse import urlparse + +import pytest + +from speechmatics.agent_stt import resolve_url + + +@pytest.fixture(autouse=True) +def _clear_env(monkeypatch): + monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) + + +def _path(url): + return urlparse(url).path + + +def test_default_url(): + url = urlparse(resolve_url()) + assert url.hostname == "eu2.rt.speechmatics.com" + assert url.path == "/v2/agent" + + +def test_agent_segment_appended_to_rt_url(monkeypatch): + monkeypatch.setenv("SPEECHMATICS_RT_URL", "wss://example.com/v2") + assert _path(resolve_url()) == "/v2/agent" + + +def test_agent_segment_not_duplicated_on_rt_url(monkeypatch): + monkeypatch.setenv("SPEECHMATICS_RT_URL", "wss://example.com/v2/agent") + assert _path(resolve_url()) == "/v2/agent" + + +def test_explicit_url_wins_over_env(monkeypatch): + monkeypatch.setenv("SPEECHMATICS_RT_URL", "wss://rt.example.com/v2") + assert urlparse(resolve_url("wss://custom.example.com/v2")).hostname == "custom.example.com" + + +def test_agent_segment_not_duplicated(): + assert _path(resolve_url("wss://example.com/v2/agent")) == "/v2/agent" + + +def test_trailing_slash_normalized(): + assert _path(resolve_url("wss://example.com/v2/")) == "/v2/agent" + + +def test_app_reported(): + params = parse_qs(urlparse(resolve_url("wss://example.com/v2", app="pipecat/1.0")).query) + assert params["sm-app"] == ["pipecat/1.0"] + + +def test_default_app_reported(): + params = parse_qs(urlparse(resolve_url("wss://example.com/v2")).query) + assert params["sm-app"][0].startswith("agent-stt-sdk/") + + +def test_existing_query_params_kept(): + params = parse_qs(urlparse(resolve_url("wss://example.com/v2?foo=bar&sm-app=existing")).query) + assert params["foo"] == ["bar"] + assert params["sm-app"] == ["existing"]