From 55c9e5d7a6e8ceabd81867afd13cc4f914d098f1 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Wed, 12 Aug 2026 22:52:19 +0100 Subject: [PATCH 01/19] Add Agent STT SDK speechmatics-agent-stt is an extension of the RT SDK for the Voice Agent Service. The service works in segments rather than word groups and reports speech and turn events, so the client consumes AddSegment/AddPartialSegment and accumulates the session transcript from them. The SDK runs no VAD or turn detection of its own. Either the service's VAD closes turns (VADMode.SERVER), or the host framework's does (VADMode.CLIENT) by calling finalize(), which sends ForceEndOfUtterance stamped with the audio position at the moment of the call. This is what Pipecat and LiveKit need, and replaces the voice SDK's in-process VAD and smart-turn models. - endpoint is the RT URL plus /agent, optionally plus a service profile - TranscriptionConfig adds vad_mode, vad_config and emit_sentences, and leaves model unset so it cannot conflict with the profile's locked operating_point - every server message is kept in client.events, including unmodelled ones - 16 kHz raw PCM defaults, as the service requires Verified against the real service with a stub transcriber behind it. Nothing in the voice SDK or Pipecat is touched; see sdk/agent_stt/PLAN.md for the Pipecat migration plan. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/release-agent-stt.yaml | 90 ++++ .github/workflows/test.yaml | 20 + Makefile | 51 +- README.md | 11 + examples/agent_stt/README.md | 19 + examples/agent_stt/client_vad/main.py | 56 ++ examples/agent_stt/file/main.py | 59 +++ examples/agent_stt/microphone/main.py | 60 +++ pyproject.toml | 2 +- sdk/agent_stt/PLAN.md | 135 +++++ sdk/agent_stt/README.md | 130 +++++ sdk/agent_stt/pyproject.toml | 69 +++ sdk/agent_stt/speechmatics/__init__.py | 1 + .../speechmatics/agent_stt/__init__.py | 109 ++++ .../speechmatics/agent_stt/_client.py | 489 ++++++++++++++++++ .../speechmatics/agent_stt/_logging.py | 23 + .../speechmatics/agent_stt/_models.py | 287 ++++++++++ .../speechmatics/agent_stt/_transcript.py | 127 +++++ .../speechmatics/agent_stt/_transport.py | 26 + sdk/agent_stt/speechmatics/agent_stt/_url.py | 72 +++ .../speechmatics/agent_stt/_version.py | 21 + sdk/agent_stt/speechmatics/agent_stt/py.typed | 0 tests/agent_stt/test_client.py | 271 ++++++++++ tests/agent_stt/test_config.py | 63 +++ tests/agent_stt/test_transcript.py | 122 +++++ tests/agent_stt/test_url.py | 75 +++ 26 files changed, 2375 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/release-agent-stt.yaml create mode 100644 examples/agent_stt/README.md create mode 100644 examples/agent_stt/client_vad/main.py create mode 100644 examples/agent_stt/file/main.py create mode 100644 examples/agent_stt/microphone/main.py create mode 100644 sdk/agent_stt/PLAN.md create mode 100644 sdk/agent_stt/README.md create mode 100644 sdk/agent_stt/pyproject.toml create mode 100644 sdk/agent_stt/speechmatics/__init__.py create mode 100644 sdk/agent_stt/speechmatics/agent_stt/__init__.py create mode 100644 sdk/agent_stt/speechmatics/agent_stt/_client.py create mode 100644 sdk/agent_stt/speechmatics/agent_stt/_logging.py create mode 100644 sdk/agent_stt/speechmatics/agent_stt/_models.py create mode 100644 sdk/agent_stt/speechmatics/agent_stt/_transcript.py create mode 100644 sdk/agent_stt/speechmatics/agent_stt/_transport.py create mode 100644 sdk/agent_stt/speechmatics/agent_stt/_url.py create mode 100644 sdk/agent_stt/speechmatics/agent_stt/_version.py create mode 100644 sdk/agent_stt/speechmatics/agent_stt/py.typed create mode 100644 tests/agent_stt/test_client.py create mode 100644 tests/agent_stt/test_config.py create mode 100644 tests/agent_stt/test_transcript.py create mode 100644 tests/agent_stt/test_url.py diff --git a/.github/workflows/release-agent-stt.yaml b/.github/workflows/release-agent-stt.yaml new file mode 100644 index 00000000..3e0ed4f5 --- /dev/null +++ b/.github/workflows/release-agent-stt.yaml @@ -0,0 +1,90 @@ +name: Release Agent STT SDK + +on: + push: + tags: + - "agent-stt/v*" + +permissions: + contents: read + id-token: write + +jobs: + extract-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.extract.outputs.version }} + steps: + - name: Extract version from tag + id: extract + run: | + # Extract version from tag (agent-stt/v1.0.0 -> 1.0.0) + VERSION=${GITHUB_REF#refs/tags/agent-stt/v} + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + test-agent-stt: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Test Agent STT SDK + run: | + make install-dev + make lint-agent-stt + make test-agent-stt + + release-build: + runs-on: ubuntu-latest + needs: [extract-version, test-agent-stt] + outputs: + version: ${{ needs.extract-version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Update package version in sdk/agent_stt/speechmatics/agent_stt/__init__.py + run: | + VERSION="${{ needs.extract-version.outputs.version }}" + sed -i "s/0\.0\.0/$VERSION/g" ./sdk/agent_stt/speechmatics/agent_stt/__init__.py + echo "Updated version to: $VERSION" + cat ./sdk/agent_stt/speechmatics/agent_stt/__init__.py | grep __version__ + + - name: Build Agent STT SDK + run: | + make install-dev + make build-agent-stt + + - name: Upload dist + uses: actions/upload-artifact@v4 + with: + name: agent-stt-release-dist + path: sdk/agent_stt/dist/ + + pypi-publish: + runs-on: ubuntu-latest + needs: [release-build] + environment: + name: pypi-agent-stt + url: https://pypi.org/project/speechmatics-agent-stt/${{ needs.release-build.outputs.version }} + + steps: + - name: Retrieve release dist + uses: actions/download-artifact@v4 + with: + name: agent-stt-release-dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + password: ${{ secrets.PYPI_ORG_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7928371f..be4f6a64 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -90,6 +90,26 @@ jobs: - name: Build Voice Agent SDK run: make build-voice + test-agent-stt: + name: Test Agent STT SDK + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: make install-dev + - name: Lint Agent STT SDK + run: make lint-agent-stt + - name: Test Agent STT SDK + run: make test-agent-stt + - name: Build Agent STT SDK + run: make build-agent-stt + test-tts: name: Test TTS SDK runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 12d4d8ae..b9f9cb99 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,12 @@ # Makefile for Speechmatics Python SDKs .PHONY: help -.PHONY: test-all test-rt test-batch test-flow test-tts test-voice -.PHONY: format-all format-rt format-batch format-flow format-tts format-voice -.PHONY: lint-all lint-rt lint-batch lint-flow lint-tts lint-voice -.PHONY: type-check-all type-check-rt type-check-batch type-check-flow type-check-tts type-check-voice -.PHONY: build-all build-rt build-batch build-flow build-tts build-voice -.PHONY: clean-all clean-rt clean-batch clean-flow clean-tts clean-voice +.PHONY: test-all test-rt test-batch test-flow test-tts test-voice test-agent-stt +.PHONY: format-all format-rt format-batch format-flow format-tts format-voice format-agent-stt +.PHONY: lint-all lint-rt lint-batch lint-flow lint-tts lint-voice lint-agent-stt +.PHONY: type-check-all type-check-rt type-check-batch type-check-flow type-check-tts type-check-voice type-check-agent-stt +.PHONY: build-all build-rt build-batch build-flow build-tts build-voice build-agent-stt +.PHONY: clean-all clean-rt clean-batch clean-flow clean-tts clean-voice clean-agent-stt help: @@ -19,6 +19,7 @@ help: @echo " test-flow Run tests for Flow SDK" @echo " test-tts Run tests for TTS SDK" @echo " test-voice Run tests for Voice Agent SDK" + @echo " test-agent-stt Run tests for Agent STT SDK" @echo "" @echo "Code formatting:" @echo " format-all Auto-fix formatting for all SDKs" @@ -27,6 +28,7 @@ help: @echo " format-flow Auto-fix formatting for Flow SDK" @echo " format-tts Auto-fix formatting for TTS SDK" @echo " format-voice Auto-fix formatting for Voice Agent SDK" + @echo " format-agent-stt Auto-fix formatting for Agent STT SDK" @echo "" @echo "Linting:" @echo " lint-all Run linting for all SDKs" @@ -35,6 +37,7 @@ help: @echo " lint-flow Run linting for Flow SDK" @echo " lint-tts Run linting for TTS SDK" @echo " lint-voice Run linting for Voice Agent SDK" + @echo " lint-agent-stt Run linting for Agent STT SDK" @echo "" @echo "Type checking:" @echo " type-check-all Run type checking for all SDKs" @@ -43,6 +46,7 @@ help: @echo " type-check-flow Run type checking for Flow SDK" @echo " type-check-tts Run type checking for TTS SDK" @echo " type-check-voice Run type checking for Voice Agent SDK" + @echo " type-check-agent-stt Run type checking for Agent STT SDK" @echo "" @echo "Building:" @echo " build-all Build all SDKs" @@ -51,6 +55,7 @@ help: @echo " build-flow Build Flow SDK" @echo " build-tts Build TTS SDK" @echo " build-voice Build Voice Agent SDK" + @echo " build-agent-stt Build Agent STT SDK" @echo "" @echo "Cleaning:" @echo " clean-all Clean all SDKs" @@ -59,10 +64,11 @@ help: @echo " clean-flow Clean Flow SDK build artifacts" @echo " clean-tts Clean TTS SDK build artifacts" @echo " clean-voice Clean Voice Agent SDK build artifacts" + @echo " clean-agent-stt Clean Agent STT SDK build artifacts" @echo "" # Testing targets -test-all: test-rt test-batch test-flow test-tts test-voice +test-all: test-rt test-batch test-flow test-tts test-voice test-agent-stt test-rt: pytest tests/rt/ -v -s @@ -78,8 +84,11 @@ test-tts: test-voice: pytest tests/voice/ -v -s +test-agent-stt: + pytest tests/agent_stt/ -v -s + # Formatting targets -format-all: format-rt format-batch format-flow format-tts format-voice format-tests format-examples +format-all: format-rt format-batch format-flow format-tts format-voice format-agent-stt format-tests format-examples format-rt: cd sdk/rt/speechmatics && black . @@ -101,6 +110,10 @@ format-voice: cd sdk/voice/speechmatics && black . cd sdk/voice/speechmatics && ruff check --fix . +format-agent-stt: + cd sdk/agent_stt/speechmatics && black . + cd sdk/agent_stt/speechmatics && ruff check --fix . + format-tests: cd tests && black . cd tests && ruff check --fix . @@ -110,7 +123,7 @@ format-examples: cd examples && ruff check --fix . # Linting targets -lint-all: lint-rt lint-batch lint-flow lint-tts lint-voice +lint-all: lint-rt lint-batch lint-flow lint-tts lint-voice lint-agent-stt lint-rt: cd sdk/rt/speechmatics && ruff check . @@ -127,8 +140,11 @@ lint-tts: lint-voice: cd sdk/voice/speechmatics && ruff check . +lint-agent-stt: + cd sdk/agent_stt/speechmatics && ruff check . + # Type checking targets -type-check-all: type-check-rt type-check-batch type-check-flow type-check-tts type-check-voice +type-check-all: type-check-rt type-check-batch type-check-flow type-check-tts type-check-voice type-check-agent-stt type-check-rt: cd sdk/rt/speechmatics && mypy . @@ -144,6 +160,9 @@ type-check-tts: type-check-voice: cd sdk/voice/speechmatics && mypy . +type-check-agent-stt: + cd sdk/agent_stt/speechmatics && mypy . + # Installation targets install-dev: python -m pip install --upgrade pip @@ -152,12 +171,13 @@ install-dev: python -m pip install -e sdk/flow[dev] python -m pip install -e sdk/tts[dev] python -m pip install -e sdk/voice[dev] + python -m pip install -e sdk/agent_stt[dev] install-build: python -m pip install --upgrade build # Building targets -build-all: build-rt build-batch build-flow build-tts build-voice +build-all: build-rt build-batch build-flow build-tts build-voice build-agent-stt build-rt: install-build cd sdk/rt && python -m build @@ -174,8 +194,11 @@ build-tts: install-build build-voice: install-build cd sdk/voice && python -m build +build-agent-stt: install-build + cd sdk/agent_stt && python -m build + # Cleaning targets -clean-all: clean-rt clean-batch clean-flow clean-tts clean-voice clean-test clean-examples +clean-all: clean-rt clean-batch clean-flow clean-tts clean-voice clean-agent-stt clean-test clean-examples clean-rt: rm -rf sdk/rt/dist sdk/rt/build sdk/rt/*.egg-info find sdk/rt -name __pycache__ -exec rm -rf {} + 2>/dev/null || true @@ -196,6 +219,10 @@ clean-voice: rm -rf sdk/voice/dist sdk/voice/build sdk/voice/*.egg-info find sdk/voice -name __pycache__ -exec rm -rf {} + 2>/dev/null || true +clean-agent-stt: + rm -rf sdk/agent_stt/dist sdk/agent_stt/build sdk/agent_stt/*.egg-info + find sdk/agent_stt -name __pycache__ -exec rm -rf {} + 2>/dev/null || true + clean-test: find tests -name __pycache__ -exec rm -rf {} + 2>/dev/null || true rm -rf .pytest_cache diff --git a/README.md b/README.md index 9a4faa0c..1bf015cd 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ [![PyPI - batch](https://img.shields.io/pypi/v/speechmatics-batch?label=batch)](https://pypi.org/project/speechmatics-batch/) [![PyPI - rt](https://img.shields.io/pypi/v/speechmatics-rt?label=rt)](https://pypi.org/project/speechmatics-rt/) [![PyPI - voice](https://img.shields.io/pypi/v/speechmatics-voice?label=voice)](https://pypi.org/project/speechmatics-voice/) +[![PyPI - agent-stt](https://img.shields.io/pypi/v/speechmatics-agent-stt?label=agent-stt)](https://pypi.org/project/speechmatics-agent-stt/) [![Python Versions](https://img.shields.io/pypi/pyversions/speechmatics-batch.svg)](https://pypi.org/project/speechmatics-batch/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/speechmatics/speechmatics-python-sdk/blob/main/LICENSE) [![Build Status](https://github.com/speechmatics/speechmatics-python-sdk/actions/workflows/test.yaml/badge.svg)](https://github.com/speechmatics/speechmatics-python-sdk/actions/workflows/test.yaml) @@ -57,6 +58,9 @@ pip install speechmatics-rt # Voice agents pip install speechmatics-voice +# Voice agents on the Agent STT service +pip install speechmatics-agent-stt + # Text-to-speech pip install speechmatics-tts ``` @@ -84,6 +88,12 @@ pip install speechmatics-tts - Speaker diarization and turn detection - Optional ML-based smart turn: `pip install speechmatics-voice[smart]` +**[speechmatics-agent-stt](./sdk/agent_stt/README.md)** - Agent STT SDK + +- Segment-level transcription for voice agents +- Server-side VAD and turn detection, or bring your own (Pipecat, LiveKit) +- Runs no models locally + **[speechmatics-tts](./sdk/tts/README.md)** - Text-to-speech - Convert text to natural-sounding speech @@ -937,6 +947,7 @@ Each SDK package includes detailed documentation: | **speechmatics-batch** | [README](./sdk/batch/README.md) • [Migration Guide](./sdk/batch/MIGRATION.md) | Async batch transcription | | **speechmatics-rt** | [README](./sdk/rt/README.md) • [Migration Guide](./sdk/rt/MIGRATION.md) | Realtime Streaming | | **speechmatics-voice** | [README](./sdk/voice/README.md) | Voice agent SDK | +| **speechmatics-agent-stt** | [README](./sdk/agent_stt/README.md) • [Plan](./sdk/agent_stt/PLAN.md) | Agent STT SDK | | **speechmatics-tts** | [README](./sdk/tts/README.md) | Text-to-speech | ### Speechmatics Academy diff --git a/examples/agent_stt/README.md b/examples/agent_stt/README.md new file mode 100644 index 00000000..a56278c9 --- /dev/null +++ b/examples/agent_stt/README.md @@ -0,0 +1,19 @@ +# Agent STT examples + +Set `SPEECHMATICS_API_KEY` first. To point at a local Voice Agent Service, set +`SPEECHMATICS_AGENT_STT_URL` (for example `ws://localhost:8000/v2/agent`). + +The service needs 16 kHz raw PCM, so the file examples take a 16 kHz WAV and default to +`tests/voice/assets/audio_01_16kHz.wav`. + +| Example | What it shows | +| --- | --- | +| [file/main.py](file/main.py) | File transcription with the service's VAD; segments, turn events, transcript at the end | +| [client_vad/main.py](client_vad/main.py) | The client owns turn boundaries and calls `finalize()`, as Pipecat and LiveKit do | +| [microphone/main.py](microphone/main.py) | Live microphone with diarization and speaker-labelled transcript (needs `pyaudio`) | + +```bash +python examples/agent_stt/file/main.py +python examples/agent_stt/client_vad/main.py +python examples/agent_stt/microphone/main.py +``` diff --git a/examples/agent_stt/client_vad/main.py b/examples/agent_stt/client_vad/main.py new file mode 100644 index 00000000..c6cb0c87 --- /dev/null +++ b/examples/agent_stt/client_vad/main.py @@ -0,0 +1,56 @@ +"""Drive turn boundaries from the client instead of the service. + +This is the mode host frameworks use: Pipecat, LiveKit and others already run a VAD, so the +service's VAD is switched off and each turn is closed by calling `finalize()`, which sends +ForceEndOfUtterance stamped with the audio position at the moment of the call. + +The fixed interval below stands in for the host framework's own end-of-speech signal. + +Run with: python examples/agent_stt/client_vad/main.py [path/to/16kHz.wav] +""" + +import asyncio +import sys +import wave + +from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import VADMode + +DEFAULT_AUDIO_FILE = "./tests/voice/assets/audio_01_16kHz.wav" +CHUNK_SIZE = 1024 +TURN_SECONDS = 5.0 + + +async def main(path: str) -> None: + config = TranscriptionConfig(language="en", enable_partials=True, vad_mode=VADMode.CLIENT) + + # Uses SPEECHMATICS_API_KEY from the environment + async with AsyncClient(config=config) as client: + + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + print(f"[final] {message['segment']['transcript']}") + + with wave.open(path, "rb") as wav: + if wav.getframerate() != 16000: + print(f"{path} is {wav.getframerate()} Hz; the Agent STT service needs 16 kHz audio") + return + + next_turn_end = TURN_SECONDS + while True: + frame = wav.readframes(CHUNK_SIZE // wav.getsampwidth()) + if not frame: + break + await client.send_audio(frame) + + if client.audio_seconds_sent >= next_turn_end: + print(f"[client vad] end of turn at {client.audio_seconds_sent:.2f}s") + await client.force_end_of_utterance() + next_turn_end += TURN_SECONDS + + print(f"\nTranscript: {client.transcript}") + + +asyncio.run(main(sys.argv[1] if len(sys.argv) > 1 else DEFAULT_AUDIO_FILE)) diff --git a/examples/agent_stt/file/main.py b/examples/agent_stt/file/main.py new file mode 100644 index 00000000..58a40a36 --- /dev/null +++ b/examples/agent_stt/file/main.py @@ -0,0 +1,59 @@ +"""Transcribe a 16 kHz WAV file with the Agent STT service. + +The service runs its own VAD here, so it reports speech and turn events and closes each +segment itself. The whole transcript is on the client when the session ends. + +Run with: python examples/agent_stt/file/main.py [path/to/16kHz.wav] +""" + +import asyncio +import sys +import wave + +from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig + +DEFAULT_AUDIO_FILE = "./tests/voice/assets/audio_01_16kHz.wav" + + +class WavSource: + """Reads raw PCM frames out of a WAV file, leaving the header behind.""" + + def __init__(self, wav: wave.Wave_read) -> None: + self._wav = wav + + def read(self, size: int) -> bytes: + return self._wav.readframes(size // self._wav.getsampwidth()) + + +async def main(path: str) -> None: + # Uses SPEECHMATICS_API_KEY from the environment + client = AsyncClient(config=TranscriptionConfig(language="en", enable_partials=True)) + + @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) + def handle_partial_segment(message): + print(f"[partial] {message['segment']['transcript']}") + + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + print(f"[final] {message['segment']['transcript']}") + + @client.on(ServerMessageType.START_OF_TURN) + def handle_start_of_turn(message): + print(f"[turn] start at {message['metadata']['start_time']}s") + + @client.on(ServerMessageType.END_OF_TURN) + def handle_end_of_turn(message): + print(f"[turn] end at {message['metadata']['end_time']}s") + + with wave.open(path, "rb") as wav: + if wav.getframerate() != 16000: + print(f"{path} is {wav.getframerate()} Hz; the Agent STT service needs 16 kHz audio") + return + await client.transcribe(WavSource(wav)) + + print(f"\nTranscript: {client.transcript}") + + +asyncio.run(main(sys.argv[1] if len(sys.argv) > 1 else DEFAULT_AUDIO_FILE)) diff --git a/examples/agent_stt/microphone/main.py b/examples/agent_stt/microphone/main.py new file mode 100644 index 00000000..2763840d --- /dev/null +++ b/examples/agent_stt/microphone/main.py @@ -0,0 +1,60 @@ +"""Transcribe the microphone with the Agent STT service. + +The service's VAD decides where turns begin and end, so this prints segments as they close +plus the speech and turn events around them. + +Run with: python examples/agent_stt/microphone/main.py +""" + +import asyncio + +from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import Microphone +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig + +SAMPLE_RATE = 16000 +CHUNK_SIZE = 1024 + + +async def main() -> None: + mic = Microphone(sample_rate=SAMPLE_RATE, chunk_size=CHUNK_SIZE) + if not mic.start(): + print("PyAudio not installed - install with: pip install pyaudio") + return + + config = TranscriptionConfig(language="en", enable_partials=True, diarization="speaker") + + # Uses SPEECHMATICS_API_KEY from the environment + async with AsyncClient(config=config) as client: + + @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) + def handle_partial_segment(message): + print(f"[partial] {message['segment']['transcript']}") + + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + speaker = message["segment"].get("speaker", "?") + print(f"[final] {speaker}: {message['segment']['transcript']}") + + @client.on(ServerMessageType.END_OF_TURN) + def handle_end_of_turn(message): + print(f"[turn] end at {message['metadata']['end_time']}s") + + print("\nMicrophone ready - speak now (Ctrl+C to stop)\n") + + try: + while True: + await client.send_audio(await mic.read(CHUNK_SIZE)) + except (asyncio.CancelledError, KeyboardInterrupt): + pass + finally: + mic.stop() + + print(f"\nTranscript: {client.transcript_text(speaker_labels=True)}") + + +try: + asyncio.run(main()) +except KeyboardInterrupt: + pass diff --git a/pyproject.toml b/pyproject.toml index 018d6c1a..bf8e6ed8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.pyright] venvPath = "." venv = ".venv" -extraPaths = ["sdk/batch", "sdk/flow", "sdk/rt", "sdk/tts", "sdk/voice"] +extraPaths = ["sdk/agent_stt", "sdk/batch", "sdk/flow", "sdk/rt", "sdk/tts", "sdk/voice"] [tool.black] line-length = 120 diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md new file mode 100644 index 00000000..589f6412 --- /dev/null +++ b/sdk/agent_stt/PLAN.md @@ -0,0 +1,135 @@ +# Agent STT SDK - plan + +## What this is + +`speechmatics-agent-stt` is a thin extension of `speechmatics-rt` that talks to the +**Voice Agent Service** (`voice-agent-service`) instead of the raw RT engine. The service sits +between the RT SaaS proxy and the transcriber and turns word-level RT output into +**segments**, plus VAD/turn signals. + +The SDK does **no VAD and no turn detection of its own**. Boundaries come from one of two places: + +| Mode | Who decides the turn boundary | Wire behaviour | +| --- | --- | --- | +| `VADMode.SERVER` | the service's own (Silero) VAD | `transcription_config.vad_config.enabled = true`; service emits `SpeechStarted`/`SpeechEnded`/`StartOfTurn`/`EndOfTurn` and forces end-of-utterance internally | +| `VADMode.CLIENT` | the host framework (Pipecat, LiveKit, ...) | `transcription_config.vad_config.enabled = false`; the host calls `client.force_end_of_utterance()` on its own VAD's stop-speaking event | + +This is the whole reason the SDK exists as a separate package: the `voice` SDK bundles VAD + +smart-turn models in-process, which duplicates what Pipecat/LiveKit already run and what the +service now does server-side. + +## Protocol delta vs the RT SDK + +Endpoint: RT URL + `/agent`, optionally + `/{profile}` +(`wss://eu2.rt.speechmatics.com/v2/agent`, service route is `/v2/agent/{profile:path}`). + +Client -> server: unchanged (`StartRecognition`, binary audio, `EndOfStream`, +`ForceEndOfUtterance`). No new client messages. + +`StartRecognition.transcription_config` gains two service-only fields, stripped by the service +before it forwards to the RT engine (`_profiles/_rt_conversion.NON_RT_API_FIELDS`): + +- `vad_config`: `{enabled, window, onset_threshold, offset_threshold}` +- `emit_sentences`: `bool` - close a segment on sentence boundaries mid-turn + +Server -> client, new messages (`voice_agent_api/_service_messages.py`): + +- `AddSegment` - `{segment: {transcript, speaker?}, metadata: {start_time, end_time}}` +- `AddPartialSegment` - same shape, interim +- `SpeechStarted` / `SpeechEnded` - `{metadata: {start_time|end_time}}` (VAD) +- `StartOfTurn` / `EndOfTurn` - `{metadata: {start_time|end_time}}` (turn detection) +- `Warning` - RT shape, also emitted by the service for profile/lock adjustments + +Server -> client, RT passthrough: `RecognitionStarted`, `AudioAdded`, `AddTranscript`, +`AddPartialTranscript`, `EndOfTranscript`, `Info`, `Warning`, `Error`, audio events. +`EndOfUtterance` is consumed by the service and never reaches the client. + +Note: the service still forwards `AddTranscript`/`AddPartialTranscript` verbatim today. The SDK +accumulates its transcript from **segments only**, but the transcript messages remain available +via handlers and the event log, so nothing is lost if a future profile mutes them. + +Constraints the service imposes: `audio_format.type` must be `raw`, sample rate `16000` +(Silero), encoding `pcm_s16le` or `pcm_f32le` (no mulaw). The SDK defaults to exactly that. + +## Milestone 1 - the SDK (done) + +``` +sdk/agent_stt/ + pyproject.toml speechmatics-agent-stt, depends on speechmatics-rt + README.md + PLAN.md this file + speechmatics/agent_stt/ + __init__.py public API + _client.py AsyncClient (subclasses rt.AsyncClient) + _models.py message enums, TranscriptionConfig, VADConfig, Segment, TimedEvent + _transcript.py Transcript - final segments, live partial, timeline, raw event log + _transport.py AgentTransport - stamps sm-sdk=python-agent-stt-sdk-vX + _url.py URL/profile resolution +tests/agent_stt/ offline unit tests (no API key needed) +examples/agent_stt/ server-VAD and client-VAD (BYO) examples +``` + +Everything inherited from `rt.AsyncClient` stays: auth (`StaticKeyAuth`/`JWTAuth`), transport, +reconnect-free lifecycle, `send_audio`, `transcribe`, `stop_session`, +`force_end_of_utterance`, the `EventEmitter` decorator API. + +What the subclass adds: + +1. URL resolution (`/agent` + profile, `SPEECHMATICS_AGENT_STT_URL` env override). +2. `TranscriptionConfig` with `vad_mode`, `vad_config`, `emit_sentences`, and `model` left + **unset** by default - the service's default profile pins `operating_point: enhanced` and + locks it, so sending `model` too would put both keys in the merged `StartRecognition`. +3. Segment-level transcript accumulation, so `client.transcript` reads like the RT flow does. +4. A raw event log (`client.events`) capturing every server message, including ones this SDK + version doesn't model yet - that is the "accept and save the other messages" requirement. +5. 16 kHz raw PCM defaults. + +Plumbing carried over from the `voice` SDK (rewritten, not imported, since that SDK goes away): + +- `connect()` / `disconnect()` and a context manager that connects on entry, the shape Pipecat + already calls +- an audio gate: frames before RecognitionStarted or after close are dropped, not raised on, and + a transport error closes the gate instead of propagating into an audio callback +- `finalize()` callable from a sync handler, stamping ForceEndOfUtterance with the audio position + **at the moment of the call** rather than when the send lands +- `app` reported as `sm-app` on the URL, alongside the SDK's own `sm-sdk` identifier +- the language pack's `word_delimiter` from RecognitionStarted driving how text is joined + +Not carried over: VAD, smart turn, the audio ring buffer, metrics/diagnostics messages, +speaker focus, fragment-level segment assembly (the service does that now). + +### Verified end to end + +Run against the real `voice-agent-service` with a stub RT transcriber behind it, in client-VAD +mode: `/v2/agent/default` routing, three `ForceEndOfUtterance` turns each flushing one +`AddSegment`, speaker labels, the event log, and `client.transcript` coming out as +`"Hello there. How are you today? Goodbye."`. + +The StartRecognition the service forwarded downstream confirmed the two design points that +mattered: `vad_config` and `emit_sentences` are stripped, and because the SDK leaves `model` +unset the message carries only the profile's locked `operating_point` - no conflicting pair. + +## Milestone 2 - Pipecat (not in this change) + +`pipecat/src/pipecat/services/speechmatics/stt.py` (~1266 lines) currently drives +`speechmatics.voice.VoiceAgentClient` and its in-process VAD/smart-turn. Nothing in Pipecat is +touched by this change. The migration: + +- swap the import block to `speechmatics.agent_stt` +- `TurnDetectionMode.EXTERNAL` -> `VADMode.CLIENT` + `force_end_of_utterance()` on + `VADUserStoppedSpeakingFrame`; `ADAPTIVE`/`SMART_TURN` -> `VADMode.SERVER` and let + `StartOfTurn`/`EndOfTurn` drive `ProposedUserStartedSpeakingFrame`/`ProposedUserStoppedSpeakingFrame` +- `AddPartialSegment` -> `InterimTranscriptionFrame`, `AddSegment` -> `TranscriptionFrame` +- drop the `pipecat-ai[speechmatics]` onnxruntime/transformers extras that only existed for the + bundled VAD and smart-turn models +- keep `SpeechmaticsSTTSettings` as the public surface so user code doesn't change + +Open questions to settle before starting milestone 2: + +- ~~speaker-focus / known-speaker features in the Pipecat service have no service-side + equivalent yet~~ - dropped for now, to be added in a later service release. `known_speakers` + still works, since `speaker_diarization_config.speakers` passes straight through. +- non-forced `EndOfUtterance` no longer flushes a segment (the default profile sets + `end_of_utterance_silence_trigger: 0.0`), so engine-silence endpointing is not available - + confirm that is intended for the `FIXED` mode Pipecat exposes. If it is, `FIXED` has to map + onto `VADMode.SERVER` (or be removed for this service) rather than onto engine silence. diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md new file mode 100644 index 00000000..b9765647 --- /dev/null +++ b/sdk/agent_stt/README.md @@ -0,0 +1,130 @@ +# Speechmatics Agent STT SDK + +Python client for the Speechmatics **Agent STT** service, built on +[`speechmatics-rt`](https://pypi.org/project/speechmatics-rt/). + +The Agent STT service works in **segments** rather than word groups, and reports the speech and +turn events a voice agent needs. This SDK runs **no VAD and no turn detection of its own** - +either the service's VAD closes turns, or your application's does. + +```bash +pip install speechmatics-agent-stt +``` + +## Quick start + +```python +import asyncio +from speechmatics.agent_stt import AsyncClient, ServerMessageType, TranscriptionConfig + +async def main(): + # Uses SPEECHMATICS_API_KEY from the environment + async with AsyncClient(config=TranscriptionConfig(language="en", enable_partials=True)) as client: + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + print(message["segment"]["transcript"]) + + while chunk := next_audio_chunk(): + await client.send_audio(chunk) + + print(client.transcript) + +asyncio.run(main()) +``` + +## Who closes the turn + +The service needs a boundary to close a segment on. Pick where it comes from: + +```python +from speechmatics.agent_stt import TranscriptionConfig, VADConfig, VADMode + +# The service's VAD (default). It emits SpeechStarted/SpeechEnded and StartOfTurn/EndOfTurn. +config = TranscriptionConfig( + vad_mode=VADMode.SERVER, + vad_config=VADConfig(window=0.2, onset_threshold=0.5, offset_threshold=0.35), +) + +# Your VAD - Pipecat, LiveKit, or your own. The service's VAD stays off. +config = TranscriptionConfig(vad_mode=VADMode.CLIENT) +``` + +With `VADMode.CLIENT`, close each turn when your VAD reports end of speech: + +```python +client.finalize() # from a sync callback +await client.force_end_of_utterance() # from async code +``` + +Either sends `ForceEndOfUtterance` stamped with the audio position at the moment of the call, so +the service cuts the turn where your VAD heard the end of speech rather than wherever the send +lands. The flushed segment comes back as a normal `AddSegment`. + +## Session output + +Every server message is dispatched to your handlers and also kept on the client: + +```python +client.transcript # final segments joined by the language's word delimiter +client.segments # list[Segment] - transcript, timing, speaker, is_final +client.partial_segment # the segment currently in flight, or None +client.timeline # list[TimedEvent] - the speech and turn events, in order +client.events # every raw message, including ones this SDK does not model +client.session_info # session id and the language pack the service reported + +client.transcript_text(speaker_labels=True, include_partial=False) +``` + +Pass `record_events=False` to `AsyncClient` for long-running sessions where the raw log is not +wanted. + +## Messages + +Emitted by the service: + +| Message | Payload | +| --- | --- | +| `AddSegment` | `segment.transcript`, optional `segment.speaker`, `metadata.start_time`, `metadata.end_time` | +| `AddPartialSegment` | interim preview of the segment being built | +| `SpeechStarted` / `SpeechEnded` | `metadata.start_time` / `metadata.end_time` (service VAD) | +| `StartOfTurn` / `EndOfTurn` | `metadata.start_time` / `metadata.end_time` (service turn detection) | + +Passed through from the RT engine: `RecognitionStarted`, `AudioAdded`, `AddTranscript`, +`AddPartialTranscript`, `EndOfTranscript`, `SpeakersResult`, `Info`, `Warning`, `Error`. +`EndOfUtterance` is consumed by the service and not forwarded. + +## Configuration + +`TranscriptionConfig` is the RT transcription config plus the service-only fields: + +| Field | Meaning | +| --- | --- | +| `vad_mode` | `VADMode.SERVER` (default) or `VADMode.CLIENT` | +| `vad_config` | `window`, `onset_threshold`, `offset_threshold` for the service's VAD | +| `emit_sentences` | Close a segment on every sentence boundary, not only at the turn boundary | + +`model` is left unset by default: the service profile pins the model for the session. + +## Endpoint + +The Agent STT endpoint is the RT endpoint plus `/agent`, optionally followed by a service +profile: + +```python +AsyncClient(url="wss://eu2.rt.speechmatics.com/v2") # -> /v2/agent +AsyncClient(url="ws://localhost:8000/v2", profile="default") # -> /v2/agent/default +AsyncClient(app="pipecat/1.0") # reported as sm-app +``` + +Resolution order: the `url` argument, `SPEECHMATICS_AGENT_STT_URL`, `SPEECHMATICS_RT_URL`, then +the EU endpoint. The `/agent` segment is appended when it is missing. + +## Audio + +The service requires **16 kHz raw PCM**, `pcm_s16le` or `pcm_f32le`, which is what the client +defaults to. Audio sent before the session is ready, or after it closes, is dropped rather than +raising, so an audio callback does not have to track session state. + +## Examples + +See [examples/agent_stt](../../examples/agent_stt). diff --git a/sdk/agent_stt/pyproject.toml b/sdk/agent_stt/pyproject.toml new file mode 100644 index 00000000..974188f2 --- /dev/null +++ b/sdk/agent_stt/pyproject.toml @@ -0,0 +1,69 @@ +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "speechmatics-agent-stt" +dynamic = ["version"] +description = "Speechmatics Agent STT Python client for voice agents" +readme = "README.md" +authors = [{ name = "Speechmatics", email = "support@speechmatics.com" }] +license = "MIT" +requires-python = ">=3.9" +dependencies = ["speechmatics-rt>=0.5.3"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Multimedia :: Sound/Audio :: Speech", + "Topic :: Software Development :: Libraries :: Python Modules", +] +keywords = [ + "speechmatics", + "speech-to-text", + "conversational-ai", + "voice", + "agents", + "real-time", + "websocket", + "pipecat", + "livekit", +] + +[project.optional-dependencies] +jwt = ["aiohttp"] +dev = [ + "black", + "ruff", + "mypy", + "pre-commit", + "pytest", + "pytest-asyncio", + "pytest-cov", + "pytest-mock", + "build", +] + +[project.urls] +homepage = "https://github.com/speechmatics/speechmatics-python-sdk" +documentation = "https://docs.speechmatics.com/" +repository = "https://github.com/speechmatics/speechmatics-python-sdk" +issues = "https://github.com/speechmatics/speechmatics-python-sdk/issues" + +[tool.setuptools.dynamic] +version = { attr = "speechmatics.agent_stt.__version__" } + +[tool.setuptools.package-data] +"speechmatics.agent_stt" = ["py.typed"] + +[tool.setuptools.packages.find] +where = ["."] + +[[tool.mypy.overrides]] +module = ["speechmatics.rt.*"] +ignore_missing_imports = true diff --git a/sdk/agent_stt/speechmatics/__init__.py b/sdk/agent_stt/speechmatics/__init__.py new file mode 100644 index 00000000..8db66d3d --- /dev/null +++ b/sdk/agent_stt/speechmatics/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/sdk/agent_stt/speechmatics/agent_stt/__init__.py b/sdk/agent_stt/speechmatics/agent_stt/__init__.py new file mode 100644 index 00000000..e1cef147 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/__init__.py @@ -0,0 +1,109 @@ +# +# Copyright (c) 2026, Speechmatics / Cantab Research Ltd +# + +"""Speechmatics Agent STT SDK. + +A client for the Speechmatics Agent STT service, built on the Speechmatics Python Real-Time +SDK. The service works in segments rather than word groups and reports speech and turn events. + +This SDK runs no VAD and no turn detection of its own: either the service's VAD closes turns, +or the application's does (Pipecat, LiveKit, ...) by calling `finalize()`. +""" + +__version__ = "0.0.0" + +from speechmatics.rt import AudioEncoding +from speechmatics.rt import AudioError +from speechmatics.rt import AudioEventsConfig +from speechmatics.rt import AudioFormat +from speechmatics.rt import AuthBase +from speechmatics.rt import AuthenticationError +from speechmatics.rt import ConfigurationError +from speechmatics.rt import ConnectionConfig +from speechmatics.rt import ConnectionError +from speechmatics.rt import ConversationConfig +from speechmatics.rt import EventEmitter +from speechmatics.rt import JWTAuth +from speechmatics.rt import Microphone +from speechmatics.rt import Model +from speechmatics.rt import SessionError +from speechmatics.rt import SpeakerDiarizationConfig +from speechmatics.rt import SpeakerIdentifier +from speechmatics.rt import StaticKeyAuth +from speechmatics.rt import TimeoutError +from speechmatics.rt import TranscriptionError +from speechmatics.rt import TranslationConfig +from speechmatics.rt import TransportError + +from ._client import AgentSTTClient +from ._client import AsyncClient +from ._models import DEFAULT_CHUNK_SIZE +from ._models import DEFAULT_SAMPLE_RATE +from ._models import DEFAULT_WORD_DELIMITER +from ._models import SEGMENT_MESSAGES +from ._models import TIMED_MESSAGES +from ._models import ClientMessageType +from ._models import LanguagePackInfo +from ._models import Segment +from ._models import ServerMessageType +from ._models import SessionInfo +from ._models import TimedEvent +from ._models import TranscriptionConfig +from ._models import VADConfig +from ._models import VADMode +from ._transcript import Transcript +from ._url import DEFAULT_AGENT_STT_URL +from ._url import resolve_url + +__all__ = [ + "DEFAULT_AGENT_STT_URL", + "DEFAULT_CHUNK_SIZE", + "DEFAULT_SAMPLE_RATE", + "DEFAULT_WORD_DELIMITER", + "SEGMENT_MESSAGES", + "TIMED_MESSAGES", + "__version__", + # Client + "AgentSTTClient", + "AsyncClient", + # Config + "AudioEncoding", + "AudioEventsConfig", + "AudioFormat", + "ConnectionConfig", + "ConversationConfig", + "SpeakerDiarizationConfig", + "SpeakerIdentifier", + "TranscriptionConfig", + "TranslationConfig", + "Model", + "VADConfig", + "VADMode", + # Auth + "AuthBase", + "JWTAuth", + "StaticKeyAuth", + # Messages + "ClientMessageType", + "ServerMessageType", + "Segment", + "TimedEvent", + # Session + "LanguagePackInfo", + "SessionInfo", + "Transcript", + "resolve_url", + # Utilities + "EventEmitter", + "Microphone", + # Exceptions + "AudioError", + "AuthenticationError", + "ConfigurationError", + "ConnectionError", + "SessionError", + "TimeoutError", + "TranscriptionError", + "TransportError", +] diff --git a/sdk/agent_stt/speechmatics/agent_stt/_client.py b/sdk/agent_stt/speechmatics/agent_stt/_client.py new file mode 100644 index 00000000..6e4f1dc5 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_client.py @@ -0,0 +1,489 @@ +from __future__ import annotations + +import asyncio +import time +import uuid +from typing import Any +from typing import BinaryIO +from typing import Optional + +from speechmatics.rt import AsyncClient as RTAsyncClient +from speechmatics.rt import AudioEncoding +from speechmatics.rt import AudioEventsConfig +from speechmatics.rt import AudioFormat +from speechmatics.rt import AuthBase +from speechmatics.rt import ConnectionConfig +from speechmatics.rt import StaticKeyAuth +from speechmatics.rt import TimeoutError as RTTimeoutError +from speechmatics.rt import TranscriptionConfig as RTTranscriptionConfig +from speechmatics.rt import TranslationConfig +from speechmatics.rt import TransportError + +from ._logging import get_logger +from ._models import DEFAULT_CHUNK_SIZE +from ._models import DEFAULT_SAMPLE_RATE +from ._models import TIMED_MESSAGES +from ._models import ClientMessageType +from ._models import LanguagePackInfo +from ._models import Segment +from ._models import ServerMessageType +from ._models import SessionInfo +from ._models import TimedEvent +from ._models import TranscriptionConfig +from ._transcript import Transcript +from ._transport import AgentTransport +from ._url import resolve_url + +_UNSET = object() + +DISCONNECT_TIMEOUT = 5.0 + + +class AsyncClient(RTAsyncClient): + """ + Asynchronous client for the Speechmatics Agent STT service. + + Extends the RT client to talk to the Agent STT endpoint (`/agent`), which works in + segments rather than word groups and reports speech and turn events. The client runs no + VAD and no turn detection of its own: either the service's VAD closes turns + (`VADMode.SERVER`) or the application's does, by calling `finalize()` (`VADMode.CLIENT`). + + Args: + auth: Authentication instance. Defaults to `StaticKeyAuth` built from `api_key` or the + `SPEECHMATICS_API_KEY` environment variable. + api_key: Speechmatics API key, used when `auth` is not given. + url: WebSocket endpoint. Defaults to `SPEECHMATICS_AGENT_STT_URL`, then + `SPEECHMATICS_RT_URL`, then the EU endpoint. An `/agent` segment is appended if absent. + profile: Service profile, appended to the endpoint path. + app: Application name reported to the service as `sm-app`. + config: Transcription config for the session, normally an + `agent_stt.TranscriptionConfig`. + audio_format: Audio format. Defaults to 16 kHz signed 16-bit PCM, which is what the + service requires. + conn_config: WebSocket connection configuration. + record_events: Whether to keep every raw server message in `events`. + + Examples: + Service VAD, transcript at the end: + >>> async with AsyncClient(api_key="your-key") as client: + ... @client.on(ServerMessageType.ADD_SEGMENT) + ... def handle_segment(message): + ... print(message["segment"]["transcript"]) + ... await client.send_audio(frame) + >>> print(client.transcript) + + Client VAD (Pipecat, LiveKit): + >>> config = TranscriptionConfig(vad_mode=VADMode.CLIENT) + >>> client = AsyncClient(api_key="your-key", config=config) + >>> await client.connect() + >>> await client.send_audio(frame) + >>> client.finalize() # on the application's own end-of-speech signal + """ + + def __init__( + self, + auth: Optional[AuthBase] = None, + *, + api_key: Optional[str] = None, + url: Optional[str] = None, + profile: Optional[str] = None, + app: Optional[str] = None, + config: Optional[RTTranscriptionConfig] = None, + audio_format: Optional[AudioFormat] = None, + conn_config: Optional[ConnectionConfig] = None, + record_events: bool = True, + ) -> None: + super().__init__( + auth, + api_key=api_key, + url=resolve_url(url, profile=profile, app=app), + conn_config=conn_config, + ) + + self._logger = get_logger("speechmatics.agent_stt.client") + + self._config: RTTranscriptionConfig = config or TranscriptionConfig() + self._audio_format = audio_format or AudioFormat( + encoding=AudioEncoding.PCM_S16LE, + sample_rate=DEFAULT_SAMPLE_RATE, + chunk_size=DEFAULT_CHUNK_SIZE, + ) + + self._session_info = SessionInfo(request_id=self._session.request_id) + self._transcript = Transcript(record_events=record_events) + + self._is_connected = False + self._is_ready_for_audio = False + self._finalize_sent_at: Optional[float] = None + self._last_finalize_latency = 0.0 + + self._register_handlers() + + @classmethod + def _create_transport_from_config( + cls, + auth: Optional[AuthBase] = None, + *, + api_key: Optional[str] = None, + url: Optional[str] = None, + conn_config: Optional[ConnectionConfig] = None, + request_id: Optional[str] = None, + ) -> AgentTransport: + """Build the Agent STT transport, so the service sees this SDK's identifier.""" + return AgentTransport( + url or resolve_url(), + conn_config or ConnectionConfig(), + auth or StaticKeyAuth(api_key), + request_id or str(uuid.uuid4()), + ) + + def _register_handlers(self) -> None: + """Track session state and accumulate segments, leaving all messages for the application.""" + self.on(ServerMessageType.RECOGNITION_STARTED, self._on_session_started) + self.on(ServerMessageType.ADD_SEGMENT, self._on_segment) + self.on(ServerMessageType.ADD_PARTIAL_SEGMENT, self._on_segment) + for message_type in TIMED_MESSAGES: + self.on(message_type, self._on_timed_event) + + # ========================================================================== + # Session lifecycle + # ========================================================================== + + async def connect(self, ws_headers: Optional[dict] = None) -> None: + """ + Open the session and wait until the service is ready for audio. + + Audio sent before this returns is dropped, so callers do not have to sequence the + handshake themselves. + + Args: + ws_headers: Additional WebSocket handshake headers. + + Raises: + ConnectionError: If the WebSocket connection fails. + TimeoutError: If the service does not accept the session in time. + + Examples: + >>> client = AsyncClient(api_key="your-key") + >>> await client.connect() + """ + if self._is_connected: + return + + await self.start_session( + transcription_config=self._config, + audio_format=self._audio_format, + ws_headers=ws_headers, + ) + self._is_connected = True + + async def disconnect(self) -> None: + """ + Close the session, flushing whatever the service still holds. + + Examples: + >>> await client.disconnect() + >>> print(client.transcript) + """ + if not self._is_connected: + await self.close() + return + + self._is_ready_for_audio = False + try: + await asyncio.wait_for(self.stop_session(), timeout=DISCONNECT_TIMEOUT) + except Exception as e: + self._logger.warning("Error closing session: %s", e) + await self.close() + finally: + self._is_connected = False + + async def __aenter__(self) -> AsyncClient: + """Open the session on entry.""" + await self.connect() + return self + + async def __aexit__(self, *args: Any) -> None: + """Close the session on exit.""" + await self.disconnect() + + async def start_session( + self, + *, + transcription_config: Optional[RTTranscriptionConfig] = None, + audio_format: Optional[AudioFormat] = None, + translation_config: Optional[TranslationConfig] = None, + audio_events_config: Optional[AudioEventsConfig] = None, + ws_headers: Optional[dict] = None, + ) -> None: + """ + Start the session, defaulting to the config this client was built with. + + Args: + transcription_config: Transcription config for the session. + audio_format: Audio format. Must be 16 kHz raw PCM for the Agent STT service. + translation_config: Optional translation config. + audio_events_config: Optional audio event detection config. + ws_headers: Additional WebSocket handshake headers. + + Raises: + ConnectionError: If the WebSocket connection fails. + TimeoutError: If the service does not accept the session in time. + """ + await super().start_session( + transcription_config=transcription_config or self._config, + audio_format=audio_format or self._audio_format, + translation_config=translation_config, + audio_events_config=audio_events_config, + ws_headers=ws_headers, + ) + + # ========================================================================== + # Audio and turn control + # ========================================================================== + + async def send_audio(self, payload: bytes) -> None: + """ + Send an audio frame. + + Frames sent before the session is ready, or after it has closed, are dropped rather + than raising, so an audio callback does not have to track session state. + + Args: + payload: Raw audio bytes in the session's audio format. + + Examples: + >>> await client.send_audio(frame) + """ + if not self._is_ready_for_audio: + return + + try: + await super().send_audio(payload) + except TransportError as e: + self._logger.warning("Error sending audio: %s", e) + self._is_ready_for_audio = False + + def finalize(self, *, timestamp: Optional[float] | object = _UNSET) -> None: + """ + Close the current turn now, from a synchronous context. + + Sends ForceEndOfUtterance stamped with the audio timestamp at the moment of the call, + so the service closes the turn where the application's VAD detected the end of speech + rather than wherever the send happens to land. The flushed segment arrives as a normal + AddSegment message. + + Use this when the application brings its own VAD (`VADMode.CLIENT`); with + `VADMode.SERVER` the service closes turns itself. + + Args: + timestamp: Audio timestamp in seconds for the end of the utterance. Defaults to + the amount of audio sent so far. Pass None to send no timestamp. + + Examples: + >>> client.finalize() + """ + resolved = self.audio_seconds_sent if timestamp is _UNSET else timestamp + try: + loop = asyncio.get_running_loop() + except RuntimeError: + self._logger.warning("finalize() needs a running event loop; use force_end_of_utterance()") + return + loop.create_task(self._send_force_end_of_utterance(resolved)) + + async def force_end_of_utterance(self, *, timestamp: Optional[float] | object = _UNSET) -> None: + """ + Close the current turn now, awaiting the send. + + The awaitable form of `finalize()`. + + Args: + timestamp: Audio timestamp in seconds for the end of the utterance. Defaults to + the amount of audio sent so far. Pass None to send no timestamp. + + Examples: + >>> await client.force_end_of_utterance() + """ + resolved = self.audio_seconds_sent if timestamp is _UNSET else timestamp + await self._send_force_end_of_utterance(resolved) + + async def _send_force_end_of_utterance(self, timestamp: Optional[float] | object) -> None: + """Send ForceEndOfUtterance, including the timestamp unless it is None.""" + message: dict[str, Any] = {"message": ClientMessageType.FORCE_END_OF_UTTERANCE} + if timestamp is not None: + message["timestamp"] = timestamp + + try: + await self.send_message(message) + except TransportError as e: + self._logger.warning("Error sending %s: %s", ClientMessageType.FORCE_END_OF_UTTERANCE, e) + return + self._finalize_sent_at = time.perf_counter() + + async def transcribe( + self, + source: BinaryIO, + *, + transcription_config: Optional[RTTranscriptionConfig] = None, + audio_format: Optional[AudioFormat] = None, + translation_config: Optional[TranslationConfig] = None, + audio_events_config: Optional[AudioEventsConfig] = None, + ws_headers: Optional[dict] = None, + timeout: Optional[float] = None, + ) -> None: + """ + Stream an audio source to the end, then close the session. + + Args: + source: Audio source with a `read()` method, holding raw PCM in the session's + audio format. + transcription_config: Transcription config for the session. + audio_format: Audio format. Must be 16 kHz raw PCM for the Agent STT service. + translation_config: Optional translation config. + audio_events_config: Optional audio event detection config. + ws_headers: Additional WebSocket handshake headers. + timeout: Maximum time in seconds to wait for the stream to finish. + + Raises: + TimeoutError: If streaming exceeds the timeout. + TranscriptionError: If the service reports an error. + + Examples: + >>> with open("speech.raw", "rb") as audio: + ... await client.transcribe(audio) + >>> print(client.transcript) + """ + if transcription_config is not None: + self._config = transcription_config + if audio_format is not None: + self._audio_format = audio_format + + if not self._is_connected: + await self.start_session( + transcription_config=self._config, + audio_format=self._audio_format, + translation_config=translation_config, + audio_events_config=audio_events_config, + ws_headers=ws_headers, + ) + self._is_connected = True + + try: + await asyncio.wait_for( + self._audio_producer(source, self._audio_format.chunk_size), + timeout=timeout, + ) + except asyncio.TimeoutError as exc: + raise RTTimeoutError("Agent STT session timed out") from exc + finally: + self._is_connected = False + self._is_ready_for_audio = False + + # ========================================================================== + # Session output + # ========================================================================== + + @property + def transcript(self) -> str: + """The final segments received so far, joined by the language's word delimiter.""" + return self._transcript.text() + + @property + def segments(self) -> list[Segment]: + """The final segments received so far, in order.""" + return self._transcript.segments + + @property + def partial_segment(self) -> Optional[Segment]: + """The segment currently being built, or None when there is nothing in flight.""" + return self._transcript.partial + + @property + def timeline(self) -> list[TimedEvent]: + """The speech and turn events received so far, in order.""" + return self._transcript.timeline + + @property + def events(self) -> list[dict[str, Any]]: + """Every raw server message received, in order, including unmodelled ones.""" + return self._transcript.events + + @property + def session_info(self) -> SessionInfo: + """Session identity and the language pack reported by the service.""" + return self._session_info + + @property + def is_connected(self) -> bool: + """Whether the session is open.""" + return self._is_connected + + @property + def is_ready_for_audio(self) -> bool: + """Whether the service has accepted the session, so audio will be sent.""" + return self._is_ready_for_audio + + @property + def last_finalize_latency(self) -> float: + """Seconds between the last `finalize()` and the segment it flushed.""" + return self._last_finalize_latency + + def transcript_text(self, *, include_partial: bool = False, speaker_labels: bool = False) -> str: + """ + Render the transcript. + + Args: + include_partial: Append the segment currently in flight. + speaker_labels: Prefix each segment with its speaker label, where one is attributed. + + Returns: + The transcript as text. + + Examples: + >>> print(client.transcript_text(speaker_labels=True)) + """ + return self._transcript.text(include_partial=include_partial, speaker_labels=speaker_labels) + + def reset_transcript(self) -> None: + """Drop the accumulated segments, timeline and event log.""" + self._transcript.reset() + + # ========================================================================== + # Message handling + # ========================================================================== + + def emit(self, event: Any, message: dict[str, Any]) -> None: + """Record every server message before dispatching it to the application's handlers.""" + self._transcript.record(message) + super().emit(event, message) + + def _on_session_started(self, message: dict[str, Any]) -> None: + """Capture the session id and language pack, and open the audio gate.""" + self._session_info.session_id = message.get("id") + self._session_info.language_pack_info = LanguagePackInfo.from_dict(message.get("language_pack_info") or {}) + self._transcript.set_delimiter(self._session_info.language_pack_info.word_delimiter) + self._is_ready_for_audio = True + + def _on_segment(self, message: dict[str, Any]) -> None: + """Accumulate a final segment, or refresh the live partial.""" + if message.get("message") != ServerMessageType.ADD_SEGMENT: + self._transcript.add_partial_segment(message) + return + + self._transcript.add_segment(message) + if self._finalize_sent_at is not None: + self._last_finalize_latency = time.perf_counter() - self._finalize_sent_at + self._finalize_sent_at = None + + def _on_timed_event(self, message: dict[str, Any]) -> None: + """Append a speech or turn event to the timeline.""" + self._transcript.add_timed_event(message) + + async def close(self) -> None: + """Close the connection without waiting for outstanding messages.""" + self._is_ready_for_audio = False + self._is_connected = False + await super().close() + + +AgentSTTClient = AsyncClient diff --git a/sdk/agent_stt/speechmatics/agent_stt/_logging.py b/sdk/agent_stt/speechmatics/agent_stt/_logging.py new file mode 100644 index 00000000..38178dbc --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_logging.py @@ -0,0 +1,23 @@ +import logging + + +def get_logger(name: str) -> logging.Logger: + """ + Get a logger that stays silent unless the application configures logging. + + Args: + name: Logger name, typically __name__ from the calling module. + + Returns: + Logger with a NullHandler attached. + + Examples: + >>> import logging + >>> logging.getLogger("speechmatics.agent_stt").setLevel(logging.DEBUG) + """ + logger = logging.getLogger(name) + logger.addHandler(logging.NullHandler()) + return logger + + +__all__ = ["get_logger"] diff --git a/sdk/agent_stt/speechmatics/agent_stt/_models.py b/sdk/agent_stt/speechmatics/agent_stt/_models.py new file mode 100644 index 00000000..c6ee58f3 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_models.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import field +from enum import Enum +from typing import Any +from typing import Optional +from warnings import warn + +from speechmatics.rt import Model +from speechmatics.rt import TranscriptionConfig as RTTranscriptionConfig + +DEFAULT_SAMPLE_RATE = 16000 +DEFAULT_CHUNK_SIZE = 1024 +DEFAULT_WORD_DELIMITER = " " + + +class ClientMessageType(str, Enum): + """ + Message types that can be sent from client to the Agent STT service. + + The Agent STT service adds no client messages of its own; these are the RT messages + it forwards downstream. + + Attributes: + START_RECOGNITION: Starts the session, carrying the audio and transcription config. + END_OF_STREAM: Signals that no more audio will be sent. + FORCE_END_OF_UTTERANCE: Closes the current turn immediately, flushing the buffered + segment. Sent by clients that run their own VAD. + SET_RECOGNITION_CONFIG: Updates the transcription config mid-session. + GET_SPEAKERS: Requests the session's speaker data. + """ + + START_RECOGNITION = "StartRecognition" + END_OF_STREAM = "EndOfStream" + FORCE_END_OF_UTTERANCE = "ForceEndOfUtterance" + SET_RECOGNITION_CONFIG = "SetRecognitionConfig" + GET_SPEAKERS = "GetSpeakers" + + +class ServerMessageType(str, Enum): + """ + Message types that can be received from the Agent STT service. + + The service emits its own segment, speech and turn messages, and passes the RT engine's + messages through unchanged. + + Attributes: + RECOGNITION_STARTED: Session accepted; carries the session id and language pack info. + AUDIO_ADDED: Audio frame acknowledged. + ADD_SEGMENT: A finalized segment, closed at a turn or content boundary. + ADD_PARTIAL_SEGMENT: Interim preview of the segment being built. + SPEECH_STARTED: The service's VAD detected speech onset. + SPEECH_ENDED: The service's VAD detected speech offset. + START_OF_TURN: The service's turn detection opened a turn. + END_OF_TURN: The service's turn detection closed a turn. + ADD_TRANSCRIPT: Word-level final transcript, passed through from the RT engine. + ADD_PARTIAL_TRANSCRIPT: Word-level partial transcript, passed through from the RT engine. + END_OF_UTTERANCE: Consumed by the service for segmentation and not forwarded; listed + so handlers stay valid against a direct RT endpoint. + END_OF_TRANSCRIPT: The service has finished sending messages. + SPEAKERS_RESULT: Response to GetSpeakers. + AUDIO_EVENT_STARTED: Start of a detected audio event. + AUDIO_EVENT_ENDED: End of a detected audio event. + INFO: Informational message. + WARNING: Warning; the session continues, possibly with adjusted config. + ERROR: Error; the session is over. + + Examples: + >>> @client.on(ServerMessageType.ADD_SEGMENT) + >>> def handle_segment(message): + ... print(message["segment"]["transcript"]) + """ + + RECOGNITION_STARTED = "RecognitionStarted" + AUDIO_ADDED = "AudioAdded" + ADD_SEGMENT = "AddSegment" + ADD_PARTIAL_SEGMENT = "AddPartialSegment" + SPEECH_STARTED = "SpeechStarted" + SPEECH_ENDED = "SpeechEnded" + START_OF_TURN = "StartOfTurn" + END_OF_TURN = "EndOfTurn" + ADD_TRANSCRIPT = "AddTranscript" + ADD_PARTIAL_TRANSCRIPT = "AddPartialTranscript" + END_OF_UTTERANCE = "EndOfUtterance" + END_OF_TRANSCRIPT = "EndOfTranscript" + SPEAKERS_RESULT = "SpeakersResult" + AUDIO_EVENT_STARTED = "AudioEventStarted" + AUDIO_EVENT_ENDED = "AudioEventEnded" + INFO = "Info" + WARNING = "Warning" + ERROR = "Error" + + +SEGMENT_MESSAGES = (ServerMessageType.ADD_SEGMENT, ServerMessageType.ADD_PARTIAL_SEGMENT) + +TIMED_MESSAGES = ( + ServerMessageType.SPEECH_STARTED, + ServerMessageType.SPEECH_ENDED, + ServerMessageType.START_OF_TURN, + ServerMessageType.END_OF_TURN, +) + + +class VADMode(str, Enum): + """ + Where turn boundaries come from. This SDK never runs a VAD itself. + + Attributes: + SERVER: The service runs its own VAD and turn detection, emitting SpeechStarted, + SpeechEnded, StartOfTurn and EndOfTurn, and closing segments itself. + CLIENT: The client runs its own VAD (Pipecat, LiveKit, ...) and closes each turn by + calling `finalize()`, which sends ForceEndOfUtterance with an audio timestamp. + """ + + SERVER = "server" + CLIENT = "client" + + +@dataclass +class VADConfig: + """ + Tuning for the service's VAD. Only applied when `VADMode.SERVER` is in use. + + Attributes: + window: Silence in seconds before the service closes the turn. + onset_threshold: Speech probability above which speech starts. + offset_threshold: Speech probability below which speech ends. + """ + + window: Optional[float] = None + onset_threshold: Optional[float] = None + offset_threshold: Optional[float] = None + + +@dataclass +class TranscriptionConfig(RTTranscriptionConfig): + """ + Transcription config for the Agent STT service. + + Extends the RT transcription config with the service-only fields (`vad_config`, + `emit_sentences`). See `speechmatics.rt.TranscriptionConfig` for the inherited fields. + + Attributes: + vad_mode: Whether the service or the client decides turn boundaries. + vad_config: Tuning for the service's VAD, used when `vad_mode` is `SERVER`. + emit_sentences: Close a segment on every sentence boundary, not just at the turn + boundary. + model: Left unset by default. The service's profile pins the model for the session, + and sending it alongside the profile's `operating_point` would put both keys in + the resolved StartRecognition. + + Examples: + Service VAD, sentence-level segments: + >>> config = TranscriptionConfig(language="en", emit_sentences=True) + + Client VAD (Pipecat, LiveKit): + >>> config = TranscriptionConfig(language="en", vad_mode=VADMode.CLIENT) + """ + + model: Optional[Model] = None + vad_mode: VADMode = VADMode.SERVER + vad_config: VADConfig = field(default_factory=VADConfig) + emit_sentences: Optional[bool] = None + + def __post_init__(self) -> None: + if self.model is not None and self.operating_point is not None: + raise ValueError("Cannot specify both 'model' and 'operating_point'. Use 'model' instead.") + if self.operating_point is not None: + warn("'operating_point' is deprecated, use 'model' instead.", DeprecationWarning, stacklevel=2) + + def to_dict(self) -> dict[str, Any]: + """ + Convert to the wire form of `StartRecognition.transcription_config`. + + Returns: + The config as a dict, excluding None values, with `vad_config.enabled` derived + from `vad_mode` and `vad_mode` itself dropped. + """ + result = super().to_dict() + result.pop("vad_mode", None) + vad_config = result.pop("vad_config", None) or {} + vad_config["enabled"] = self.vad_mode is VADMode.SERVER + result["vad_config"] = vad_config + return result + + +@dataclass +class Segment: + """ + A segment of transcript, the unit the Agent STT service works in. + + Attributes: + transcript: The rendered segment text. + start_time: Segment start in seconds from the start of the session. + end_time: Segment end in seconds from the start of the session. + speaker: Speaker label (e.g. "S1"), when diarization attributed one. + is_final: True for AddSegment, False for AddPartialSegment. + """ + + transcript: str + start_time: float = 0.0 + end_time: float = 0.0 + speaker: Optional[str] = None + is_final: bool = True + + @classmethod + def from_message(cls, message: dict[str, Any]) -> Segment: + """Create a Segment from an AddSegment or AddPartialSegment message.""" + segment = message.get("segment") or {} + metadata = message.get("metadata") or {} + return cls( + transcript=segment.get("transcript", ""), + start_time=metadata.get("start_time", 0.0), + end_time=metadata.get("end_time", 0.0), + speaker=segment.get("speaker"), + is_final=message.get("message") == ServerMessageType.ADD_SEGMENT, + ) + + +@dataclass +class TimedEvent: + """ + A speech or turn event from the service, reduced to its single timestamp. + + Attributes: + message: The message type (SpeechStarted, SpeechEnded, StartOfTurn, EndOfTurn). + time: The event time in seconds from the start of the session. + """ + + message: str + time: float = 0.0 + + @classmethod + def from_message(cls, message: dict[str, Any]) -> TimedEvent: + """Create a TimedEvent from a speech or turn message.""" + metadata = message.get("metadata") or {} + time = metadata.get("start_time", metadata.get("end_time", 0.0)) + return cls(message=message.get("message", ""), time=time) + + +@dataclass +class LanguagePackInfo: + """ + Language pack details reported in RecognitionStarted. + + Attributes: + language_description: Human-readable language name. + word_delimiter: The separator between words for this language, used when joining + segments into a transcript. + writing_direction: "ltr" or "rtl". + itn: Whether inverse text normalization is applied. + adapted: Whether the language pack is adapted. + """ + + language_description: str = "" + word_delimiter: str = DEFAULT_WORD_DELIMITER + writing_direction: str = "ltr" + itn: bool = True + adapted: bool = False + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> LanguagePackInfo: + """Create LanguagePackInfo from the RecognitionStarted language_pack_info block.""" + return cls( + language_description=data.get("language_description", ""), + word_delimiter=data.get("word_delimiter", DEFAULT_WORD_DELIMITER), + writing_direction=data.get("writing_direction", "ltr"), + itn=data.get("itn", True), + adapted=data.get("adapted", False), + ) + + +@dataclass +class SessionInfo: + """ + State of the current Agent STT session. + + Attributes: + request_id: Client-generated id for this session. + session_id: Service-assigned session id, set once RecognitionStarted arrives. + language_pack_info: Language pack reported in RecognitionStarted. + """ + + request_id: str + session_id: Optional[str] = None + language_pack_info: LanguagePackInfo = field(default_factory=LanguagePackInfo) diff --git a/sdk/agent_stt/speechmatics/agent_stt/_transcript.py b/sdk/agent_stt/speechmatics/agent_stt/_transcript.py new file mode 100644 index 00000000..d869c715 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_transcript.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from typing import Any +from typing import Optional + +from ._models import DEFAULT_WORD_DELIMITER +from ._models import Segment +from ._models import TimedEvent + + +class Transcript: + """ + Accumulates a session's output: final segments, the live partial, speech/turn events + and the raw message log. + + A session's transcript is the concatenation of its final segments, so the text reads the + same way an RT session's does, one segment at a time instead of one word group at a time. + + Args: + record_events: Whether to keep every raw server message in `events`. Messages this SDK + version does not model are still captured there. + delimiter: Separator used between segments. Replaced by the language pack's word + delimiter once RecognitionStarted arrives. + + Examples: + >>> transcript = Transcript() + >>> transcript.add_segment({"message": "AddSegment", + ... "segment": {"transcript": "Hello world"}, + ... "metadata": {"start_time": 0.0, "end_time": 1.0}}) + >>> transcript.text() + 'Hello world' + """ + + def __init__(self, *, record_events: bool = True, delimiter: str = DEFAULT_WORD_DELIMITER) -> None: + self._record_events = record_events + self._delimiter = delimiter + self._segments: list[Segment] = [] + self._partial: Optional[Segment] = None + self._events: list[dict[str, Any]] = [] + self._timeline: list[TimedEvent] = [] + + @property + def segments(self) -> list[Segment]: + """The final segments received so far, in order.""" + return list(self._segments) + + @property + def partial(self) -> Optional[Segment]: + """The segment currently being built, or None when there is nothing in flight.""" + return self._partial + + @property + def events(self) -> list[dict[str, Any]]: + """Every raw server message received, in order, when recording is enabled.""" + return list(self._events) + + @property + def timeline(self) -> list[TimedEvent]: + """The speech and turn events received so far, in order.""" + return list(self._timeline) + + @property + def delimiter(self) -> str: + """The separator used between segments.""" + return self._delimiter + + def set_delimiter(self, delimiter: str) -> None: + """Set the separator between segments, from the language pack's word delimiter.""" + self._delimiter = delimiter + + def record(self, message: dict[str, Any]) -> None: + """Append a raw server message to the event log.""" + if self._record_events: + self._events.append(message) + + def add_segment(self, message: dict[str, Any]) -> Segment: + """Add a final segment from an AddSegment message and clear the live partial.""" + segment = Segment.from_message(message) + self._segments.append(segment) + self._partial = None + return segment + + def add_partial_segment(self, message: dict[str, Any]) -> Segment: + """Replace the live partial with the one in an AddPartialSegment message.""" + self._partial = Segment.from_message(message) + return self._partial + + def add_timed_event(self, message: dict[str, Any]) -> TimedEvent: + """Add a speech or turn event to the timeline.""" + event = TimedEvent.from_message(message) + self._timeline.append(event) + return event + + def text(self, *, include_partial: bool = False, speaker_labels: bool = False) -> str: + """ + Render the transcript. + + Args: + include_partial: Append the segment currently in flight. + speaker_labels: Prefix each segment with its speaker label, where one is attributed. + + Returns: + The final segments joined by the session's delimiter. + """ + segments = list(self._segments) + if include_partial and self._partial is not None: + segments.append(self._partial) + + parts = [] + for segment in segments: + if not segment.transcript: + continue + if speaker_labels and segment.speaker: + parts.append(f"{segment.speaker}: {segment.transcript}") + else: + parts.append(segment.transcript) + return self._delimiter.join(parts) + + def reset(self) -> None: + """Drop all accumulated state, keeping the delimiter.""" + self._segments.clear() + self._partial = None + self._events.clear() + self._timeline.clear() + + def __str__(self) -> str: + return self.text() diff --git a/sdk/agent_stt/speechmatics/agent_stt/_transport.py b/sdk/agent_stt/speechmatics/agent_stt/_transport.py new file mode 100644 index 00000000..34bad960 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_transport.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from urllib.parse import parse_qsl +from urllib.parse import urlencode +from urllib.parse import urlparse +from urllib.parse import urlunparse + +from speechmatics.rt._transport import Transport + +from ._version import get_version + + +class AgentTransport(Transport): + """ + RT transport that identifies itself as the Agent STT SDK. + + Only the SDK identifier on the connection URL differs; the WebSocket handling, + authentication and message framing are the RT transport's. + """ + + def _prepare_url(self) -> str: + """Return the connection URL with the Agent STT SDK version as `sm-sdk`.""" + parsed = urlparse(self._url) + params = dict(parse_qsl(parsed.query, keep_blank_values=True)) + params["sm-sdk"] = f"python-agent-stt-sdk-v{get_version()}" + return urlunparse(parsed._replace(query=urlencode(params))) diff --git a/sdk/agent_stt/speechmatics/agent_stt/_url.py b/sdk/agent_stt/speechmatics/agent_stt/_url.py new file mode 100644 index 00000000..af6550d7 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_url.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import os +from typing import Optional +from urllib.parse import parse_qs +from urllib.parse import urlencode +from urllib.parse import urlparse +from urllib.parse import urlunparse + +from ._version import get_version + +DEFAULT_AGENT_STT_URL = "wss://eu2.rt.speechmatics.com/v2/agent" +AGENT_PATH_SEGMENT = "agent" + + +def resolve_url( + url: Optional[str] = None, + profile: Optional[str] = None, + app: Optional[str] = None, +) -> str: + """ + Resolve the Agent STT WebSocket URL. + + The Agent STT endpoint is the RT endpoint plus an `/agent` path segment, optionally + followed by a service profile name. + + Args: + url: Explicit endpoint. Falls back to the `SPEECHMATICS_AGENT_STT_URL` environment + variable, then `SPEECHMATICS_RT_URL`, then the EU endpoint. + profile: Optional service profile, appended as a final path segment. + app: Optional application name reported to the service as `sm-app`. + + Returns: + The complete WebSocket URL. + + Examples: + >>> resolve_url() + 'wss://eu2.rt.speechmatics.com/v2/agent?sm-app=agent-stt-sdk%2F0.0.0' + >>> resolve_url("wss://host/v2", profile="default", app="pipecat/1.0") + 'wss://host/v2/agent/default?sm-app=pipecat%2F1.0' + """ + base = url or os.getenv("SPEECHMATICS_AGENT_STT_URL") or os.getenv("SPEECHMATICS_RT_URL") or DEFAULT_AGENT_STT_URL + parsed = urlparse(base) + return urlunparse( + parsed._replace( + path=_resolve_path(parsed.path, profile), + query=_resolve_query(parsed.query, app), + ) + ) + + +def _resolve_path(path: str, profile: Optional[str]) -> str: + """Append the /agent segment and the profile to the path, skipping any already present.""" + segments = [segment for segment in path.split("/") if segment] + + if AGENT_PATH_SEGMENT not in segments: + segments.append(AGENT_PATH_SEGMENT) + + if profile: + normalized = profile.strip("/") + if normalized and segments[-1] != normalized: + segments.append(normalized) + + return "/" + "/".join(segments) + + +def _resolve_query(query: str, app: Optional[str]) -> str: + """Set the sm-app parameter, keeping any parameters already on the URL.""" + params = parse_qs(query, keep_blank_values=True) + existing_app = params.get("sm-app", [None])[0] + params["sm-app"] = [app or existing_app or f"agent-stt-sdk/{get_version()}"] + return urlencode(params, doseq=True) diff --git a/sdk/agent_stt/speechmatics/agent_stt/_version.py b/sdk/agent_stt/speechmatics/agent_stt/_version.py new file mode 100644 index 00000000..78876859 --- /dev/null +++ b/sdk/agent_stt/speechmatics/agent_stt/_version.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import importlib.metadata + + +def get_version() -> str: + """ + Get the installed version of the speechmatics-agent-stt package. + + Returns: + The package version, or "0.0.0" when it cannot be determined. + """ + try: + return importlib.metadata.version("speechmatics-agent-stt") + except importlib.metadata.PackageNotFoundError: + try: + from . import __version__ + + return __version__ + except ImportError: + return "0.0.0" diff --git a/sdk/agent_stt/speechmatics/agent_stt/py.typed b/sdk/agent_stt/speechmatics/agent_stt/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/tests/agent_stt/test_client.py b/tests/agent_stt/test_client.py new file mode 100644 index 00000000..73967f0c --- /dev/null +++ b/tests/agent_stt/test_client.py @@ -0,0 +1,271 @@ +import json + +import pytest + +from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import AudioEncoding +from speechmatics.agent_stt import ClientMessageType +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import VADMode + +API_KEY = "test-key" + + +class StubTransport: + """Captures what the client sends instead of opening a WebSocket.""" + + def __init__(self): + self.sent = [] + self.closed = False + + async def send_message(self, payload): + self.sent.append(payload) + + async def close(self): + self.closed = True + + @property + def messages(self): + return [json.loads(payload) for payload in self.sent if isinstance(payload, str)] + + @property + def audio(self): + return [payload for payload in self.sent if isinstance(payload, bytes)] + + +@pytest.fixture +def client(monkeypatch): + monkeypatch.delenv("SPEECHMATICS_AGENT_STT_URL", raising=False) + monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) + return AsyncClient(api_key=API_KEY) + + +def recognition_started(word_delimiter=" "): + return { + "message": ServerMessageType.RECOGNITION_STARTED, + "id": "session-1", + "language_pack_info": {"language_description": "English", "word_delimiter": word_delimiter}, + } + + +def segment_message(transcript, is_final=True, speaker=None, start=0.0, end=1.0): + segment = {"transcript": transcript} + if speaker is not None: + segment["speaker"] = speaker + return { + "message": ServerMessageType.ADD_SEGMENT if is_final else ServerMessageType.ADD_PARTIAL_SEGMENT, + "segment": segment, + "metadata": {"start_time": start, "end_time": end}, + } + + +def start_session(client): + """Put the client in the state it reaches after RecognitionStarted.""" + transport = StubTransport() + client._transport = transport + client.emit(ServerMessageType.RECOGNITION_STARTED, recognition_started()) + return transport + + +@pytest.mark.asyncio +async def test_endpoint_is_the_agent_path(client): + assert client._transport._url.startswith("wss://eu2.rt.speechmatics.com/v2/agent") + + +@pytest.mark.asyncio +async def test_profile_and_app_reach_the_url(monkeypatch): + monkeypatch.delenv("SPEECHMATICS_AGENT_STT_URL", raising=False) + monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) + client = AsyncClient(api_key=API_KEY, profile="default", app="pipecat/1.0") + assert "/v2/agent/default" in client._transport._url + assert "sm-app=pipecat%2F1.0" in client._transport._url + + +@pytest.mark.asyncio +async def test_sdk_identifier(client): + assert "sm-sdk=python-agent-stt-sdk-v" in client._transport._prepare_url() + + +@pytest.mark.asyncio +async def test_audio_format_defaults_to_16k_pcm(client): + assert client._audio_format.encoding == AudioEncoding.PCM_S16LE + assert client._audio_format.sample_rate == 16000 + + +@pytest.mark.asyncio +async def test_recognition_started_sets_session_state(client): + start_session(client) + assert client.is_ready_for_audio + assert client.session_info.session_id == "session-1" + assert client.session_info.language_pack_info.language_description == "English" + + +@pytest.mark.asyncio +async def test_transcript_accumulates_from_segments(client): + start_session(client) + client.emit(ServerMessageType.ADD_PARTIAL_SEGMENT, segment_message("Hello th", is_final=False)) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hello there.")) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("How are you?")) + + assert client.transcript == "Hello there. How are you?" + assert client.partial_segment is None + assert [segment.transcript for segment in client.segments] == ["Hello there.", "How are you?"] + + +@pytest.mark.asyncio +async def test_transcript_uses_language_pack_delimiter(client): + client._transport = StubTransport() + client.emit(ServerMessageType.RECOGNITION_STARTED, recognition_started(word_delimiter="")) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("你好")) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("世界")) + assert client.transcript == "你好世界" + + +@pytest.mark.asyncio +async def test_transcript_with_speaker_labels(client): + start_session(client) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hello.", speaker="S1")) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hi.", speaker="S2")) + assert client.transcript_text(speaker_labels=True) == "S1: Hello. S2: Hi." + + +@pytest.mark.asyncio +async def test_timeline_records_speech_and_turn_events(client): + start_session(client) + client.emit(ServerMessageType.SPEECH_STARTED, {"message": "SpeechStarted", "metadata": {"start_time": 0.5}}) + client.emit(ServerMessageType.START_OF_TURN, {"message": "StartOfTurn", "metadata": {"start_time": 0.5}}) + client.emit(ServerMessageType.SPEECH_ENDED, {"message": "SpeechEnded", "metadata": {"end_time": 2.0}}) + client.emit(ServerMessageType.END_OF_TURN, {"message": "EndOfTurn", "metadata": {"end_time": 2.2}}) + + assert [(event.message, event.time) for event in client.timeline] == [ + ("SpeechStarted", 0.5), + ("StartOfTurn", 0.5), + ("SpeechEnded", 2.0), + ("EndOfTurn", 2.2), + ] + + +@pytest.mark.asyncio +async def test_every_message_is_recorded(client): + start_session(client) + unmodelled = {"message": "SomeFutureMessage", "metadata": {"value": 1}} + client.emit("SomeFutureMessage", unmodelled) + + recorded = [event["message"] for event in client.events] + assert recorded == [ServerMessageType.RECOGNITION_STARTED, "SomeFutureMessage"] + assert client.events[-1] == unmodelled + + +@pytest.mark.asyncio +async def test_event_recording_can_be_disabled(monkeypatch): + monkeypatch.delenv("SPEECHMATICS_AGENT_STT_URL", raising=False) + client = AsyncClient(api_key=API_KEY, record_events=False) + start_session(client) + assert client.events == [] + assert client.session_info.session_id == "session-1" + + +@pytest.mark.asyncio +async def test_audio_dropped_until_session_is_ready(client): + transport = StubTransport() + client._transport = transport + + await client.send_audio(b"\x00" * 32) + assert transport.audio == [] + + client.emit(ServerMessageType.RECOGNITION_STARTED, recognition_started()) + await client.send_audio(b"\x00" * 32) + assert transport.audio == [b"\x00" * 32] + + +@pytest.mark.asyncio +async def test_finalize_sends_force_end_of_utterance_with_timestamp(client): + transport = start_session(client) + await client.send_audio(b"\x00" * 32000) # 1 second of 16 kHz signed 16-bit PCM + + await client.force_end_of_utterance() + + assert transport.messages == [ + {"message": ClientMessageType.FORCE_END_OF_UTTERANCE.value, "timestamp": 1.0}, + ] + + +@pytest.mark.asyncio +async def test_finalize_accepts_an_explicit_timestamp(client): + transport = start_session(client) + await client.force_end_of_utterance(timestamp=4.2) + assert transport.messages[0]["timestamp"] == 4.2 + + +@pytest.mark.asyncio +async def test_finalize_can_omit_the_timestamp(client): + transport = start_session(client) + await client.force_end_of_utterance(timestamp=None) + assert "timestamp" not in transport.messages[0] + + +@pytest.mark.asyncio +async def test_sync_finalize_schedules_the_send(client): + transport = start_session(client) + await client.send_audio(b"\x00" * 16000) # 0.5 seconds + + client.finalize() + assert transport.messages == [] # scheduled, not yet sent + + await _drain() + assert transport.messages == [ + {"message": ClientMessageType.FORCE_END_OF_UTTERANCE.value, "timestamp": 0.5}, + ] + + +@pytest.mark.asyncio +async def test_finalize_latency_measured_against_the_flushed_segment(client): + start_session(client) + assert client.last_finalize_latency == 0.0 + + await client.force_end_of_utterance() + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hello there.")) + assert client.last_finalize_latency > 0.0 + + +@pytest.mark.asyncio +async def test_client_vad_mode_config_reaches_start_recognition(client): + transport = StubTransport() + client._transport = transport + client._config = TranscriptionConfig(language="en", vad_mode=VADMode.CLIENT) + + await client.send_message( + { + "message": ClientMessageType.START_RECOGNITION.value, + "transcription_config": client._config.to_dict(), + } + ) + + assert transport.messages[0]["transcription_config"]["vad_config"] == {"enabled": False} + + +@pytest.mark.asyncio +async def test_reset_transcript(client): + start_session(client) + client.emit(ServerMessageType.ADD_SEGMENT, segment_message("Hello.")) + client.reset_transcript() + assert client.transcript == "" + assert client.events == [] + + +@pytest.mark.asyncio +async def test_close_clears_session_state(client): + transport = start_session(client) + await client.close() + assert transport.closed + assert not client.is_connected + assert not client.is_ready_for_audio + + +async def _drain(): + """Let scheduled tasks run.""" + import asyncio + + await asyncio.sleep(0) + await asyncio.sleep(0) diff --git a/tests/agent_stt/test_config.py b/tests/agent_stt/test_config.py new file mode 100644 index 00000000..3b7e75f1 --- /dev/null +++ b/tests/agent_stt/test_config.py @@ -0,0 +1,63 @@ +import pytest + +from speechmatics.agent_stt import Model +from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import VADConfig +from speechmatics.agent_stt import VADMode + + +def test_server_vad_is_the_default(): + assert TranscriptionConfig().to_dict()["vad_config"] == {"enabled": True} + + +def test_client_vad_disables_service_vad(): + config = TranscriptionConfig(vad_mode=VADMode.CLIENT) + assert config.to_dict()["vad_config"] == {"enabled": False} + + +def test_vad_mode_is_not_sent(): + assert "vad_mode" not in TranscriptionConfig().to_dict() + + +def test_vad_tuning_passed_through(): + config = TranscriptionConfig(vad_config=VADConfig(window=0.3, onset_threshold=0.6, offset_threshold=0.4)) + assert config.to_dict()["vad_config"] == { + "window": 0.3, + "onset_threshold": 0.6, + "offset_threshold": 0.4, + "enabled": True, + } + + +def test_model_unset_by_default(): + """The service profile pins the model, so the SDK must not send one unasked.""" + assert "model" not in TranscriptionConfig().to_dict() + + +def test_model_sent_when_given(): + assert TranscriptionConfig(model=Model.ENHANCED).to_dict()["model"] == Model.ENHANCED + + +def test_emit_sentences_omitted_unless_set(): + assert "emit_sentences" not in TranscriptionConfig().to_dict() + assert TranscriptionConfig(emit_sentences=True).to_dict()["emit_sentences"] is True + assert TranscriptionConfig(emit_sentences=False).to_dict()["emit_sentences"] is False + + +def test_rt_fields_still_work(): + config = TranscriptionConfig(language="es", diarization="speaker", enable_partials=True, max_delay=1.5) + result = config.to_dict() + assert result["language"] == "es" + assert result["diarization"] == "speaker" + assert result["enable_partials"] is True + assert result["max_delay"] == 1.5 + + +def test_model_and_operating_point_conflict(): + with pytest.raises(ValueError): + TranscriptionConfig(model=Model.ENHANCED, operating_point="enhanced") + + +def test_operating_point_deprecated(): + with pytest.warns(DeprecationWarning): + TranscriptionConfig(operating_point="enhanced") diff --git a/tests/agent_stt/test_transcript.py b/tests/agent_stt/test_transcript.py new file mode 100644 index 00000000..bbbb347f --- /dev/null +++ b/tests/agent_stt/test_transcript.py @@ -0,0 +1,122 @@ +from speechmatics.agent_stt import Segment +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TimedEvent +from speechmatics.agent_stt import Transcript + + +def segment_message(transcript, start=0.0, end=1.0, speaker=None, is_final=True): + segment = {"transcript": transcript} + if speaker is not None: + segment["speaker"] = speaker + return { + "message": ServerMessageType.ADD_SEGMENT if is_final else ServerMessageType.ADD_PARTIAL_SEGMENT, + "segment": segment, + "metadata": {"start_time": start, "end_time": end}, + } + + +def test_segment_from_message(): + segment = Segment.from_message(segment_message("Hello world", 1.0, 2.5, speaker="S1")) + assert segment == Segment(transcript="Hello world", start_time=1.0, end_time=2.5, speaker="S1", is_final=True) + + +def test_partial_segment_is_not_final(): + assert Segment.from_message(segment_message("Hello", is_final=False)).is_final is False + + +def test_segment_without_speaker(): + assert Segment.from_message(segment_message("Hello")).speaker is None + + +def test_timed_event_from_start_time(): + event = TimedEvent.from_message({"message": "StartOfTurn", "metadata": {"start_time": 1.25}}) + assert event == TimedEvent(message="StartOfTurn", time=1.25) + + +def test_timed_event_from_end_time(): + event = TimedEvent.from_message({"message": "EndOfTurn", "metadata": {"end_time": 3.5}}) + assert event == TimedEvent(message="EndOfTurn", time=3.5) + + +def test_finals_accumulate(): + transcript = Transcript() + transcript.add_segment(segment_message("Hello there.")) + transcript.add_segment(segment_message("How are you?")) + assert transcript.text() == "Hello there. How are you?" + assert len(transcript.segments) == 2 + + +def test_partial_replaced_and_cleared_by_final(): + transcript = Transcript() + transcript.add_partial_segment(segment_message("Hel", is_final=False)) + transcript.add_partial_segment(segment_message("Hello wor", is_final=False)) + assert transcript.partial is not None + assert transcript.partial.transcript == "Hello wor" + assert transcript.text() == "" + + transcript.add_segment(segment_message("Hello world.")) + assert transcript.partial is None + assert transcript.text() == "Hello world." + + +def test_text_including_partial(): + transcript = Transcript() + transcript.add_segment(segment_message("Hello.")) + transcript.add_partial_segment(segment_message("How are", is_final=False)) + assert transcript.text() == "Hello." + assert transcript.text(include_partial=True) == "Hello. How are" + + +def test_text_with_speaker_labels(): + transcript = Transcript() + transcript.add_segment(segment_message("Hello.", speaker="S1")) + transcript.add_segment(segment_message("Hi.", speaker="S2")) + transcript.add_segment(segment_message("No speaker.")) + assert transcript.text(speaker_labels=True) == "S1: Hello. S2: Hi. No speaker." + + +def test_delimiter_from_language_pack(): + transcript = Transcript() + transcript.set_delimiter("") + transcript.add_segment(segment_message("你好")) + transcript.add_segment(segment_message("世界")) + assert transcript.text() == "你好世界" + + +def test_empty_segments_skipped(): + transcript = Transcript() + transcript.add_segment(segment_message("Hello.")) + transcript.add_segment(segment_message("")) + assert transcript.text() == "Hello." + + +def test_events_recorded(): + transcript = Transcript() + message = segment_message("Hello.") + transcript.record(message) + transcript.record({"message": "SomeFutureMessage", "payload": 1}) + assert transcript.events == [message, {"message": "SomeFutureMessage", "payload": 1}] + + +def test_event_recording_can_be_disabled(): + transcript = Transcript(record_events=False) + transcript.record(segment_message("Hello.")) + assert transcript.events == [] + + +def test_timeline(): + transcript = Transcript() + transcript.add_timed_event({"message": "SpeechStarted", "metadata": {"start_time": 0.5}}) + transcript.add_timed_event({"message": "EndOfTurn", "metadata": {"end_time": 2.0}}) + assert [event.message for event in transcript.timeline] == ["SpeechStarted", "EndOfTurn"] + + +def test_reset_keeps_delimiter(): + transcript = Transcript() + transcript.set_delimiter("") + transcript.add_segment(segment_message("Hello.")) + transcript.record(segment_message("Hello.")) + transcript.reset() + assert transcript.segments == [] + assert transcript.events == [] + assert transcript.delimiter == "" diff --git a/tests/agent_stt/test_url.py b/tests/agent_stt/test_url.py new file mode 100644 index 00000000..d168249e --- /dev/null +++ b/tests/agent_stt/test_url.py @@ -0,0 +1,75 @@ +from urllib.parse import parse_qs +from urllib.parse import urlparse + +import pytest + +from speechmatics.agent_stt import DEFAULT_AGENT_STT_URL +from speechmatics.agent_stt import resolve_url + + +@pytest.fixture(autouse=True) +def _clear_env(monkeypatch): + monkeypatch.delenv("SPEECHMATICS_AGENT_STT_URL", raising=False) + monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) + + +def _path(url): + return urlparse(url).path + + +def test_default_url(): + assert _path(resolve_url()) == _path(DEFAULT_AGENT_STT_URL) + + +def test_agent_segment_appended_to_rt_url(monkeypatch): + monkeypatch.setenv("SPEECHMATICS_RT_URL", "wss://example.com/v2") + assert _path(resolve_url()) == "/v2/agent" + + +def test_agent_stt_env_takes_precedence(monkeypatch): + monkeypatch.setenv("SPEECHMATICS_RT_URL", "wss://rt.example.com/v2") + monkeypatch.setenv("SPEECHMATICS_AGENT_STT_URL", "wss://agent.example.com/v2/agent") + url = resolve_url() + assert urlparse(url).hostname == "agent.example.com" + assert _path(url) == "/v2/agent" + + +def test_explicit_url_wins_over_env(monkeypatch): + monkeypatch.setenv("SPEECHMATICS_AGENT_STT_URL", "wss://agent.example.com/v2/agent") + assert urlparse(resolve_url("wss://custom.example.com/v2")).hostname == "custom.example.com" + + +def test_agent_segment_not_duplicated(): + assert _path(resolve_url("wss://example.com/v2/agent")) == "/v2/agent" + + +def test_trailing_slash_normalized(): + assert _path(resolve_url("wss://example.com/v2/")) == "/v2/agent" + + +def test_profile_appended(): + assert _path(resolve_url("wss://example.com/v2", profile="default")) == "/v2/agent/default" + + +def test_profile_not_duplicated(): + assert _path(resolve_url("wss://example.com/v2/agent/default", profile="default")) == "/v2/agent/default" + + +def test_profile_slashes_stripped(): + assert _path(resolve_url("wss://example.com/v2", profile="/default/")) == "/v2/agent/default" + + +def test_app_reported(): + params = parse_qs(urlparse(resolve_url("wss://example.com/v2", app="pipecat/1.0")).query) + assert params["sm-app"] == ["pipecat/1.0"] + + +def test_default_app_reported(): + params = parse_qs(urlparse(resolve_url("wss://example.com/v2")).query) + assert params["sm-app"][0].startswith("agent-stt-sdk/") + + +def test_existing_query_params_kept(): + params = parse_qs(urlparse(resolve_url("wss://example.com/v2?foo=bar&sm-app=existing")).query) + assert params["foo"] == ["bar"] + assert params["sm-app"] == ["existing"] From 98fc00a52b277b3e31657155ae976e11dbe72452 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Thu, 13 Aug 2026 10:17:21 +0100 Subject: [PATCH 02/19] Drop FIXED turn detection from the Pipecat migration plan Engine silence-based end of utterance is off for the Agent STT service, so TurnDetectionMode.FIXED has nothing to map onto and is removed rather than aliased. Records the settings that go with it, and notes in the SDK README that a turn ends only via the service VAD or finalize(). SMART_TURN stays open: it ran an in-process model, which this SDK does not. Co-Authored-By: Claude Opus 5 (1M context) --- sdk/agent_stt/PLAN.md | 35 +++++++++++++++++++++++++++-------- sdk/agent_stt/README.md | 4 ++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index 589f6412..b361d0e1 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -116,20 +116,39 @@ unset the message carries only the profile's locked `operating_point` - no confl touched by this change. The migration: - swap the import block to `speechmatics.agent_stt` -- `TurnDetectionMode.EXTERNAL` -> `VADMode.CLIENT` + `force_end_of_utterance()` on - `VADUserStoppedSpeakingFrame`; `ADAPTIVE`/`SMART_TURN` -> `VADMode.SERVER` and let - `StartOfTurn`/`EndOfTurn` drive `ProposedUserStartedSpeakingFrame`/`ProposedUserStoppedSpeakingFrame` +- collapse `TurnDetectionMode` onto the two modes that exist: + - `EXTERNAL` -> `VADMode.CLIENT`, with `finalize()` on `VADUserStoppedSpeakingFrame` + - `ADAPTIVE` -> `VADMode.SERVER`, with `StartOfTurn`/`EndOfTurn` driving + `ProposedUserStartedSpeakingFrame`/`ProposedUserStoppedSpeakingFrame` + - `FIXED` -> removed (see below) - `AddPartialSegment` -> `InterimTranscriptionFrame`, `AddSegment` -> `TranscriptionFrame` - drop the `pipecat-ai[speechmatics]` onnxruntime/transformers extras that only existed for the bundled VAD and smart-turn models -- keep `SpeechmaticsSTTSettings` as the public surface so user code doesn't change +- drop `end_of_utterance_silence_trigger` and `end_of_utterance_max_delay` from + `SpeechmaticsSTTSettings` and `InputParams`, along with the passthrough at + `stt.py:788` and `stt.py:1252` - engine silence-based end of utterance is off for this + service, so both are no-ops +- `_enable_vad` (`stt.py:530`) becomes `vad_mode is VADMode.CLIENT`, since that is now the only + mode where Pipecat's own VAD drives the boundary +- otherwise keep `SpeechmaticsSTTSettings` as the public surface so user code doesn't change + +### `FIXED` mode is removed + +`end_of_utterance_silence_trigger` is off for this service: the default profile pins it to `0.0`, +the service consumes `EndOfUtterance` rather than forwarding it, and a non-forced end of utterance +does not close a segment. So there is nothing for `TurnDetectionMode.FIXED` to mean here and it +goes away rather than being aliased to another mode. + +Turns end in exactly two ways, which is what `VADMode` models: the service's VAD, or the +client calling `finalize()`. Open questions to settle before starting milestone 2: - ~~speaker-focus / known-speaker features in the Pipecat service have no service-side equivalent yet~~ - dropped for now, to be added in a later service release. `known_speakers` still works, since `speaker_diarization_config.speakers` passes straight through. -- non-forced `EndOfUtterance` no longer flushes a segment (the default profile sets - `end_of_utterance_silence_trigger: 0.0`), so engine-silence endpointing is not available - - confirm that is intended for the `FIXED` mode Pipecat exposes. If it is, `FIXED` has to map - onto `VADMode.SERVER` (or be removed for this service) rather than onto engine silence. +- ~~engine-silence endpointing / `FIXED` mode~~ - removed, see above. +- `SMART_TURN` has no service-side equivalent either: the ML turn model ran in-process in the + `voice` SDK, and this SDK loads no models. It cannot be honoured as specified, so it either + goes the same way as `FIXED`, or maps to `VADMode.SERVER` with a deprecation warning - a + behaviour downgrade for anyone relying on it, so worth calling out in the changelog either way. diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index b9765647..5772e88a 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -105,6 +105,10 @@ Passed through from the RT engine: `RecognitionStarted`, `AudioAdded`, `AddTrans `model` is left unset by default: the service profile pins the model for the session. +Engine silence-based end of utterance is off for this service, and `EndOfUtterance` is not +forwarded, so `conversation_config.end_of_utterance_silence_trigger` does not close segments. A +turn ends either because the service's VAD said so, or because you called `finalize()`. + ## Endpoint The Agent STT endpoint is the RT endpoint plus `/agent`, optionally followed by a service From e8ad6f401073ad4e988e17bc42ef89821d205a3e Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Thu, 13 Aug 2026 10:23:46 +0100 Subject: [PATCH 03/19] Drop SMART_TURN from the Pipecat migration plan The service has no smart-turn endpoint yet and this SDK loads no models, so the mode cannot be honoured. It costs nothing: any host-side endpointing, including Pipecat's own turn analyzer, reaches the service through finalize(). Makes VADMode.CLIENT explicit that it is agnostic about what produced the end-of-speech signal - VAD, turn model, or push-to-talk. Co-Authored-By: Claude Opus 5 (1M context) --- sdk/agent_stt/PLAN.md | 24 ++++++++++++------- sdk/agent_stt/README.md | 11 +++++---- .../speechmatics/agent_stt/_models.py | 8 ++++--- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index b361d0e1..439b483f 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -120,7 +120,7 @@ touched by this change. The migration: - `EXTERNAL` -> `VADMode.CLIENT`, with `finalize()` on `VADUserStoppedSpeakingFrame` - `ADAPTIVE` -> `VADMode.SERVER`, with `StartOfTurn`/`EndOfTurn` driving `ProposedUserStartedSpeakingFrame`/`ProposedUserStoppedSpeakingFrame` - - `FIXED` -> removed (see below) + - `FIXED` and `SMART_TURN` -> removed (see below) - `AddPartialSegment` -> `InterimTranscriptionFrame`, `AddSegment` -> `TranscriptionFrame` - drop the `pipecat-ai[speechmatics]` onnxruntime/transformers extras that only existed for the bundled VAD and smart-turn models @@ -132,15 +132,23 @@ touched by this change. The migration: mode where Pipecat's own VAD drives the boundary - otherwise keep `SpeechmaticsSTTSettings` as the public surface so user code doesn't change -### `FIXED` mode is removed +### `FIXED` and `SMART_TURN` are removed `end_of_utterance_silence_trigger` is off for this service: the default profile pins it to `0.0`, the service consumes `EndOfUtterance` rather than forwarding it, and a non-forced end of utterance does not close a segment. So there is nothing for `TurnDetectionMode.FIXED` to mean here and it goes away rather than being aliased to another mode. -Turns end in exactly two ways, which is what `VADMode` models: the service's VAD, or the -client calling `finalize()`. +`SMART_TURN` goes for the same reason: the service has no smart-turn endpoint yet (planned for a +later release), and this SDK loads no models, so the mode cannot be honoured as specified. It +comes back when the service does, as a `VADMode.SERVER` variant. + +Removing it costs Pipecat users nothing, because Pipecat's own turn analyzer still works: any +host-side endpointing - VAD, ML turn model, push-to-talk - reaches the service the same way, +through `finalize()`. `VADMode.CLIENT` is agnostic about what produced the signal. + +Turns therefore end in exactly two ways, which is what `VADMode` models: the service's VAD, or +the client calling `finalize()`. Open questions to settle before starting milestone 2: @@ -148,7 +156,7 @@ Open questions to settle before starting milestone 2: equivalent yet~~ - dropped for now, to be added in a later service release. `known_speakers` still works, since `speaker_diarization_config.speakers` passes straight through. - ~~engine-silence endpointing / `FIXED` mode~~ - removed, see above. -- `SMART_TURN` has no service-side equivalent either: the ML turn model ran in-process in the - `voice` SDK, and this SDK loads no models. It cannot be honoured as specified, so it either - goes the same way as `FIXED`, or maps to `VADMode.SERVER` with a deprecation warning - a - behaviour downgrade for anyone relying on it, so worth calling out in the changelog either way. +- ~~`SMART_TURN`~~ - removed until the service implements it. Host-side turn models keep working + through `VADMode.CLIENT`. + +Both removals are user-visible, so they need a changelog entry when the Pipecat change lands. diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index 5772e88a..eecc6d6d 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -45,11 +45,11 @@ config = TranscriptionConfig( vad_config=VADConfig(window=0.2, onset_threshold=0.5, offset_threshold=0.35), ) -# Your VAD - Pipecat, LiveKit, or your own. The service's VAD stays off. +# Your endpointing - Pipecat, LiveKit, or your own. The service's VAD stays off. config = TranscriptionConfig(vad_mode=VADMode.CLIENT) ``` -With `VADMode.CLIENT`, close each turn when your VAD reports end of speech: +With `VADMode.CLIENT`, close each turn when your side decides speech has ended: ```python client.finalize() # from a sync callback @@ -57,8 +57,11 @@ await client.force_end_of_utterance() # from async code ``` Either sends `ForceEndOfUtterance` stamped with the audio position at the moment of the call, so -the service cuts the turn where your VAD heard the end of speech rather than wherever the send -lands. The flushed segment comes back as a normal `AddSegment`. +the service cuts the turn where you heard the end of speech rather than wherever the send lands. +The flushed segment comes back as a normal `AddSegment`. + +What decides that is entirely yours - a VAD, an ML turn model, or a push-to-talk button. The SDK +only cares that something calls `finalize()`. ## Session output diff --git a/sdk/agent_stt/speechmatics/agent_stt/_models.py b/sdk/agent_stt/speechmatics/agent_stt/_models.py index c6ee58f3..9f9a5317 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_models.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_models.py @@ -104,13 +104,15 @@ class ServerMessageType(str, Enum): class VADMode(str, Enum): """ - Where turn boundaries come from. This SDK never runs a VAD itself. + Where turn boundaries come from. This SDK never detects them itself. Attributes: SERVER: The service runs its own VAD and turn detection, emitting SpeechStarted, SpeechEnded, StartOfTurn and EndOfTurn, and closing segments itself. - CLIENT: The client runs its own VAD (Pipecat, LiveKit, ...) and closes each turn by - calling `finalize()`, which sends ForceEndOfUtterance with an audio timestamp. + CLIENT: The application closes each turn by calling `finalize()`, which sends + ForceEndOfUtterance with an audio timestamp. Whatever produced that signal is up to + the application - a VAD, a turn model, or a push-to-talk button - so a host + framework's own endpointing (Pipecat, LiveKit, ...) works unchanged. """ SERVER = "server" From b639e9c9c2c1c278026e13712894fd933a258897 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Thu, 13 Aug 2026 11:40:20 +0100 Subject: [PATCH 04/19] Add AdditionalVocabEntry to the Agent STT SDK A typed entry for transcription_config.additional_vocab, so callers can pass words with pronunciation hints instead of raw dicts. Both forms are accepted. Co-Authored-By: Claude Opus 5 (1M context) --- .../speechmatics/agent_stt/__init__.py | 2 ++ .../speechmatics/agent_stt/_models.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/sdk/agent_stt/speechmatics/agent_stt/__init__.py b/sdk/agent_stt/speechmatics/agent_stt/__init__.py index e1cef147..75ab889f 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/__init__.py +++ b/sdk/agent_stt/speechmatics/agent_stt/__init__.py @@ -43,6 +43,7 @@ from ._models import DEFAULT_WORD_DELIMITER from ._models import SEGMENT_MESSAGES from ._models import TIMED_MESSAGES +from ._models import AdditionalVocabEntry from ._models import ClientMessageType from ._models import LanguagePackInfo from ._models import Segment @@ -68,6 +69,7 @@ "AgentSTTClient", "AsyncClient", # Config + "AdditionalVocabEntry", "AudioEncoding", "AudioEventsConfig", "AudioFormat", diff --git a/sdk/agent_stt/speechmatics/agent_stt/_models.py b/sdk/agent_stt/speechmatics/agent_stt/_models.py index 9f9a5317..0647d6e7 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_models.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_models.py @@ -5,6 +5,7 @@ from enum import Enum from typing import Any from typing import Optional +from typing import Union from warnings import warn from speechmatics.rt import Model @@ -135,6 +136,23 @@ class VADConfig: offset_threshold: Optional[float] = None +@dataclass +class AdditionalVocabEntry: + """ + A word to bias the engine towards, optionally with pronunciation hints. + + Attributes: + content: The word or phrase. + sounds_like: Alternative pronunciations, written as they sound. + + Examples: + >>> AdditionalVocabEntry(content="Speechmatics", sounds_like=["speech matics"]) + """ + + content: str + sounds_like: Optional[list[str]] = None + + @dataclass class TranscriptionConfig(RTTranscriptionConfig): """ @@ -148,6 +166,8 @@ class TranscriptionConfig(RTTranscriptionConfig): vad_config: Tuning for the service's VAD, used when `vad_mode` is `SERVER`. emit_sentences: Close a segment on every sentence boundary, not just at the turn boundary. + additional_vocab: Words to bias the engine towards, as `AdditionalVocabEntry` objects + or raw dicts. model: Left unset by default. The service's profile pins the model for the session, and sending it alongside the profile's `operating_point` would put both keys in the resolved StartRecognition. @@ -161,6 +181,7 @@ class TranscriptionConfig(RTTranscriptionConfig): """ model: Optional[Model] = None + additional_vocab: Optional[list[Union[AdditionalVocabEntry, dict[str, Any]]]] = None vad_mode: VADMode = VADMode.SERVER vad_config: VADConfig = field(default_factory=VADConfig) emit_sentences: Optional[bool] = None From 50ade3ddddcef5a5abe293f2f3f134a1269ff6ae Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Mon, 17 Aug 2026 22:38:29 +0100 Subject: [PATCH 05/19] Default the Agent STT SDK to the linden-1 model The SDK previously left `model` unset, on the reasoning that the service's default profile pins and locks `operating_point: enhanced`, so sending a model name alongside it would put both keys in the merged StartRecognition. That reasoning held for a direct connection to the service websocket, but it is not how requests actually reach it: they go through a proxy first, which resolves the Agent STT model name onto the engine's operating point. The transcriber therefore never sees a name it has no notion of, and sending `linden-1` is safe. Replace the re-exported RT `Model` with an Agent STT one. The RT models (`enhanced`, `standard`) are not Agent STT models, so leaving them reachable from this package only invited configs the service would reject. Default resolution uses the same `_UNSET` sentinel pattern as `rt.TranscriptionConfig` rather than a plain default, because the deprecated `operating_point` has to suppress the model rather than collide with it: passing `operating_point` leaves `model` unsent, and passing both still raises. `DEFAULT_MODEL` is a separate constant and the docstrings name it rather than the value, so linden-2 is one enum member plus, if it becomes the default, one line. The end-to-end verification recorded in PLAN.md predates this and ran against the service with a stub transcriber and no proxy, so it exercised neither the model default nor its resolution. Noted there as still needing a run against the real proxy. Co-Authored-By: Claude Opus 5 (1M context) --- sdk/agent_stt/PLAN.md | 19 +++++++---- sdk/agent_stt/README.md | 13 +++++++- .../speechmatics/agent_stt/__init__.py | 4 ++- .../speechmatics/agent_stt/_models.py | 32 +++++++++++++++---- tests/agent_stt/test_config.py | 17 +++++++--- 5 files changed, 66 insertions(+), 19 deletions(-) diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index 439b483f..d6e28fbb 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -76,9 +76,12 @@ reconnect-free lifecycle, `send_audio`, `transcribe`, `stop_session`, What the subclass adds: 1. URL resolution (`/agent` + profile, `SPEECHMATICS_AGENT_STT_URL` env override). -2. `TranscriptionConfig` with `vad_mode`, `vad_config`, `emit_sentences`, and `model` left - **unset** by default - the service's default profile pins `operating_point: enhanced` and - locks it, so sending `model` too would put both keys in the merged `StartRecognition`. +2. `TranscriptionConfig` with `vad_mode`, `vad_config`, `emit_sentences`, and an Agent STT + `Model` enum defaulting to `linden-1`. The request goes to the proxy rather than the service + websocket directly, and the proxy resolves the Agent STT model name onto the engine's + operating point, so the transcriber never sees a name it has no notion of. The deprecated + `operating_point` suppresses the `model` default, so the merged `StartRecognition` never + carries both keys. `linden-2` lands as one more enum member. 3. Segment-level transcript accumulation, so `client.transcript` reads like the RT flow does. 4. A raw event log (`client.events`) capturing every server message, including ones this SDK version doesn't model yet - that is the "accept and save the other messages" requirement. @@ -105,9 +108,13 @@ mode: `/v2/agent/default` routing, three `ForceEndOfUtterance` turns each flushi `AddSegment`, speaker labels, the event log, and `client.transcript` coming out as `"Hello there. How are you today? Goodbye."`. -The StartRecognition the service forwarded downstream confirmed the two design points that -mattered: `vad_config` and `emit_sentences` are stripped, and because the SDK leaves `model` -unset the message carries only the profile's locked `operating_point` - no conflicting pair. +The StartRecognition the service forwarded downstream confirmed that `vad_config` and +`emit_sentences` are stripped before it reaches the RT engine. + +That run predates the `linden-1` default, so it sent no `model` at all and the message carried +only the profile's locked `operating_point`. Model resolution happens in the proxy ahead of the +service, which this stub setup does not exercise, so it still needs verifying against the real +proxy. ## Milestone 2 - Pipecat (not in this change) diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index eecc6d6d..4ce22fcb 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -106,7 +106,18 @@ Passed through from the RT engine: `RecognitionStarted`, `AudioAdded`, `AddTrans | `vad_config` | `window`, `onset_threshold`, `offset_threshold` for the service's VAD | | `emit_sentences` | Close a segment on every sentence boundary, not only at the turn boundary | -`model` is left unset by default: the service profile pins the model for the session. +`model` takes an Agent STT `Model` and defaults to `DEFAULT_MODEL` (`Model.LINDEN_1`): + +```python +from speechmatics.agent_stt import Model, TranscriptionConfig + +config = TranscriptionConfig(model=Model.LINDEN_1) +``` + +The proxy in front of the service resolves the Agent STT model name onto the engine's operating +point, so the transcriber never sees a name it has no notion of. The RT models (`enhanced`, +`standard`) are not Agent STT models and are not accepted here; the deprecated `operating_point` +still passes through, and suppresses the `model` default so the two never arrive together. Engine silence-based end of utterance is off for this service, and `EndOfUtterance` is not forwarded, so `conversation_config.end_of_utterance_silence_trigger` does not close segments. A diff --git a/sdk/agent_stt/speechmatics/agent_stt/__init__.py b/sdk/agent_stt/speechmatics/agent_stt/__init__.py index 75ab889f..c27a08af 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/__init__.py +++ b/sdk/agent_stt/speechmatics/agent_stt/__init__.py @@ -26,7 +26,6 @@ from speechmatics.rt import EventEmitter from speechmatics.rt import JWTAuth from speechmatics.rt import Microphone -from speechmatics.rt import Model from speechmatics.rt import SessionError from speechmatics.rt import SpeakerDiarizationConfig from speechmatics.rt import SpeakerIdentifier @@ -39,6 +38,7 @@ from ._client import AgentSTTClient from ._client import AsyncClient from ._models import DEFAULT_CHUNK_SIZE +from ._models import DEFAULT_MODEL from ._models import DEFAULT_SAMPLE_RATE from ._models import DEFAULT_WORD_DELIMITER from ._models import SEGMENT_MESSAGES @@ -46,6 +46,7 @@ from ._models import AdditionalVocabEntry from ._models import ClientMessageType from ._models import LanguagePackInfo +from ._models import Model from ._models import Segment from ._models import ServerMessageType from ._models import SessionInfo @@ -60,6 +61,7 @@ __all__ = [ "DEFAULT_AGENT_STT_URL", "DEFAULT_CHUNK_SIZE", + "DEFAULT_MODEL", "DEFAULT_SAMPLE_RATE", "DEFAULT_WORD_DELIMITER", "SEGMENT_MESSAGES", diff --git a/sdk/agent_stt/speechmatics/agent_stt/_models.py b/sdk/agent_stt/speechmatics/agent_stt/_models.py index 0647d6e7..edd29322 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_models.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_models.py @@ -6,9 +6,9 @@ from typing import Any from typing import Optional from typing import Union +from typing import cast from warnings import warn -from speechmatics.rt import Model from speechmatics.rt import TranscriptionConfig as RTTranscriptionConfig DEFAULT_SAMPLE_RATE = 16000 @@ -16,6 +16,22 @@ DEFAULT_WORD_DELIMITER = " " +class Model(str, Enum): + """ + Models available on the Agent STT service. + + Attributes: + LINDEN_1: The first Agent STT model. + """ + + LINDEN_1 = "linden-1" + + +DEFAULT_MODEL = Model.LINDEN_1 + +_UNSET = cast(Model, object()) + + class ClientMessageType(str, Enum): """ Message types that can be sent from client to the Agent STT service. @@ -168,9 +184,9 @@ class TranscriptionConfig(RTTranscriptionConfig): boundary. additional_vocab: Words to bias the engine towards, as `AdditionalVocabEntry` objects or raw dicts. - model: Left unset by default. The service's profile pins the model for the session, - and sending it alongside the profile's `operating_point` would put both keys in - the resolved StartRecognition. + model: Agent STT model, defaulting to `DEFAULT_MODEL`. The proxy in front of the + service resolves the name to the engine's operating point, so a model the + transcriber has no notion of still routes correctly. Examples: Service VAD, sentence-level segments: @@ -180,15 +196,17 @@ class TranscriptionConfig(RTTranscriptionConfig): >>> config = TranscriptionConfig(language="en", vad_mode=VADMode.CLIENT) """ - model: Optional[Model] = None + model: Model = _UNSET additional_vocab: Optional[list[Union[AdditionalVocabEntry, dict[str, Any]]]] = None vad_mode: VADMode = VADMode.SERVER vad_config: VADConfig = field(default_factory=VADConfig) emit_sentences: Optional[bool] = None def __post_init__(self) -> None: - if self.model is not None and self.operating_point is not None: + if self.model is not _UNSET and self.operating_point is not None: raise ValueError("Cannot specify both 'model' and 'operating_point'. Use 'model' instead.") + if self.model is _UNSET and self.operating_point is None: + self.model = DEFAULT_MODEL if self.operating_point is not None: warn("'operating_point' is deprecated, use 'model' instead.", DeprecationWarning, stacklevel=2) @@ -201,6 +219,8 @@ def to_dict(self) -> dict[str, Any]: from `vad_mode` and `vad_mode` itself dropped. """ result = super().to_dict() + if self.model is _UNSET: + result.pop("model", None) result.pop("vad_mode", None) vad_config = result.pop("vad_config", None) or {} vad_config["enabled"] = self.vad_mode is VADMode.SERVER diff --git a/tests/agent_stt/test_config.py b/tests/agent_stt/test_config.py index 3b7e75f1..9939794a 100644 --- a/tests/agent_stt/test_config.py +++ b/tests/agent_stt/test_config.py @@ -29,13 +29,20 @@ def test_vad_tuning_passed_through(): } -def test_model_unset_by_default(): - """The service profile pins the model, so the SDK must not send one unasked.""" - assert "model" not in TranscriptionConfig().to_dict() +def test_model_defaults_to_linden_1(): + assert TranscriptionConfig().to_dict()["model"] == "linden-1" def test_model_sent_when_given(): - assert TranscriptionConfig(model=Model.ENHANCED).to_dict()["model"] == Model.ENHANCED + assert TranscriptionConfig(model=Model.LINDEN_1).to_dict()["model"] == Model.LINDEN_1 + + +def test_model_omitted_when_operating_point_is_used(): + """The deprecated operating_point must not arrive alongside a defaulted model.""" + with pytest.warns(DeprecationWarning): + config = TranscriptionConfig(operating_point="enhanced") + assert "model" not in config.to_dict() + assert config.to_dict()["operating_point"] == "enhanced" def test_emit_sentences_omitted_unless_set(): @@ -55,7 +62,7 @@ def test_rt_fields_still_work(): def test_model_and_operating_point_conflict(): with pytest.raises(ValueError): - TranscriptionConfig(model=Model.ENHANCED, operating_point="enhanced") + TranscriptionConfig(model=Model.LINDEN_1, operating_point="enhanced") def test_operating_point_deprecated(): From 7189cfb2a40822fa7be51f8140d8f4c0b25874bf Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 10:56:26 +0100 Subject: [PATCH 06/19] description improvement for versioning --- sdk/agent_stt/speechmatics/agent_stt/_version.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/agent_stt/speechmatics/agent_stt/_version.py b/sdk/agent_stt/speechmatics/agent_stt/_version.py index 78876859..dca9855c 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_version.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_version.py @@ -7,8 +7,12 @@ def get_version() -> str: """ Get the installed version of the speechmatics-agent-stt package. + This function attempts to retrieve the package version using multiple + fallback strategies to ensure it works in various deployment scenarios. + Returns: - The package version, or "0.0.0" when it cannot be determined. + str: The package version string (e.g., "1.2.3"), or "0.0.0" if + version cannot be determined. """ try: return importlib.metadata.version("speechmatics-agent-stt") From dfdaf6d6627c5005eb81364e3c068e4700738436 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 12:13:39 +0100 Subject: [PATCH 07/19] add examples for testing purposes --- examples/agent_stt/README.md | 4 + examples/agent_stt/microphone_windows/main.py | 150 ++++++++++++++++ examples/agent_stt/realtime_file/main.py | 169 ++++++++++++++++++ 3 files changed, 323 insertions(+) create mode 100644 examples/agent_stt/microphone_windows/main.py create mode 100644 examples/agent_stt/realtime_file/main.py diff --git a/examples/agent_stt/README.md b/examples/agent_stt/README.md index a56278c9..9e007bd8 100644 --- a/examples/agent_stt/README.md +++ b/examples/agent_stt/README.md @@ -9,11 +9,15 @@ The service needs 16 kHz raw PCM, so the file examples take a 16 kHz WAV and def | Example | What it shows | | --- | --- | | [file/main.py](file/main.py) | File transcription with the service's VAD; segments, turn events, transcript at the end | +| [realtime_file/main.py](realtime_file/main.py) | The same file paced at wall-clock speed, with the lag of each message behind the audio | | [client_vad/main.py](client_vad/main.py) | The client owns turn boundaries and calls `finalize()`, as Pipecat and LiveKit do | | [microphone/main.py](microphone/main.py) | Live microphone with diarization and speaker-labelled transcript (needs `pyaudio`) | +| [microphone_windows/main.py](microphone_windows/main.py) | The same, set up for Windows: device selection, in-place partials, Ctrl+C shutdown | ```bash python examples/agent_stt/file/main.py +python examples/agent_stt/realtime_file/main.py python examples/agent_stt/client_vad/main.py python examples/agent_stt/microphone/main.py +py examples\agent_stt\microphone_windows\main.py ``` diff --git a/examples/agent_stt/microphone_windows/main.py b/examples/agent_stt/microphone_windows/main.py new file mode 100644 index 00000000..1451f9ec --- /dev/null +++ b/examples/agent_stt/microphone_windows/main.py @@ -0,0 +1,150 @@ +"""Live microphone transcription with the Agent STT service, set up for Windows. + +The service runs its own VAD, so it decides where each turn ends; this script only captures +the microphone and prints what comes back. Partials are rewritten in place on one line and +each closed segment is printed above them. + +Setup in PowerShell: + + py -m pip install speechmatics-agent-stt pyaudio + $env:SPEECHMATICS_API_KEY = "your-key" + py examples\\agent_stt\\microphone_windows\\main.py + +Add `$env:SPEECHMATICS_RT_URL = "wss://preview.rt.speechmatics.com/v2"` to point at a +service. Use `--list-devices` and `--device N` when Windows picks the wrong input. +""" + +import argparse +import asyncio +import signal +import sys + +from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import Microphone +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig + +SAMPLE_RATE = 16000 +CHUNK_SIZE = 1024 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--list-devices", action="store_true", help="list input devices and exit") + parser.add_argument("--device", type=int, help="input device index, from --list-devices") + parser.add_argument("--language", default="en") + parser.add_argument("--diarization", action="store_true", help="label segments by speaker") + parser.add_argument("--no-partials", action="store_true") + return parser.parse_args() + + +class Console: + """Keeps the partial on one rewritten line, with finals printed above it.""" + + def __init__(self) -> None: + self._partial_width = 0 + + def partial(self, text: str) -> None: + line = f"[partial] {text}" + print("\r" + line.ljust(self._partial_width), end="", flush=True) + self._partial_width = len(line) + + def line(self, text: str) -> None: + print("\r" + text.ljust(self._partial_width), flush=True) + self._partial_width = 0 + + +def list_devices() -> None: + devices = Microphone.list_devices() + if not devices: + print("No input devices found. Is pyaudio installed, and does Windows list a microphone?") + return + for device in devices: + print(f" {device['index']:>2} {device['name']} ({device['channels']} ch)") + + +def build_client(args: argparse.Namespace, console: Console) -> AsyncClient: + config = TranscriptionConfig( + language=args.language, + enable_partials=not args.no_partials, + diarization="speaker" if args.diarization else None, + ) + + # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_AGENT_STT_URL to point at a local service + client = AsyncClient(config=config) + + @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) + def handle_partial_segment(message): + console.partial(message["segment"]["transcript"]) + + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + segment = message["segment"] + speaker = f"{segment['speaker']}: " if segment.get("speaker") else "" + console.line(f"[final] {speaker}{segment['transcript']}") + + @client.on(ServerMessageType.END_OF_TURN) + def handle_end_of_turn(message): + console.line(f"[turn] end at {message['metadata']['end_time']:.2f}s") + + @client.on(ServerMessageType.ERROR) + def handle_error(message): + console.line(f"[error] {message}") + + return client + + +async def capture(client: AsyncClient, mic: Microphone, stop: asyncio.Event) -> None: + """Pump microphone frames until Ctrl+C, which sets `stop`.""" + while not stop.is_set(): + await client.send_audio(await mic.read(CHUNK_SIZE)) + + +async def main() -> None: + args = parse_args() + + if args.list_devices: + list_devices() + return + + mic = Microphone(sample_rate=SAMPLE_RATE, chunk_size=CHUNK_SIZE, device_index=args.device) + if not mic.is_available: + print("pyaudio is not installed. Install it with: py -m pip install pyaudio") + return + if not mic.start(): + print(f"Could not open the microphone at {SAMPLE_RATE} Hz. Available inputs:") + list_devices() + print( + "\nPick one with --device N. If none open, set the device's Default Format to\n" + "16000 Hz in Sound Control Panel > Recording > Properties > Advanced." + ) + return + + # Ctrl+C on Windows will not interrupt a pending await, so shut down through an event + stop = asyncio.Event() + signal.signal(signal.SIGINT, lambda *_: stop.set()) + + console = Console() + client = build_client(args, console) + + try: + async with client: + print("\nMicrophone ready - speak now (Ctrl+C to stop)\n") + await capture(client, mic, stop) + finally: + mic.stop() + + console.line("") + print(f"Transcript: {client.transcript_text(speaker_labels=args.diarization)}") + + +# Windows spawns child processes by re-importing this file, so keep startup behind the guard +if __name__ == "__main__": + # The Windows console defaults to a legacy code page that cannot print every transcript + if sys.platform == "win32": + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + + asyncio.run(main()) diff --git a/examples/agent_stt/realtime_file/main.py b/examples/agent_stt/realtime_file/main.py new file mode 100644 index 00000000..0cb8d030 --- /dev/null +++ b/examples/agent_stt/realtime_file/main.py @@ -0,0 +1,169 @@ +"""Stream a 16 kHz WAV file to the Agent STT service at wall-clock speed. + +A file read at full speed reaches the service far ahead of real time, so VAD windows, turn +boundaries and latency all read wrong. This paces the send loop so each frame leaves at the +moment its audio would have been captured live, and prints how far behind the audio position +every message arrives - the number worth watching when testing locally. + +Run with: python examples/agent_stt/realtime_file/main.py [path/to/16kHz.wav] +""" + +import argparse +import asyncio +import time +import wave +from typing import Optional + +from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import ServerMessageType +from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import VADConfig +from speechmatics.agent_stt import VADMode + +DEFAULT_AUDIO_FILE = "./tests/voice/assets/audio_01_16kHz.wav" +SAMPLE_RATE = 16000 +BYTES_PER_SAMPLE = 2 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("audio", nargs="?", default=DEFAULT_AUDIO_FILE, help="16 kHz mono WAV file") + parser.add_argument("--language", default="en") + parser.add_argument("--chunk-ms", type=float, default=20.0, help="audio frame size in milliseconds") + parser.add_argument( + "--vad", + choices=[VADMode.SERVER.value, VADMode.CLIENT.value], + default=VADMode.SERVER.value, + help="who closes turns: the service, or this script calling finalize()", + ) + parser.add_argument("--vad-window", type=float, help="silence in seconds before the service closes a turn") + parser.add_argument( + "--turn-seconds", type=float, default=5.0, help="fake client-VAD turn length, --vad client only" + ) + parser.add_argument("--no-partials", action="store_true") + parser.add_argument("--emit-sentences", action="store_true", help="close a segment on every sentence boundary") + return parser.parse_args() + + +class Clock: + """Wall clock for the streaming run, and the lag of each message behind the audio.""" + + def __init__(self) -> None: + self._started_at = time.monotonic() + self.segment_lags = [] + + def start(self) -> None: + """Reset to zero, so the connection handshake does not count as streaming time.""" + self._started_at = time.monotonic() + + @property + def elapsed(self) -> float: + return time.monotonic() - self._started_at + + def log(self, tag: str, text: str, *, audio_time: Optional[float] = None, record: bool = False) -> None: + lag = "" + if audio_time is not None: + behind = self.elapsed - audio_time + if record: + self.segment_lags.append(behind) + lag = f"+{behind * 1000:>5.0f}ms" + print(f"[{self.elapsed:6.2f}s] {tag:<10}{lag:<10} {text}") + + +def build_client(args: argparse.Namespace, clock: Clock) -> AsyncClient: + config = TranscriptionConfig( + language=args.language, + enable_partials=not args.no_partials, + vad_mode=VADMode(args.vad), + vad_config=VADConfig(window=args.vad_window) if args.vad_window is not None else None, + emit_sentences=args.emit_sentences, + ) + + # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_AGENT_STT_URL to point at a local service + client = AsyncClient(config=config) + + @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) + def handle_partial_segment(message): + clock.log("[partial]", message["segment"]["transcript"], audio_time=message["metadata"]["end_time"]) + + @client.on(ServerMessageType.ADD_SEGMENT) + def handle_segment(message): + clock.log( + "[final]", + message["segment"]["transcript"], + audio_time=message["metadata"]["end_time"], + record=True, + ) + + @client.on(ServerMessageType.SPEECH_STARTED) + def handle_speech_started(message): + clock.log("[speech]", f"started at {message['metadata']['start_time']:.2f}s") + + @client.on(ServerMessageType.SPEECH_ENDED) + def handle_speech_ended(message): + clock.log("[speech]", f"ended at {message['metadata']['end_time']:.2f}s") + + @client.on(ServerMessageType.START_OF_TURN) + def handle_start_of_turn(message): + clock.log("[turn]", f"start at {message['metadata']['start_time']:.2f}s") + + @client.on(ServerMessageType.END_OF_TURN) + def handle_end_of_turn(message): + end_time = message["metadata"]["end_time"] + clock.log("[turn]", f"end at {end_time:.2f}s", audio_time=end_time) + + @client.on(ServerMessageType.ERROR) + def handle_error(message): + clock.log("[error]", str(message)) + + return client + + +async def stream(client: AsyncClient, wav: wave.Wave_read, args: argparse.Namespace, clock: Clock) -> None: + """Send the file frame by frame, releasing each frame no earlier than its capture time.""" + frames_per_chunk = int(SAMPLE_RATE * args.chunk_ms / 1000) + next_turn_end = args.turn_seconds + clock.start() + + while frame := wav.readframes(frames_per_chunk): + # Hold each frame until the wall clock reaches the end of the audio it carries, so a + # frame leaves exactly when a live capture would have finished recording it. Paced off + # the session clock rather than per-frame sleeps, so the send does not drift. + frame_end = client.audio_seconds_sent + len(frame) / (SAMPLE_RATE * BYTES_PER_SAMPLE) + early = frame_end - clock.elapsed + if early > 0: + await asyncio.sleep(early) + + await client.send_audio(frame) + + if args.vad == VADMode.CLIENT.value and client.audio_seconds_sent >= next_turn_end: + clock.log("[client]", f"end of turn at {client.audio_seconds_sent:.2f}s") + await client.force_end_of_utterance() + next_turn_end += args.turn_seconds + + +async def main() -> None: + args = parse_args() + clock = Clock() + client = build_client(args, clock) + + with wave.open(args.audio, "rb") as wav: + if (wav.getframerate(), wav.getnchannels(), wav.getsampwidth()) != (SAMPLE_RATE, 1, BYTES_PER_SAMPLE): + print(f"{args.audio} must be 16 kHz mono 16-bit PCM for the Agent STT service") + return + duration = wav.getnframes() / SAMPLE_RATE + + async with client: + await stream(client, wav, args, clock) + + print(f"\nTranscript: {client.transcript}") + print(f"Streamed {duration:.2f}s of audio in {clock.elapsed:.2f}s") + if clock.segment_lags: + mean = sum(clock.segment_lags) / len(clock.segment_lags) + print(f"Final segment lag behind audio: mean {mean * 1000:.0f}ms, max {max(clock.segment_lags) * 1000:.0f}ms") + + +asyncio.run(main()) From 3a5a9db92c26425902ea51a71c698afdbecafdd9 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 12:59:07 +0100 Subject: [PATCH 08/19] remove agent url, use generic rt url, with added /agent if needed --- examples/agent_stt/README.md | 3 ++- examples/agent_stt/microphone_windows/main.py | 2 +- examples/agent_stt/realtime_file/main.py | 2 +- sdk/agent_stt/PLAN.md | 2 +- sdk/agent_stt/README.md | 4 ++-- .../speechmatics/agent_stt/__init__.py | 2 -- sdk/agent_stt/speechmatics/agent_stt/_client.py | 4 ++-- sdk/agent_stt/speechmatics/agent_stt/_url.py | 8 ++++---- tests/agent_stt/test_client.py | 4 +--- tests/agent_stt/test_url.py | 17 +++++++---------- 10 files changed, 21 insertions(+), 27 deletions(-) diff --git a/examples/agent_stt/README.md b/examples/agent_stt/README.md index 9e007bd8..5118cbe4 100644 --- a/examples/agent_stt/README.md +++ b/examples/agent_stt/README.md @@ -1,7 +1,8 @@ # Agent STT examples Set `SPEECHMATICS_API_KEY` first. To point at a local Voice Agent Service, set -`SPEECHMATICS_AGENT_STT_URL` (for example `ws://localhost:8000/v2/agent`). +`SPEECHMATICS_RT_URL` (for example `ws://localhost:8000/v2`); the `/agent` segment is appended +when it is missing. The service needs 16 kHz raw PCM, so the file examples take a 16 kHz WAV and default to `tests/voice/assets/audio_01_16kHz.wav`. diff --git a/examples/agent_stt/microphone_windows/main.py b/examples/agent_stt/microphone_windows/main.py index 1451f9ec..9f8d6ad0 100644 --- a/examples/agent_stt/microphone_windows/main.py +++ b/examples/agent_stt/microphone_windows/main.py @@ -73,7 +73,7 @@ def build_client(args: argparse.Namespace, console: Console) -> AsyncClient: diarization="speaker" if args.diarization else None, ) - # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_AGENT_STT_URL to point at a local service + # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_RT_URL to point at a local service client = AsyncClient(config=config) @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) diff --git a/examples/agent_stt/realtime_file/main.py b/examples/agent_stt/realtime_file/main.py index 0cb8d030..58b5ac4f 100644 --- a/examples/agent_stt/realtime_file/main.py +++ b/examples/agent_stt/realtime_file/main.py @@ -82,7 +82,7 @@ def build_client(args: argparse.Namespace, clock: Clock) -> AsyncClient: emit_sentences=args.emit_sentences, ) - # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_AGENT_STT_URL to point at a local service + # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_RT_URL to point at a local service client = AsyncClient(config=config) @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index d6e28fbb..ce956654 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -75,7 +75,7 @@ reconnect-free lifecycle, `send_audio`, `transcribe`, `stop_session`, What the subclass adds: -1. URL resolution (`/agent` + profile, `SPEECHMATICS_AGENT_STT_URL` env override). +1. URL resolution (`/agent` + profile, on top of the `SPEECHMATICS_RT_URL` endpoint). 2. `TranscriptionConfig` with `vad_mode`, `vad_config`, `emit_sentences`, and an Agent STT `Model` enum defaulting to `linden-1`. The request goes to the proxy rather than the service websocket directly, and the proxy resolves the Agent STT model name onto the engine's diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index 4ce22fcb..00ca4db5 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -134,8 +134,8 @@ AsyncClient(url="ws://localhost:8000/v2", profile="default") # -> /v2/agent/def AsyncClient(app="pipecat/1.0") # reported as sm-app ``` -Resolution order: the `url` argument, `SPEECHMATICS_AGENT_STT_URL`, `SPEECHMATICS_RT_URL`, then -the EU endpoint. The `/agent` segment is appended when it is missing. +Resolution order: the `url` argument, `SPEECHMATICS_RT_URL`, then the EU endpoint. The `/agent` +segment is appended when it is missing. ## Audio diff --git a/sdk/agent_stt/speechmatics/agent_stt/__init__.py b/sdk/agent_stt/speechmatics/agent_stt/__init__.py index c27a08af..effd244a 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/__init__.py +++ b/sdk/agent_stt/speechmatics/agent_stt/__init__.py @@ -55,11 +55,9 @@ from ._models import VADConfig from ._models import VADMode from ._transcript import Transcript -from ._url import DEFAULT_AGENT_STT_URL from ._url import resolve_url __all__ = [ - "DEFAULT_AGENT_STT_URL", "DEFAULT_CHUNK_SIZE", "DEFAULT_MODEL", "DEFAULT_SAMPLE_RATE", diff --git a/sdk/agent_stt/speechmatics/agent_stt/_client.py b/sdk/agent_stt/speechmatics/agent_stt/_client.py index 6e4f1dc5..82e93d66 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_client.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_client.py @@ -52,8 +52,8 @@ class AsyncClient(RTAsyncClient): auth: Authentication instance. Defaults to `StaticKeyAuth` built from `api_key` or the `SPEECHMATICS_API_KEY` environment variable. api_key: Speechmatics API key, used when `auth` is not given. - url: WebSocket endpoint. Defaults to `SPEECHMATICS_AGENT_STT_URL`, then - `SPEECHMATICS_RT_URL`, then the EU endpoint. An `/agent` segment is appended if absent. + url: WebSocket endpoint. Defaults to `SPEECHMATICS_RT_URL`, then the EU endpoint. + An `/agent` segment is appended if absent. profile: Service profile, appended to the endpoint path. app: Application name reported to the service as `sm-app`. config: Transcription config for the session, normally an diff --git a/sdk/agent_stt/speechmatics/agent_stt/_url.py b/sdk/agent_stt/speechmatics/agent_stt/_url.py index af6550d7..764af86b 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_url.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_url.py @@ -9,7 +9,7 @@ from ._version import get_version -DEFAULT_AGENT_STT_URL = "wss://eu2.rt.speechmatics.com/v2/agent" +DEFAULT_RT_URL = "wss://eu2.rt.speechmatics.com/v2" AGENT_PATH_SEGMENT = "agent" @@ -25,8 +25,8 @@ def resolve_url( followed by a service profile name. Args: - url: Explicit endpoint. Falls back to the `SPEECHMATICS_AGENT_STT_URL` environment - variable, then `SPEECHMATICS_RT_URL`, then the EU endpoint. + url: Explicit endpoint. Falls back to the `SPEECHMATICS_RT_URL` environment + variable, then the EU endpoint. profile: Optional service profile, appended as a final path segment. app: Optional application name reported to the service as `sm-app`. @@ -39,7 +39,7 @@ def resolve_url( >>> resolve_url("wss://host/v2", profile="default", app="pipecat/1.0") 'wss://host/v2/agent/default?sm-app=pipecat%2F1.0' """ - base = url or os.getenv("SPEECHMATICS_AGENT_STT_URL") or os.getenv("SPEECHMATICS_RT_URL") or DEFAULT_AGENT_STT_URL + base = url or os.getenv("SPEECHMATICS_RT_URL") or DEFAULT_RT_URL parsed = urlparse(base) return urlunparse( parsed._replace( diff --git a/tests/agent_stt/test_client.py b/tests/agent_stt/test_client.py index 73967f0c..c7fe8af6 100644 --- a/tests/agent_stt/test_client.py +++ b/tests/agent_stt/test_client.py @@ -36,7 +36,6 @@ def audio(self): @pytest.fixture def client(monkeypatch): - monkeypatch.delenv("SPEECHMATICS_AGENT_STT_URL", raising=False) monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) return AsyncClient(api_key=API_KEY) @@ -75,7 +74,6 @@ async def test_endpoint_is_the_agent_path(client): @pytest.mark.asyncio async def test_profile_and_app_reach_the_url(monkeypatch): - monkeypatch.delenv("SPEECHMATICS_AGENT_STT_URL", raising=False) monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) client = AsyncClient(api_key=API_KEY, profile="default", app="pipecat/1.0") assert "/v2/agent/default" in client._transport._url @@ -159,7 +157,7 @@ async def test_every_message_is_recorded(client): @pytest.mark.asyncio async def test_event_recording_can_be_disabled(monkeypatch): - monkeypatch.delenv("SPEECHMATICS_AGENT_STT_URL", raising=False) + monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) client = AsyncClient(api_key=API_KEY, record_events=False) start_session(client) assert client.events == [] diff --git a/tests/agent_stt/test_url.py b/tests/agent_stt/test_url.py index d168249e..d258df49 100644 --- a/tests/agent_stt/test_url.py +++ b/tests/agent_stt/test_url.py @@ -3,13 +3,11 @@ import pytest -from speechmatics.agent_stt import DEFAULT_AGENT_STT_URL from speechmatics.agent_stt import resolve_url @pytest.fixture(autouse=True) def _clear_env(monkeypatch): - monkeypatch.delenv("SPEECHMATICS_AGENT_STT_URL", raising=False) monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) @@ -18,7 +16,9 @@ def _path(url): def test_default_url(): - assert _path(resolve_url()) == _path(DEFAULT_AGENT_STT_URL) + url = urlparse(resolve_url()) + assert url.hostname == "eu2.rt.speechmatics.com" + assert url.path == "/v2/agent" def test_agent_segment_appended_to_rt_url(monkeypatch): @@ -26,16 +26,13 @@ def test_agent_segment_appended_to_rt_url(monkeypatch): assert _path(resolve_url()) == "/v2/agent" -def test_agent_stt_env_takes_precedence(monkeypatch): - monkeypatch.setenv("SPEECHMATICS_RT_URL", "wss://rt.example.com/v2") - monkeypatch.setenv("SPEECHMATICS_AGENT_STT_URL", "wss://agent.example.com/v2/agent") - url = resolve_url() - assert urlparse(url).hostname == "agent.example.com" - assert _path(url) == "/v2/agent" +def test_agent_segment_not_duplicated_on_rt_url(monkeypatch): + monkeypatch.setenv("SPEECHMATICS_RT_URL", "wss://example.com/v2/agent") + assert _path(resolve_url()) == "/v2/agent" def test_explicit_url_wins_over_env(monkeypatch): - monkeypatch.setenv("SPEECHMATICS_AGENT_STT_URL", "wss://agent.example.com/v2/agent") + monkeypatch.setenv("SPEECHMATICS_RT_URL", "wss://rt.example.com/v2") assert urlparse(resolve_url("wss://custom.example.com/v2")).hostname == "custom.example.com" From e4554f7639e980b5986cdee1d9d39029f0be1ba3 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 13:14:46 +0100 Subject: [PATCH 09/19] remove unused msgs from/to the service --- sdk/agent_stt/PLAN.md | 5 +++-- sdk/agent_stt/README.md | 10 +++++++--- sdk/agent_stt/speechmatics/agent_stt/_models.py | 13 ------------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index ce956654..1937d277 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -45,8 +45,9 @@ Server -> client, RT passthrough: `RecognitionStarted`, `AudioAdded`, `AddTransc `EndOfUtterance` is consumed by the service and never reaches the client. Note: the service still forwards `AddTranscript`/`AddPartialTranscript` verbatim today. The SDK -accumulates its transcript from **segments only**, but the transcript messages remain available -via handlers and the event log, so nothing is lost if a future profile mutes them. +accumulates its transcript from **segments only** and models neither those nor audio events; +they still reach the event log and any handler registered under their name, so nothing is lost +if a future profile mutes them. Constraints the service imposes: `audio_format.type` must be `raw`, sample rate `16000` (Silero), encoding `pcm_s16le` or `pcm_f32le` (no mulaw). The SDK defaults to exactly that. diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index 00ca4db5..b9791342 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -92,9 +92,13 @@ Emitted by the service: | `SpeechStarted` / `SpeechEnded` | `metadata.start_time` / `metadata.end_time` (service VAD) | | `StartOfTurn` / `EndOfTurn` | `metadata.start_time` / `metadata.end_time` (service turn detection) | -Passed through from the RT engine: `RecognitionStarted`, `AudioAdded`, `AddTranscript`, -`AddPartialTranscript`, `EndOfTranscript`, `SpeakersResult`, `Info`, `Warning`, `Error`. -`EndOfUtterance` is consumed by the service and not forwarded. +Passed through from the RT engine: `RecognitionStarted`, `AudioAdded`, `EndOfTranscript`, +`SpeakersResult`, `Info`, `Warning`, `Error`. `EndOfUtterance` is consumed by the service and +not forwarded. + +Anything else the engine sends - the word-level `AddTranscript`/`AddPartialTranscript`, audio +events - is not modelled here, but still reaches `client.events` and any handler registered +under its name. ## Configuration diff --git a/sdk/agent_stt/speechmatics/agent_stt/_models.py b/sdk/agent_stt/speechmatics/agent_stt/_models.py index edd29322..76a4f100 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_models.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_models.py @@ -36,22 +36,17 @@ class ClientMessageType(str, Enum): """ Message types that can be sent from client to the Agent STT service. - The Agent STT service adds no client messages of its own; these are the RT messages - it forwards downstream. - Attributes: START_RECOGNITION: Starts the session, carrying the audio and transcription config. END_OF_STREAM: Signals that no more audio will be sent. FORCE_END_OF_UTTERANCE: Closes the current turn immediately, flushing the buffered segment. Sent by clients that run their own VAD. - SET_RECOGNITION_CONFIG: Updates the transcription config mid-session. GET_SPEAKERS: Requests the session's speaker data. """ START_RECOGNITION = "StartRecognition" END_OF_STREAM = "EndOfStream" FORCE_END_OF_UTTERANCE = "ForceEndOfUtterance" - SET_RECOGNITION_CONFIG = "SetRecognitionConfig" GET_SPEAKERS = "GetSpeakers" @@ -71,14 +66,10 @@ class ServerMessageType(str, Enum): SPEECH_ENDED: The service's VAD detected speech offset. START_OF_TURN: The service's turn detection opened a turn. END_OF_TURN: The service's turn detection closed a turn. - ADD_TRANSCRIPT: Word-level final transcript, passed through from the RT engine. - ADD_PARTIAL_TRANSCRIPT: Word-level partial transcript, passed through from the RT engine. END_OF_UTTERANCE: Consumed by the service for segmentation and not forwarded; listed so handlers stay valid against a direct RT endpoint. END_OF_TRANSCRIPT: The service has finished sending messages. SPEAKERS_RESULT: Response to GetSpeakers. - AUDIO_EVENT_STARTED: Start of a detected audio event. - AUDIO_EVENT_ENDED: End of a detected audio event. INFO: Informational message. WARNING: Warning; the session continues, possibly with adjusted config. ERROR: Error; the session is over. @@ -97,13 +88,9 @@ class ServerMessageType(str, Enum): SPEECH_ENDED = "SpeechEnded" START_OF_TURN = "StartOfTurn" END_OF_TURN = "EndOfTurn" - ADD_TRANSCRIPT = "AddTranscript" - ADD_PARTIAL_TRANSCRIPT = "AddPartialTranscript" END_OF_UTTERANCE = "EndOfUtterance" END_OF_TRANSCRIPT = "EndOfTranscript" SPEAKERS_RESULT = "SpeakersResult" - AUDIO_EVENT_STARTED = "AudioEventStarted" - AUDIO_EVENT_ENDED = "AudioEventEnded" INFO = "Info" WARNING = "Warning" ERROR = "Error" From 591d1bae9aa6733b070b610888b036a3e0af52a4 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 13:20:05 +0100 Subject: [PATCH 10/19] refactor vadmode to better naming and structure --- examples/agent_stt/README.md | 2 +- examples/agent_stt/client_vad/main.py | 6 +-- examples/agent_stt/realtime_file/main.py | 18 ++++----- sdk/agent_stt/PLAN.md | 25 ++++++------ sdk/agent_stt/README.md | 10 ++--- .../speechmatics/agent_stt/__init__.py | 4 +- .../speechmatics/agent_stt/_client.py | 11 +++--- .../speechmatics/agent_stt/_models.py | 39 +++++++++++-------- tests/agent_stt/test_client.py | 6 +-- tests/agent_stt/test_config.py | 12 +++--- 10 files changed, 70 insertions(+), 63 deletions(-) diff --git a/examples/agent_stt/README.md b/examples/agent_stt/README.md index 5118cbe4..1069db35 100644 --- a/examples/agent_stt/README.md +++ b/examples/agent_stt/README.md @@ -11,7 +11,7 @@ The service needs 16 kHz raw PCM, so the file examples take a 16 kHz WAV and def | --- | --- | | [file/main.py](file/main.py) | File transcription with the service's VAD; segments, turn events, transcript at the end | | [realtime_file/main.py](realtime_file/main.py) | The same file paced at wall-clock speed, with the lag of each message behind the audio | -| [client_vad/main.py](client_vad/main.py) | The client owns turn boundaries and calls `finalize()`, as Pipecat and LiveKit do | +| [client_vad/main.py](client_vad/main.py) | `TurnDetectionMode.EXTERNAL`: the application owns turn boundaries and calls `finalize()`, as Pipecat and LiveKit do | | [microphone/main.py](microphone/main.py) | Live microphone with diarization and speaker-labelled transcript (needs `pyaudio`) | | [microphone_windows/main.py](microphone_windows/main.py) | The same, set up for Windows: device selection, in-place partials, Ctrl+C shutdown | diff --git a/examples/agent_stt/client_vad/main.py b/examples/agent_stt/client_vad/main.py index c6cb0c87..f10a6abb 100644 --- a/examples/agent_stt/client_vad/main.py +++ b/examples/agent_stt/client_vad/main.py @@ -1,4 +1,4 @@ -"""Drive turn boundaries from the client instead of the service. +"""Drive turn boundaries from the application instead of the service. This is the mode host frameworks use: Pipecat, LiveKit and others already run a VAD, so the service's VAD is switched off and each turn is closed by calling `finalize()`, which sends @@ -16,7 +16,7 @@ from speechmatics.agent_stt import AsyncClient from speechmatics.agent_stt import ServerMessageType from speechmatics.agent_stt import TranscriptionConfig -from speechmatics.agent_stt import VADMode +from speechmatics.agent_stt import TurnDetectionMode DEFAULT_AUDIO_FILE = "./tests/voice/assets/audio_01_16kHz.wav" CHUNK_SIZE = 1024 @@ -24,7 +24,7 @@ async def main(path: str) -> None: - config = TranscriptionConfig(language="en", enable_partials=True, vad_mode=VADMode.CLIENT) + config = TranscriptionConfig(language="en", enable_partials=True, turn_detection_mode=TurnDetectionMode.EXTERNAL) # Uses SPEECHMATICS_API_KEY from the environment async with AsyncClient(config=config) as client: diff --git a/examples/agent_stt/realtime_file/main.py b/examples/agent_stt/realtime_file/main.py index 58b5ac4f..24250116 100644 --- a/examples/agent_stt/realtime_file/main.py +++ b/examples/agent_stt/realtime_file/main.py @@ -17,8 +17,8 @@ from speechmatics.agent_stt import AsyncClient from speechmatics.agent_stt import ServerMessageType from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import TurnDetectionMode from speechmatics.agent_stt import VADConfig -from speechmatics.agent_stt import VADMode DEFAULT_AUDIO_FILE = "./tests/voice/assets/audio_01_16kHz.wav" SAMPLE_RATE = 16000 @@ -34,14 +34,14 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--language", default="en") parser.add_argument("--chunk-ms", type=float, default=20.0, help="audio frame size in milliseconds") parser.add_argument( - "--vad", - choices=[VADMode.SERVER.value, VADMode.CLIENT.value], - default=VADMode.SERVER.value, - help="who closes turns: the service, or this script calling finalize()", + "--turn-detection", + choices=[TurnDetectionMode.VAD.value, TurnDetectionMode.EXTERNAL.value], + default=TurnDetectionMode.VAD.value, + help="which mechanism closes turns: the service's VAD, or this script calling finalize()", ) parser.add_argument("--vad-window", type=float, help="silence in seconds before the service closes a turn") parser.add_argument( - "--turn-seconds", type=float, default=5.0, help="fake client-VAD turn length, --vad client only" + "--turn-seconds", type=float, default=5.0, help="fake turn length, --turn-detection external only" ) parser.add_argument("--no-partials", action="store_true") parser.add_argument("--emit-sentences", action="store_true", help="close a segment on every sentence boundary") @@ -77,7 +77,7 @@ def build_client(args: argparse.Namespace, clock: Clock) -> AsyncClient: config = TranscriptionConfig( language=args.language, enable_partials=not args.no_partials, - vad_mode=VADMode(args.vad), + turn_detection_mode=TurnDetectionMode(args.turn_detection), vad_config=VADConfig(window=args.vad_window) if args.vad_window is not None else None, emit_sentences=args.emit_sentences, ) @@ -139,8 +139,8 @@ async def stream(client: AsyncClient, wav: wave.Wave_read, args: argparse.Namesp await client.send_audio(frame) - if args.vad == VADMode.CLIENT.value and client.audio_seconds_sent >= next_turn_end: - clock.log("[client]", f"end of turn at {client.audio_seconds_sent:.2f}s") + if args.turn_detection == TurnDetectionMode.EXTERNAL.value and client.audio_seconds_sent >= next_turn_end: + clock.log("[external]", f"end of turn at {client.audio_seconds_sent:.2f}s") await client.force_end_of_utterance() next_turn_end += args.turn_seconds diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index 1937d277..7da03c38 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -11,8 +11,8 @@ The SDK does **no VAD and no turn detection of its own**. Boundaries come from o | Mode | Who decides the turn boundary | Wire behaviour | | --- | --- | --- | -| `VADMode.SERVER` | the service's own (Silero) VAD | `transcription_config.vad_config.enabled = true`; service emits `SpeechStarted`/`SpeechEnded`/`StartOfTurn`/`EndOfTurn` and forces end-of-utterance internally | -| `VADMode.CLIENT` | the host framework (Pipecat, LiveKit, ...) | `transcription_config.vad_config.enabled = false`; the host calls `client.force_end_of_utterance()` on its own VAD's stop-speaking event | +| `TurnDetectionMode.VAD` | the service's own (Silero) VAD | `transcription_config.vad_config.enabled = true`; service emits `SpeechStarted`/`SpeechEnded`/`StartOfTurn`/`EndOfTurn` and forces end-of-utterance internally | +| `TurnDetectionMode.EXTERNAL` | the host framework (Pipecat, LiveKit, ...) | `transcription_config.vad_config.enabled = false`; the host calls `client.force_end_of_utterance()` on its own VAD's stop-speaking event | This is the whole reason the SDK exists as a separate package: the `voice` SDK bundles VAD + smart-turn models in-process, which duplicates what Pipecat/LiveKit already run and what the @@ -77,7 +77,7 @@ reconnect-free lifecycle, `send_audio`, `transcribe`, `stop_session`, What the subclass adds: 1. URL resolution (`/agent` + profile, on top of the `SPEECHMATICS_RT_URL` endpoint). -2. `TranscriptionConfig` with `vad_mode`, `vad_config`, `emit_sentences`, and an Agent STT +2. `TranscriptionConfig` with `turn_detection_mode`, `vad_config`, `emit_sentences`, and an Agent STT `Model` enum defaulting to `linden-1`. The request goes to the proxy rather than the service websocket directly, and the proxy resolves the Agent STT model name onto the engine's operating point, so the transcriber never sees a name it has no notion of. The deprecated @@ -124,9 +124,10 @@ proxy. touched by this change. The migration: - swap the import block to `speechmatics.agent_stt` -- collapse `TurnDetectionMode` onto the two modes that exist: - - `EXTERNAL` -> `VADMode.CLIENT`, with `finalize()` on `VADUserStoppedSpeakingFrame` - - `ADAPTIVE` -> `VADMode.SERVER`, with `StartOfTurn`/`EndOfTurn` driving +- collapse Pipecat's own `TurnDetectionMode` onto the two modes that exist: + - its `EXTERNAL` -> `TurnDetectionMode.EXTERNAL`, with `finalize()` on + `VADUserStoppedSpeakingFrame` + - its `ADAPTIVE` -> `TurnDetectionMode.VAD`, with `StartOfTurn`/`EndOfTurn` driving `ProposedUserStartedSpeakingFrame`/`ProposedUserStoppedSpeakingFrame` - `FIXED` and `SMART_TURN` -> removed (see below) - `AddPartialSegment` -> `InterimTranscriptionFrame`, `AddSegment` -> `TranscriptionFrame` @@ -136,7 +137,7 @@ touched by this change. The migration: `SpeechmaticsSTTSettings` and `InputParams`, along with the passthrough at `stt.py:788` and `stt.py:1252` - engine silence-based end of utterance is off for this service, so both are no-ops -- `_enable_vad` (`stt.py:530`) becomes `vad_mode is VADMode.CLIENT`, since that is now the only +- `_enable_vad` (`stt.py:530`) becomes `turn_detection_mode is TurnDetectionMode.EXTERNAL`, since that is now the only mode where Pipecat's own VAD drives the boundary - otherwise keep `SpeechmaticsSTTSettings` as the public surface so user code doesn't change @@ -149,14 +150,14 @@ goes away rather than being aliased to another mode. `SMART_TURN` goes for the same reason: the service has no smart-turn endpoint yet (planned for a later release), and this SDK loads no models, so the mode cannot be honoured as specified. It -comes back when the service does, as a `VADMode.SERVER` variant. +comes back when the service does, as a `TurnDetectionMode.SMART_TURN` member alongside `TurnDetectionMode.VAD`. Removing it costs Pipecat users nothing, because Pipecat's own turn analyzer still works: any host-side endpointing - VAD, ML turn model, push-to-talk - reaches the service the same way, -through `finalize()`. `VADMode.CLIENT` is agnostic about what produced the signal. +through `finalize()`. `TurnDetectionMode.EXTERNAL` is agnostic about what produced the signal. -Turns therefore end in exactly two ways, which is what `VADMode` models: the service's VAD, or -the client calling `finalize()`. +Turns therefore end in exactly two ways, which is what `TurnDetectionMode` models: the service's VAD, or +the application calling `finalize()`. Open questions to settle before starting milestone 2: @@ -165,6 +166,6 @@ Open questions to settle before starting milestone 2: still works, since `speaker_diarization_config.speakers` passes straight through. - ~~engine-silence endpointing / `FIXED` mode~~ - removed, see above. - ~~`SMART_TURN`~~ - removed until the service implements it. Host-side turn models keep working - through `VADMode.CLIENT`. + through `TurnDetectionMode.EXTERNAL`. Both removals are user-visible, so they need a changelog entry when the Pipecat change lands. diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index b9791342..5c124006 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -37,19 +37,19 @@ asyncio.run(main()) The service needs a boundary to close a segment on. Pick where it comes from: ```python -from speechmatics.agent_stt import TranscriptionConfig, VADConfig, VADMode +from speechmatics.agent_stt import TranscriptionConfig, TurnDetectionMode, VADConfig # The service's VAD (default). It emits SpeechStarted/SpeechEnded and StartOfTurn/EndOfTurn. config = TranscriptionConfig( - vad_mode=VADMode.SERVER, + turn_detection_mode=TurnDetectionMode.VAD, vad_config=VADConfig(window=0.2, onset_threshold=0.5, offset_threshold=0.35), ) # Your endpointing - Pipecat, LiveKit, or your own. The service's VAD stays off. -config = TranscriptionConfig(vad_mode=VADMode.CLIENT) +config = TranscriptionConfig(turn_detection_mode=TurnDetectionMode.EXTERNAL) ``` -With `VADMode.CLIENT`, close each turn when your side decides speech has ended: +With `TurnDetectionMode.EXTERNAL`, close each turn when your side decides speech has ended: ```python client.finalize() # from a sync callback @@ -106,7 +106,7 @@ under its name. | Field | Meaning | | --- | --- | -| `vad_mode` | `VADMode.SERVER` (default) or `VADMode.CLIENT` | +| `turn_detection_mode` | `TurnDetectionMode.VAD` (default) or `TurnDetectionMode.EXTERNAL` | | `vad_config` | `window`, `onset_threshold`, `offset_threshold` for the service's VAD | | `emit_sentences` | Close a segment on every sentence boundary, not only at the turn boundary | diff --git a/sdk/agent_stt/speechmatics/agent_stt/__init__.py b/sdk/agent_stt/speechmatics/agent_stt/__init__.py index effd244a..4fa2514c 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/__init__.py +++ b/sdk/agent_stt/speechmatics/agent_stt/__init__.py @@ -52,8 +52,8 @@ from ._models import SessionInfo from ._models import TimedEvent from ._models import TranscriptionConfig +from ._models import TurnDetectionMode from ._models import VADConfig -from ._models import VADMode from ._transcript import Transcript from ._url import resolve_url @@ -79,9 +79,9 @@ "SpeakerIdentifier", "TranscriptionConfig", "TranslationConfig", + "TurnDetectionMode", "Model", "VADConfig", - "VADMode", # Auth "AuthBase", "JWTAuth", diff --git a/sdk/agent_stt/speechmatics/agent_stt/_client.py b/sdk/agent_stt/speechmatics/agent_stt/_client.py index 82e93d66..501f74a7 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_client.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_client.py @@ -46,7 +46,8 @@ class AsyncClient(RTAsyncClient): Extends the RT client to talk to the Agent STT endpoint (`/agent`), which works in segments rather than word groups and reports speech and turn events. The client runs no VAD and no turn detection of its own: either the service's VAD closes turns - (`VADMode.SERVER`) or the application's does, by calling `finalize()` (`VADMode.CLIENT`). + (`TurnDetectionMode.VAD`) or the application's does, by calling `finalize()` + (`TurnDetectionMode.EXTERNAL`). Args: auth: Authentication instance. Defaults to `StaticKeyAuth` built from `api_key` or the @@ -72,8 +73,8 @@ class AsyncClient(RTAsyncClient): ... await client.send_audio(frame) >>> print(client.transcript) - Client VAD (Pipecat, LiveKit): - >>> config = TranscriptionConfig(vad_mode=VADMode.CLIENT) + External endpointing (Pipecat, LiveKit): + >>> config = TranscriptionConfig(turn_detection_mode=TurnDetectionMode.EXTERNAL) >>> client = AsyncClient(api_key="your-key", config=config) >>> await client.connect() >>> await client.send_audio(frame) @@ -273,8 +274,8 @@ def finalize(self, *, timestamp: Optional[float] | object = _UNSET) -> None: rather than wherever the send happens to land. The flushed segment arrives as a normal AddSegment message. - Use this when the application brings its own VAD (`VADMode.CLIENT`); with - `VADMode.SERVER` the service closes turns itself. + Use this when the application brings its own VAD (`TurnDetectionMode.EXTERNAL`); with + `TurnDetectionMode.VAD` the service closes turns itself. Args: timestamp: Audio timestamp in seconds for the end of the utterance. Defaults to diff --git a/sdk/agent_stt/speechmatics/agent_stt/_models.py b/sdk/agent_stt/speechmatics/agent_stt/_models.py index 76a4f100..4ba8516d 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_models.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_models.py @@ -106,27 +106,31 @@ class ServerMessageType(str, Enum): ) -class VADMode(str, Enum): +class TurnDetectionMode(str, Enum): """ - Where turn boundaries come from. This SDK never detects them itself. + Which mechanism closes a turn. This SDK never detects turn boundaries itself. + + The mode names the mechanism rather than the side it runs on, so a second service-side + mechanism - smart turn, once the service implements it - joins as another member without + changing what the existing ones mean. Attributes: - SERVER: The service runs its own VAD and turn detection, emitting SpeechStarted, - SpeechEnded, StartOfTurn and EndOfTurn, and closing segments itself. - CLIENT: The application closes each turn by calling `finalize()`, which sends + VAD: The service's own VAD closes turns, emitting SpeechStarted, SpeechEnded, + StartOfTurn and EndOfTurn, and closing segments itself. + EXTERNAL: The application closes each turn by calling `finalize()`, which sends ForceEndOfUtterance with an audio timestamp. Whatever produced that signal is up to the application - a VAD, a turn model, or a push-to-talk button - so a host framework's own endpointing (Pipecat, LiveKit, ...) works unchanged. """ - SERVER = "server" - CLIENT = "client" + VAD = "vad" + EXTERNAL = "external" @dataclass class VADConfig: """ - Tuning for the service's VAD. Only applied when `VADMode.SERVER` is in use. + Tuning for the service's VAD. Only applied when `TurnDetectionMode.VAD` is in use. Attributes: window: Silence in seconds before the service closes the turn. @@ -165,8 +169,8 @@ class TranscriptionConfig(RTTranscriptionConfig): `emit_sentences`). See `speechmatics.rt.TranscriptionConfig` for the inherited fields. Attributes: - vad_mode: Whether the service or the client decides turn boundaries. - vad_config: Tuning for the service's VAD, used when `vad_mode` is `SERVER`. + turn_detection_mode: Which mechanism closes a turn: the service's VAD, or the application. + vad_config: Tuning for the service's VAD, used when `turn_detection_mode` is `VAD`. emit_sentences: Close a segment on every sentence boundary, not just at the turn boundary. additional_vocab: Words to bias the engine towards, as `AdditionalVocabEntry` objects @@ -179,13 +183,13 @@ class TranscriptionConfig(RTTranscriptionConfig): Service VAD, sentence-level segments: >>> config = TranscriptionConfig(language="en", emit_sentences=True) - Client VAD (Pipecat, LiveKit): - >>> config = TranscriptionConfig(language="en", vad_mode=VADMode.CLIENT) + External endpointing (Pipecat, LiveKit): + >>> config = TranscriptionConfig(language="en", turn_detection_mode=TurnDetectionMode.EXTERNAL) """ model: Model = _UNSET additional_vocab: Optional[list[Union[AdditionalVocabEntry, dict[str, Any]]]] = None - vad_mode: VADMode = VADMode.SERVER + turn_detection_mode: TurnDetectionMode = TurnDetectionMode.VAD vad_config: VADConfig = field(default_factory=VADConfig) emit_sentences: Optional[bool] = None @@ -202,15 +206,16 @@ def to_dict(self) -> dict[str, Any]: Convert to the wire form of `StartRecognition.transcription_config`. Returns: - The config as a dict, excluding None values, with `vad_config.enabled` derived - from `vad_mode` and `vad_mode` itself dropped. + The config as a dict, excluding None values. `vad_config.enabled` is derived + from `turn_detection_mode` - on unless the application closes turns itself - and + the mode itself is dropped. """ result = super().to_dict() if self.model is _UNSET: result.pop("model", None) - result.pop("vad_mode", None) + result.pop("turn_detection_mode", None) vad_config = result.pop("vad_config", None) or {} - vad_config["enabled"] = self.vad_mode is VADMode.SERVER + vad_config["enabled"] = self.turn_detection_mode is not TurnDetectionMode.EXTERNAL result["vad_config"] = vad_config return result diff --git a/tests/agent_stt/test_client.py b/tests/agent_stt/test_client.py index c7fe8af6..8fcb5cd7 100644 --- a/tests/agent_stt/test_client.py +++ b/tests/agent_stt/test_client.py @@ -7,7 +7,7 @@ from speechmatics.agent_stt import ClientMessageType from speechmatics.agent_stt import ServerMessageType from speechmatics.agent_stt import TranscriptionConfig -from speechmatics.agent_stt import VADMode +from speechmatics.agent_stt import TurnDetectionMode API_KEY = "test-key" @@ -228,10 +228,10 @@ async def test_finalize_latency_measured_against_the_flushed_segment(client): @pytest.mark.asyncio -async def test_client_vad_mode_config_reaches_start_recognition(client): +async def test_external_turn_detection_reaches_start_recognition(client): transport = StubTransport() client._transport = transport - client._config = TranscriptionConfig(language="en", vad_mode=VADMode.CLIENT) + client._config = TranscriptionConfig(language="en", turn_detection_mode=TurnDetectionMode.EXTERNAL) await client.send_message( { diff --git a/tests/agent_stt/test_config.py b/tests/agent_stt/test_config.py index 9939794a..99ecf756 100644 --- a/tests/agent_stt/test_config.py +++ b/tests/agent_stt/test_config.py @@ -2,21 +2,21 @@ from speechmatics.agent_stt import Model from speechmatics.agent_stt import TranscriptionConfig +from speechmatics.agent_stt import TurnDetectionMode from speechmatics.agent_stt import VADConfig -from speechmatics.agent_stt import VADMode -def test_server_vad_is_the_default(): +def test_service_vad_is_the_default(): assert TranscriptionConfig().to_dict()["vad_config"] == {"enabled": True} -def test_client_vad_disables_service_vad(): - config = TranscriptionConfig(vad_mode=VADMode.CLIENT) +def test_external_mode_disables_service_vad(): + config = TranscriptionConfig(turn_detection_mode=TurnDetectionMode.EXTERNAL) assert config.to_dict()["vad_config"] == {"enabled": False} -def test_vad_mode_is_not_sent(): - assert "vad_mode" not in TranscriptionConfig().to_dict() +def test_turn_detection_mode_is_not_sent(): + assert "turn_detection_mode" not in TranscriptionConfig().to_dict() def test_vad_tuning_passed_through(): From 907518587caf03950aff27c95d970b1af5aa88ff Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 13:26:04 +0100 Subject: [PATCH 11/19] remove profile mentions --- sdk/agent_stt/PLAN.md | 15 +++++++------ sdk/agent_stt/README.md | 9 ++++---- .../speechmatics/agent_stt/_client.py | 4 +--- sdk/agent_stt/speechmatics/agent_stt/_url.py | 21 ++++++------------- tests/agent_stt/test_client.py | 6 +++--- tests/agent_stt/test_url.py | 12 ----------- 6 files changed, 21 insertions(+), 46 deletions(-) diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index 7da03c38..cdb902f7 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -20,8 +20,7 @@ service now does server-side. ## Protocol delta vs the RT SDK -Endpoint: RT URL + `/agent`, optionally + `/{profile}` -(`wss://eu2.rt.speechmatics.com/v2/agent`, service route is `/v2/agent/{profile:path}`). +Endpoint: RT URL + `/agent` (`wss://eu2.rt.speechmatics.com/v2/agent`). Client -> server: unchanged (`StartRecognition`, binary audio, `EndOfStream`, `ForceEndOfUtterance`). No new client messages. @@ -38,7 +37,7 @@ Server -> client, new messages (`voice_agent_api/_service_messages.py`): - `AddPartialSegment` - same shape, interim - `SpeechStarted` / `SpeechEnded` - `{metadata: {start_time|end_time}}` (VAD) - `StartOfTurn` / `EndOfTurn` - `{metadata: {start_time|end_time}}` (turn detection) -- `Warning` - RT shape, also emitted by the service for profile/lock adjustments +- `Warning` - RT shape, also emitted by the service for config adjustments Server -> client, RT passthrough: `RecognitionStarted`, `AudioAdded`, `AddTranscript`, `AddPartialTranscript`, `EndOfTranscript`, `Info`, `Warning`, `Error`, audio events. @@ -47,7 +46,7 @@ Server -> client, RT passthrough: `RecognitionStarted`, `AudioAdded`, `AddTransc Note: the service still forwards `AddTranscript`/`AddPartialTranscript` verbatim today. The SDK accumulates its transcript from **segments only** and models neither those nor audio events; they still reach the event log and any handler registered under their name, so nothing is lost -if a future profile mutes them. +if the service stops sending them. Constraints the service imposes: `audio_format.type` must be `raw`, sample rate `16000` (Silero), encoding `pcm_s16le` or `pcm_f32le` (no mulaw). The SDK defaults to exactly that. @@ -65,7 +64,7 @@ sdk/agent_stt/ _models.py message enums, TranscriptionConfig, VADConfig, Segment, TimedEvent _transcript.py Transcript - final segments, live partial, timeline, raw event log _transport.py AgentTransport - stamps sm-sdk=python-agent-stt-sdk-vX - _url.py URL/profile resolution + _url.py URL resolution tests/agent_stt/ offline unit tests (no API key needed) examples/agent_stt/ server-VAD and client-VAD (BYO) examples ``` @@ -76,7 +75,7 @@ reconnect-free lifecycle, `send_audio`, `transcribe`, `stop_session`, What the subclass adds: -1. URL resolution (`/agent` + profile, on top of the `SPEECHMATICS_RT_URL` endpoint). +1. URL resolution (`/agent` on top of the `SPEECHMATICS_RT_URL` endpoint). 2. `TranscriptionConfig` with `turn_detection_mode`, `vad_config`, `emit_sentences`, and an Agent STT `Model` enum defaulting to `linden-1`. The request goes to the proxy rather than the service websocket directly, and the proxy resolves the Agent STT model name onto the engine's @@ -113,7 +112,7 @@ The StartRecognition the service forwarded downstream confirmed that `vad_config `emit_sentences` are stripped before it reaches the RT engine. That run predates the `linden-1` default, so it sent no `model` at all and the message carried -only the profile's locked `operating_point`. Model resolution happens in the proxy ahead of the +only the service's locked `operating_point`. Model resolution happens in the proxy ahead of the service, which this stub setup does not exercise, so it still needs verifying against the real proxy. @@ -143,7 +142,7 @@ touched by this change. The migration: ### `FIXED` and `SMART_TURN` are removed -`end_of_utterance_silence_trigger` is off for this service: the default profile pins it to `0.0`, +`end_of_utterance_silence_trigger` is off for this service: the service pins it to `0.0`, the service consumes `EndOfUtterance` rather than forwarding it, and a non-forced end of utterance does not close a segment. So there is nothing for `TurnDetectionMode.FIXED` to mean here and it goes away rather than being aliased to another mode. diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index 5c124006..1eb66881 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -129,13 +129,12 @@ turn ends either because the service's VAD said so, or because you called `final ## Endpoint -The Agent STT endpoint is the RT endpoint plus `/agent`, optionally followed by a service -profile: +The Agent STT endpoint is the RT endpoint plus `/agent`: ```python -AsyncClient(url="wss://eu2.rt.speechmatics.com/v2") # -> /v2/agent -AsyncClient(url="ws://localhost:8000/v2", profile="default") # -> /v2/agent/default -AsyncClient(app="pipecat/1.0") # reported as sm-app +AsyncClient(url="wss://eu2.rt.speechmatics.com/v2") # -> /v2/agent +AsyncClient(url="ws://localhost:8000/v2") # -> /v2/agent +AsyncClient(app="pipecat/1.0") # reported as sm-app ``` Resolution order: the `url` argument, `SPEECHMATICS_RT_URL`, then the EU endpoint. The `/agent` diff --git a/sdk/agent_stt/speechmatics/agent_stt/_client.py b/sdk/agent_stt/speechmatics/agent_stt/_client.py index 501f74a7..23d3063a 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_client.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_client.py @@ -55,7 +55,6 @@ class AsyncClient(RTAsyncClient): api_key: Speechmatics API key, used when `auth` is not given. url: WebSocket endpoint. Defaults to `SPEECHMATICS_RT_URL`, then the EU endpoint. An `/agent` segment is appended if absent. - profile: Service profile, appended to the endpoint path. app: Application name reported to the service as `sm-app`. config: Transcription config for the session, normally an `agent_stt.TranscriptionConfig`. @@ -87,7 +86,6 @@ def __init__( *, api_key: Optional[str] = None, url: Optional[str] = None, - profile: Optional[str] = None, app: Optional[str] = None, config: Optional[RTTranscriptionConfig] = None, audio_format: Optional[AudioFormat] = None, @@ -97,7 +95,7 @@ def __init__( super().__init__( auth, api_key=api_key, - url=resolve_url(url, profile=profile, app=app), + url=resolve_url(url, app=app), conn_config=conn_config, ) diff --git a/sdk/agent_stt/speechmatics/agent_stt/_url.py b/sdk/agent_stt/speechmatics/agent_stt/_url.py index 764af86b..572d7086 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_url.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_url.py @@ -15,19 +15,16 @@ def resolve_url( url: Optional[str] = None, - profile: Optional[str] = None, app: Optional[str] = None, ) -> str: """ Resolve the Agent STT WebSocket URL. - The Agent STT endpoint is the RT endpoint plus an `/agent` path segment, optionally - followed by a service profile name. + The Agent STT endpoint is the RT endpoint plus an `/agent` path segment. Args: url: Explicit endpoint. Falls back to the `SPEECHMATICS_RT_URL` environment variable, then the EU endpoint. - profile: Optional service profile, appended as a final path segment. app: Optional application name reported to the service as `sm-app`. Returns: @@ -36,31 +33,25 @@ def resolve_url( Examples: >>> resolve_url() 'wss://eu2.rt.speechmatics.com/v2/agent?sm-app=agent-stt-sdk%2F0.0.0' - >>> resolve_url("wss://host/v2", profile="default", app="pipecat/1.0") - 'wss://host/v2/agent/default?sm-app=pipecat%2F1.0' + >>> resolve_url("wss://host/v2", app="pipecat/1.0") + 'wss://host/v2/agent?sm-app=pipecat%2F1.0' """ base = url or os.getenv("SPEECHMATICS_RT_URL") or DEFAULT_RT_URL parsed = urlparse(base) return urlunparse( parsed._replace( - path=_resolve_path(parsed.path, profile), + path=_resolve_path(parsed.path), query=_resolve_query(parsed.query, app), ) ) -def _resolve_path(path: str, profile: Optional[str]) -> str: - """Append the /agent segment and the profile to the path, skipping any already present.""" +def _resolve_path(path: str) -> str: + """Append the /agent segment, skipping any already present.""" segments = [segment for segment in path.split("/") if segment] if AGENT_PATH_SEGMENT not in segments: segments.append(AGENT_PATH_SEGMENT) - - if profile: - normalized = profile.strip("/") - if normalized and segments[-1] != normalized: - segments.append(normalized) - return "/" + "/".join(segments) diff --git a/tests/agent_stt/test_client.py b/tests/agent_stt/test_client.py index 8fcb5cd7..0910527d 100644 --- a/tests/agent_stt/test_client.py +++ b/tests/agent_stt/test_client.py @@ -73,10 +73,10 @@ async def test_endpoint_is_the_agent_path(client): @pytest.mark.asyncio -async def test_profile_and_app_reach_the_url(monkeypatch): +async def test_app_reaches_the_url(monkeypatch): monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) - client = AsyncClient(api_key=API_KEY, profile="default", app="pipecat/1.0") - assert "/v2/agent/default" in client._transport._url + client = AsyncClient(api_key=API_KEY, app="pipecat/1.0") + assert "/v2/agent" in client._transport._url assert "sm-app=pipecat%2F1.0" in client._transport._url diff --git a/tests/agent_stt/test_url.py b/tests/agent_stt/test_url.py index d258df49..ef8b1d0d 100644 --- a/tests/agent_stt/test_url.py +++ b/tests/agent_stt/test_url.py @@ -44,18 +44,6 @@ def test_trailing_slash_normalized(): assert _path(resolve_url("wss://example.com/v2/")) == "/v2/agent" -def test_profile_appended(): - assert _path(resolve_url("wss://example.com/v2", profile="default")) == "/v2/agent/default" - - -def test_profile_not_duplicated(): - assert _path(resolve_url("wss://example.com/v2/agent/default", profile="default")) == "/v2/agent/default" - - -def test_profile_slashes_stripped(): - assert _path(resolve_url("wss://example.com/v2", profile="/default/")) == "/v2/agent/default" - - def test_app_reported(): params = parse_qs(urlparse(resolve_url("wss://example.com/v2", app="pipecat/1.0")).query) assert params["sm-app"] == ["pipecat/1.0"] From 1c109731525bdbbcc7e2d01c0a1e8a9b5ae32d5d Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 13:27:50 +0100 Subject: [PATCH 12/19] description update --- sdk/agent_stt/PLAN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index cdb902f7..81add3d5 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -26,7 +26,7 @@ Client -> server: unchanged (`StartRecognition`, binary audio, `EndOfStream`, `ForceEndOfUtterance`). No new client messages. `StartRecognition.transcription_config` gains two service-only fields, stripped by the service -before it forwards to the RT engine (`_profiles/_rt_conversion.NON_RT_API_FIELDS`): +before it forwards to the RT engine: - `vad_config`: `{enabled, window, onset_threshold, offset_threshold}` - `emit_sentences`: `bool` - close a segment on sentence boundaries mid-turn From e686074df63ffa3edad072ba83dee285a8ffe417 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 13:30:56 +0100 Subject: [PATCH 13/19] rename to AgentSttAsyncclient --- examples/agent_stt/client_vad/main.py | 4 ++-- examples/agent_stt/file/main.py | 4 ++-- examples/agent_stt/microphone/main.py | 4 ++-- examples/agent_stt/microphone_windows/main.py | 8 ++++---- examples/agent_stt/realtime_file/main.py | 8 ++++---- sdk/agent_stt/PLAN.md | 2 +- sdk/agent_stt/README.md | 12 ++++++------ sdk/agent_stt/speechmatics/agent_stt/__init__.py | 4 ++-- sdk/agent_stt/speechmatics/agent_stt/_client.py | 16 ++++++++-------- tests/agent_stt/test_client.py | 8 ++++---- 10 files changed, 35 insertions(+), 35 deletions(-) diff --git a/examples/agent_stt/client_vad/main.py b/examples/agent_stt/client_vad/main.py index f10a6abb..51ea7a9a 100644 --- a/examples/agent_stt/client_vad/main.py +++ b/examples/agent_stt/client_vad/main.py @@ -13,7 +13,7 @@ import sys import wave -from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import AgentSttAsyncClient from speechmatics.agent_stt import ServerMessageType from speechmatics.agent_stt import TranscriptionConfig from speechmatics.agent_stt import TurnDetectionMode @@ -27,7 +27,7 @@ async def main(path: str) -> None: config = TranscriptionConfig(language="en", enable_partials=True, turn_detection_mode=TurnDetectionMode.EXTERNAL) # Uses SPEECHMATICS_API_KEY from the environment - async with AsyncClient(config=config) as client: + async with AgentSttAsyncClient(config=config) as client: @client.on(ServerMessageType.ADD_SEGMENT) def handle_segment(message): diff --git a/examples/agent_stt/file/main.py b/examples/agent_stt/file/main.py index 58a40a36..d7a510e5 100644 --- a/examples/agent_stt/file/main.py +++ b/examples/agent_stt/file/main.py @@ -10,7 +10,7 @@ import sys import wave -from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import AgentSttAsyncClient from speechmatics.agent_stt import ServerMessageType from speechmatics.agent_stt import TranscriptionConfig @@ -29,7 +29,7 @@ def read(self, size: int) -> bytes: async def main(path: str) -> None: # Uses SPEECHMATICS_API_KEY from the environment - client = AsyncClient(config=TranscriptionConfig(language="en", enable_partials=True)) + client = AgentSttAsyncClient(config=TranscriptionConfig(language="en", enable_partials=True)) @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) def handle_partial_segment(message): diff --git a/examples/agent_stt/microphone/main.py b/examples/agent_stt/microphone/main.py index 2763840d..98c95ac7 100644 --- a/examples/agent_stt/microphone/main.py +++ b/examples/agent_stt/microphone/main.py @@ -8,7 +8,7 @@ import asyncio -from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import AgentSttAsyncClient from speechmatics.agent_stt import Microphone from speechmatics.agent_stt import ServerMessageType from speechmatics.agent_stt import TranscriptionConfig @@ -26,7 +26,7 @@ async def main() -> None: config = TranscriptionConfig(language="en", enable_partials=True, diarization="speaker") # Uses SPEECHMATICS_API_KEY from the environment - async with AsyncClient(config=config) as client: + async with AgentSttAsyncClient(config=config) as client: @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) def handle_partial_segment(message): diff --git a/examples/agent_stt/microphone_windows/main.py b/examples/agent_stt/microphone_windows/main.py index 9f8d6ad0..6804d378 100644 --- a/examples/agent_stt/microphone_windows/main.py +++ b/examples/agent_stt/microphone_windows/main.py @@ -19,7 +19,7 @@ import signal import sys -from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import AgentSttAsyncClient from speechmatics.agent_stt import Microphone from speechmatics.agent_stt import ServerMessageType from speechmatics.agent_stt import TranscriptionConfig @@ -66,7 +66,7 @@ def list_devices() -> None: print(f" {device['index']:>2} {device['name']} ({device['channels']} ch)") -def build_client(args: argparse.Namespace, console: Console) -> AsyncClient: +def build_client(args: argparse.Namespace, console: Console) -> AgentSttAsyncClient: config = TranscriptionConfig( language=args.language, enable_partials=not args.no_partials, @@ -74,7 +74,7 @@ def build_client(args: argparse.Namespace, console: Console) -> AsyncClient: ) # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_RT_URL to point at a local service - client = AsyncClient(config=config) + client = AgentSttAsyncClient(config=config) @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) def handle_partial_segment(message): @@ -97,7 +97,7 @@ def handle_error(message): return client -async def capture(client: AsyncClient, mic: Microphone, stop: asyncio.Event) -> None: +async def capture(client: AgentSttAsyncClient, mic: Microphone, stop: asyncio.Event) -> None: """Pump microphone frames until Ctrl+C, which sets `stop`.""" while not stop.is_set(): await client.send_audio(await mic.read(CHUNK_SIZE)) diff --git a/examples/agent_stt/realtime_file/main.py b/examples/agent_stt/realtime_file/main.py index 24250116..2c11eadb 100644 --- a/examples/agent_stt/realtime_file/main.py +++ b/examples/agent_stt/realtime_file/main.py @@ -14,7 +14,7 @@ import wave from typing import Optional -from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import AgentSttAsyncClient from speechmatics.agent_stt import ServerMessageType from speechmatics.agent_stt import TranscriptionConfig from speechmatics.agent_stt import TurnDetectionMode @@ -73,7 +73,7 @@ def log(self, tag: str, text: str, *, audio_time: Optional[float] = None, record print(f"[{self.elapsed:6.2f}s] {tag:<10}{lag:<10} {text}") -def build_client(args: argparse.Namespace, clock: Clock) -> AsyncClient: +def build_client(args: argparse.Namespace, clock: Clock) -> AgentSttAsyncClient: config = TranscriptionConfig( language=args.language, enable_partials=not args.no_partials, @@ -83,7 +83,7 @@ def build_client(args: argparse.Namespace, clock: Clock) -> AsyncClient: ) # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_RT_URL to point at a local service - client = AsyncClient(config=config) + client = AgentSttAsyncClient(config=config) @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) def handle_partial_segment(message): @@ -122,7 +122,7 @@ def handle_error(message): return client -async def stream(client: AsyncClient, wav: wave.Wave_read, args: argparse.Namespace, clock: Clock) -> None: +async def stream(client: AgentSttAsyncClient, wav: wave.Wave_read, args: argparse.Namespace, clock: Clock) -> None: """Send the file frame by frame, releasing each frame no earlier than its capture time.""" frames_per_chunk = int(SAMPLE_RATE * args.chunk_ms / 1000) next_turn_end = args.turn_seconds diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index 81add3d5..3e916083 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -60,7 +60,7 @@ sdk/agent_stt/ PLAN.md this file speechmatics/agent_stt/ __init__.py public API - _client.py AsyncClient (subclasses rt.AsyncClient) + _client.py AgentSttAsyncClient (subclasses rt.AsyncClient) _models.py message enums, TranscriptionConfig, VADConfig, Segment, TimedEvent _transcript.py Transcript - final segments, live partial, timeline, raw event log _transport.py AgentTransport - stamps sm-sdk=python-agent-stt-sdk-vX diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index 1eb66881..9bfd4fce 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -15,11 +15,11 @@ pip install speechmatics-agent-stt ```python import asyncio -from speechmatics.agent_stt import AsyncClient, ServerMessageType, TranscriptionConfig +from speechmatics.agent_stt import AgentSttAsyncClient, ServerMessageType, TranscriptionConfig async def main(): # Uses SPEECHMATICS_API_KEY from the environment - async with AsyncClient(config=TranscriptionConfig(language="en", enable_partials=True)) as client: + async with AgentSttAsyncClient(config=TranscriptionConfig(language="en", enable_partials=True)) as client: @client.on(ServerMessageType.ADD_SEGMENT) def handle_segment(message): print(message["segment"]["transcript"]) @@ -78,7 +78,7 @@ client.session_info # session id and the language pack the service report client.transcript_text(speaker_labels=True, include_partial=False) ``` -Pass `record_events=False` to `AsyncClient` for long-running sessions where the raw log is not +Pass `record_events=False` to `AgentSttAsyncClient` for long-running sessions where the raw log is not wanted. ## Messages @@ -132,9 +132,9 @@ turn ends either because the service's VAD said so, or because you called `final The Agent STT endpoint is the RT endpoint plus `/agent`: ```python -AsyncClient(url="wss://eu2.rt.speechmatics.com/v2") # -> /v2/agent -AsyncClient(url="ws://localhost:8000/v2") # -> /v2/agent -AsyncClient(app="pipecat/1.0") # reported as sm-app +AgentSttAsyncClient(url="wss://eu2.rt.speechmatics.com/v2") # -> /v2/agent +AgentSttAsyncClient(url="ws://localhost:8000/v2") # -> /v2/agent +AgentSttAsyncClient(app="pipecat/1.0") # reported as sm-app ``` Resolution order: the `url` argument, `SPEECHMATICS_RT_URL`, then the EU endpoint. The `/agent` diff --git a/sdk/agent_stt/speechmatics/agent_stt/__init__.py b/sdk/agent_stt/speechmatics/agent_stt/__init__.py index 4fa2514c..696a6f94 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/__init__.py +++ b/sdk/agent_stt/speechmatics/agent_stt/__init__.py @@ -35,8 +35,8 @@ from speechmatics.rt import TranslationConfig from speechmatics.rt import TransportError +from ._client import AgentSttAsyncClient from ._client import AgentSTTClient -from ._client import AsyncClient from ._models import DEFAULT_CHUNK_SIZE from ._models import DEFAULT_MODEL from ._models import DEFAULT_SAMPLE_RATE @@ -67,7 +67,7 @@ "__version__", # Client "AgentSTTClient", - "AsyncClient", + "AgentSttAsyncClient", # Config "AdditionalVocabEntry", "AudioEncoding", diff --git a/sdk/agent_stt/speechmatics/agent_stt/_client.py b/sdk/agent_stt/speechmatics/agent_stt/_client.py index 23d3063a..f8910e02 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_client.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_client.py @@ -36,10 +36,10 @@ _UNSET = object() -DISCONNECT_TIMEOUT = 5.0 +DISCONNECT_TIMEOUT_S = 5.0 -class AsyncClient(RTAsyncClient): +class AgentSttAsyncClient(RTAsyncClient): """ Asynchronous client for the Speechmatics Agent STT service. @@ -65,7 +65,7 @@ class AsyncClient(RTAsyncClient): Examples: Service VAD, transcript at the end: - >>> async with AsyncClient(api_key="your-key") as client: + >>> async with AgentSttAsyncClient(api_key="your-key") as client: ... @client.on(ServerMessageType.ADD_SEGMENT) ... def handle_segment(message): ... print(message["segment"]["transcript"]) @@ -74,7 +74,7 @@ class AsyncClient(RTAsyncClient): External endpointing (Pipecat, LiveKit): >>> config = TranscriptionConfig(turn_detection_mode=TurnDetectionMode.EXTERNAL) - >>> client = AsyncClient(api_key="your-key", config=config) + >>> client = AgentSttAsyncClient(api_key="your-key", config=config) >>> await client.connect() >>> await client.send_audio(frame) >>> client.finalize() # on the application's own end-of-speech signal @@ -163,7 +163,7 @@ async def connect(self, ws_headers: Optional[dict] = None) -> None: TimeoutError: If the service does not accept the session in time. Examples: - >>> client = AsyncClient(api_key="your-key") + >>> client = AgentSttAsyncClient(api_key="your-key") >>> await client.connect() """ if self._is_connected: @@ -190,14 +190,14 @@ async def disconnect(self) -> None: self._is_ready_for_audio = False try: - await asyncio.wait_for(self.stop_session(), timeout=DISCONNECT_TIMEOUT) + await asyncio.wait_for(self.stop_session(), timeout=DISCONNECT_TIMEOUT_S) except Exception as e: self._logger.warning("Error closing session: %s", e) await self.close() finally: self._is_connected = False - async def __aenter__(self) -> AsyncClient: + async def __aenter__(self) -> AgentSttAsyncClient: """Open the session on entry.""" await self.connect() return self @@ -485,4 +485,4 @@ async def close(self) -> None: await super().close() -AgentSTTClient = AsyncClient +AgentSTTClient = AgentSttAsyncClient diff --git a/tests/agent_stt/test_client.py b/tests/agent_stt/test_client.py index 0910527d..4f8d91be 100644 --- a/tests/agent_stt/test_client.py +++ b/tests/agent_stt/test_client.py @@ -2,7 +2,7 @@ import pytest -from speechmatics.agent_stt import AsyncClient +from speechmatics.agent_stt import AgentSttAsyncClient from speechmatics.agent_stt import AudioEncoding from speechmatics.agent_stt import ClientMessageType from speechmatics.agent_stt import ServerMessageType @@ -37,7 +37,7 @@ def audio(self): @pytest.fixture def client(monkeypatch): monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) - return AsyncClient(api_key=API_KEY) + return AgentSttAsyncClient(api_key=API_KEY) def recognition_started(word_delimiter=" "): @@ -75,7 +75,7 @@ async def test_endpoint_is_the_agent_path(client): @pytest.mark.asyncio async def test_app_reaches_the_url(monkeypatch): monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) - client = AsyncClient(api_key=API_KEY, app="pipecat/1.0") + client = AgentSttAsyncClient(api_key=API_KEY, app="pipecat/1.0") assert "/v2/agent" in client._transport._url assert "sm-app=pipecat%2F1.0" in client._transport._url @@ -158,7 +158,7 @@ async def test_every_message_is_recorded(client): @pytest.mark.asyncio async def test_event_recording_can_be_disabled(monkeypatch): monkeypatch.delenv("SPEECHMATICS_RT_URL", raising=False) - client = AsyncClient(api_key=API_KEY, record_events=False) + client = AgentSttAsyncClient(api_key=API_KEY, record_events=False) start_session(client) assert client.events == [] assert client.session_info.session_id == "session-1" From f141bee61fbe0152897607e789423c1939a80019 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 13:35:59 +0100 Subject: [PATCH 14/19] remove audio events ans translation config --- sdk/agent_stt/speechmatics/agent_stt/__init__.py | 4 ---- sdk/agent_stt/speechmatics/agent_stt/_client.py | 14 -------------- 2 files changed, 18 deletions(-) diff --git a/sdk/agent_stt/speechmatics/agent_stt/__init__.py b/sdk/agent_stt/speechmatics/agent_stt/__init__.py index 696a6f94..f5d22888 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/__init__.py +++ b/sdk/agent_stt/speechmatics/agent_stt/__init__.py @@ -15,7 +15,6 @@ from speechmatics.rt import AudioEncoding from speechmatics.rt import AudioError -from speechmatics.rt import AudioEventsConfig from speechmatics.rt import AudioFormat from speechmatics.rt import AuthBase from speechmatics.rt import AuthenticationError @@ -32,7 +31,6 @@ from speechmatics.rt import StaticKeyAuth from speechmatics.rt import TimeoutError from speechmatics.rt import TranscriptionError -from speechmatics.rt import TranslationConfig from speechmatics.rt import TransportError from ._client import AgentSttAsyncClient @@ -71,14 +69,12 @@ # Config "AdditionalVocabEntry", "AudioEncoding", - "AudioEventsConfig", "AudioFormat", "ConnectionConfig", "ConversationConfig", "SpeakerDiarizationConfig", "SpeakerIdentifier", "TranscriptionConfig", - "TranslationConfig", "TurnDetectionMode", "Model", "VADConfig", diff --git a/sdk/agent_stt/speechmatics/agent_stt/_client.py b/sdk/agent_stt/speechmatics/agent_stt/_client.py index f8910e02..90e2d383 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_client.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_client.py @@ -9,14 +9,12 @@ from speechmatics.rt import AsyncClient as RTAsyncClient from speechmatics.rt import AudioEncoding -from speechmatics.rt import AudioEventsConfig from speechmatics.rt import AudioFormat from speechmatics.rt import AuthBase from speechmatics.rt import ConnectionConfig from speechmatics.rt import StaticKeyAuth from speechmatics.rt import TimeoutError as RTTimeoutError from speechmatics.rt import TranscriptionConfig as RTTranscriptionConfig -from speechmatics.rt import TranslationConfig from speechmatics.rt import TransportError from ._logging import get_logger @@ -211,8 +209,6 @@ async def start_session( *, transcription_config: Optional[RTTranscriptionConfig] = None, audio_format: Optional[AudioFormat] = None, - translation_config: Optional[TranslationConfig] = None, - audio_events_config: Optional[AudioEventsConfig] = None, ws_headers: Optional[dict] = None, ) -> None: """ @@ -221,8 +217,6 @@ async def start_session( Args: transcription_config: Transcription config for the session. audio_format: Audio format. Must be 16 kHz raw PCM for the Agent STT service. - translation_config: Optional translation config. - audio_events_config: Optional audio event detection config. ws_headers: Additional WebSocket handshake headers. Raises: @@ -232,8 +226,6 @@ async def start_session( await super().start_session( transcription_config=transcription_config or self._config, audio_format=audio_format or self._audio_format, - translation_config=translation_config, - audio_events_config=audio_events_config, ws_headers=ws_headers, ) @@ -325,8 +317,6 @@ async def transcribe( *, transcription_config: Optional[RTTranscriptionConfig] = None, audio_format: Optional[AudioFormat] = None, - translation_config: Optional[TranslationConfig] = None, - audio_events_config: Optional[AudioEventsConfig] = None, ws_headers: Optional[dict] = None, timeout: Optional[float] = None, ) -> None: @@ -338,8 +328,6 @@ async def transcribe( audio format. transcription_config: Transcription config for the session. audio_format: Audio format. Must be 16 kHz raw PCM for the Agent STT service. - translation_config: Optional translation config. - audio_events_config: Optional audio event detection config. ws_headers: Additional WebSocket handshake headers. timeout: Maximum time in seconds to wait for the stream to finish. @@ -361,8 +349,6 @@ async def transcribe( await self.start_session( transcription_config=self._config, audio_format=self._audio_format, - translation_config=translation_config, - audio_events_config=audio_events_config, ws_headers=ws_headers, ) self._is_connected = True From cecf62c7a3a98bb132c24597d378255f9e8668cc Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 13:50:41 +0100 Subject: [PATCH 15/19] small printing adjustment --- examples/agent_stt/realtime_file/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/agent_stt/realtime_file/main.py b/examples/agent_stt/realtime_file/main.py index 2c11eadb..379dfb1a 100644 --- a/examples/agent_stt/realtime_file/main.py +++ b/examples/agent_stt/realtime_file/main.py @@ -70,7 +70,7 @@ def log(self, tag: str, text: str, *, audio_time: Optional[float] = None, record if record: self.segment_lags.append(behind) lag = f"+{behind * 1000:>5.0f}ms" - print(f"[{self.elapsed:6.2f}s] {tag:<10}{lag:<10} {text}") + print(f"[{self.elapsed:6.2f}s] {tag:<17}{lag:<10} {text}") def build_client(args: argparse.Namespace, clock: Clock) -> AgentSttAsyncClient: @@ -100,20 +100,20 @@ def handle_segment(message): @client.on(ServerMessageType.SPEECH_STARTED) def handle_speech_started(message): - clock.log("[speech]", f"started at {message['metadata']['start_time']:.2f}s") + clock.log("[speech started]", f"{message['metadata']['start_time']:.2f}s") @client.on(ServerMessageType.SPEECH_ENDED) def handle_speech_ended(message): - clock.log("[speech]", f"ended at {message['metadata']['end_time']:.2f}s") + clock.log("[speech ended]", f"{message['metadata']['end_time']:.2f}s") @client.on(ServerMessageType.START_OF_TURN) def handle_start_of_turn(message): - clock.log("[turn]", f"start at {message['metadata']['start_time']:.2f}s") + clock.log("[turn started]", f"{message['metadata']['start_time']:.2f}s") @client.on(ServerMessageType.END_OF_TURN) def handle_end_of_turn(message): end_time = message["metadata"]["end_time"] - clock.log("[turn]", f"end at {end_time:.2f}s", audio_time=end_time) + clock.log("[turn ended]", f"{end_time:.2f}s", audio_time=end_time) @client.on(ServerMessageType.ERROR) def handle_error(message): From 17f65f4f6ac05b8c2b68b0a7cda48c4a2cfc1879 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 13:51:30 +0100 Subject: [PATCH 16/19] remove windows specific file as the generic microphone works on win anw --- examples/agent_stt/microphone_windows/main.py | 150 ------------------ 1 file changed, 150 deletions(-) delete mode 100644 examples/agent_stt/microphone_windows/main.py diff --git a/examples/agent_stt/microphone_windows/main.py b/examples/agent_stt/microphone_windows/main.py deleted file mode 100644 index 6804d378..00000000 --- a/examples/agent_stt/microphone_windows/main.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Live microphone transcription with the Agent STT service, set up for Windows. - -The service runs its own VAD, so it decides where each turn ends; this script only captures -the microphone and prints what comes back. Partials are rewritten in place on one line and -each closed segment is printed above them. - -Setup in PowerShell: - - py -m pip install speechmatics-agent-stt pyaudio - $env:SPEECHMATICS_API_KEY = "your-key" - py examples\\agent_stt\\microphone_windows\\main.py - -Add `$env:SPEECHMATICS_RT_URL = "wss://preview.rt.speechmatics.com/v2"` to point at a -service. Use `--list-devices` and `--device N` when Windows picks the wrong input. -""" - -import argparse -import asyncio -import signal -import sys - -from speechmatics.agent_stt import AgentSttAsyncClient -from speechmatics.agent_stt import Microphone -from speechmatics.agent_stt import ServerMessageType -from speechmatics.agent_stt import TranscriptionConfig - -SAMPLE_RATE = 16000 -CHUNK_SIZE = 1024 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--list-devices", action="store_true", help="list input devices and exit") - parser.add_argument("--device", type=int, help="input device index, from --list-devices") - parser.add_argument("--language", default="en") - parser.add_argument("--diarization", action="store_true", help="label segments by speaker") - parser.add_argument("--no-partials", action="store_true") - return parser.parse_args() - - -class Console: - """Keeps the partial on one rewritten line, with finals printed above it.""" - - def __init__(self) -> None: - self._partial_width = 0 - - def partial(self, text: str) -> None: - line = f"[partial] {text}" - print("\r" + line.ljust(self._partial_width), end="", flush=True) - self._partial_width = len(line) - - def line(self, text: str) -> None: - print("\r" + text.ljust(self._partial_width), flush=True) - self._partial_width = 0 - - -def list_devices() -> None: - devices = Microphone.list_devices() - if not devices: - print("No input devices found. Is pyaudio installed, and does Windows list a microphone?") - return - for device in devices: - print(f" {device['index']:>2} {device['name']} ({device['channels']} ch)") - - -def build_client(args: argparse.Namespace, console: Console) -> AgentSttAsyncClient: - config = TranscriptionConfig( - language=args.language, - enable_partials=not args.no_partials, - diarization="speaker" if args.diarization else None, - ) - - # Uses SPEECHMATICS_API_KEY, and SPEECHMATICS_RT_URL to point at a local service - client = AgentSttAsyncClient(config=config) - - @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) - def handle_partial_segment(message): - console.partial(message["segment"]["transcript"]) - - @client.on(ServerMessageType.ADD_SEGMENT) - def handle_segment(message): - segment = message["segment"] - speaker = f"{segment['speaker']}: " if segment.get("speaker") else "" - console.line(f"[final] {speaker}{segment['transcript']}") - - @client.on(ServerMessageType.END_OF_TURN) - def handle_end_of_turn(message): - console.line(f"[turn] end at {message['metadata']['end_time']:.2f}s") - - @client.on(ServerMessageType.ERROR) - def handle_error(message): - console.line(f"[error] {message}") - - return client - - -async def capture(client: AgentSttAsyncClient, mic: Microphone, stop: asyncio.Event) -> None: - """Pump microphone frames until Ctrl+C, which sets `stop`.""" - while not stop.is_set(): - await client.send_audio(await mic.read(CHUNK_SIZE)) - - -async def main() -> None: - args = parse_args() - - if args.list_devices: - list_devices() - return - - mic = Microphone(sample_rate=SAMPLE_RATE, chunk_size=CHUNK_SIZE, device_index=args.device) - if not mic.is_available: - print("pyaudio is not installed. Install it with: py -m pip install pyaudio") - return - if not mic.start(): - print(f"Could not open the microphone at {SAMPLE_RATE} Hz. Available inputs:") - list_devices() - print( - "\nPick one with --device N. If none open, set the device's Default Format to\n" - "16000 Hz in Sound Control Panel > Recording > Properties > Advanced." - ) - return - - # Ctrl+C on Windows will not interrupt a pending await, so shut down through an event - stop = asyncio.Event() - signal.signal(signal.SIGINT, lambda *_: stop.set()) - - console = Console() - client = build_client(args, console) - - try: - async with client: - print("\nMicrophone ready - speak now (Ctrl+C to stop)\n") - await capture(client, mic, stop) - finally: - mic.stop() - - console.line("") - print(f"Transcript: {client.transcript_text(speaker_labels=args.diarization)}") - - -# Windows spawns child processes by re-importing this file, so keep startup behind the guard -if __name__ == "__main__": - # The Windows console defaults to a legacy code page that cannot print every transcript - if sys.platform == "win32": - sys.stdout.reconfigure(encoding="utf-8", errors="replace") - - asyncio.run(main()) From 381511243ce99047ac0ab16cf4b6e4b4ecaef631 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 13:56:37 +0100 Subject: [PATCH 17/19] remove EndOfUtterance msg, as that's superseded by endOfTurn --- sdk/agent_stt/PLAN.md | 6 ++---- sdk/agent_stt/README.md | 9 ++++----- sdk/agent_stt/speechmatics/agent_stt/_models.py | 3 --- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/sdk/agent_stt/PLAN.md b/sdk/agent_stt/PLAN.md index 3e916083..625b27bf 100644 --- a/sdk/agent_stt/PLAN.md +++ b/sdk/agent_stt/PLAN.md @@ -41,7 +41,6 @@ Server -> client, new messages (`voice_agent_api/_service_messages.py`): Server -> client, RT passthrough: `RecognitionStarted`, `AudioAdded`, `AddTranscript`, `AddPartialTranscript`, `EndOfTranscript`, `Info`, `Warning`, `Error`, audio events. -`EndOfUtterance` is consumed by the service and never reaches the client. Note: the service still forwards `AddTranscript`/`AddPartialTranscript` verbatim today. The SDK accumulates its transcript from **segments only** and models neither those nor audio events; @@ -142,9 +141,8 @@ touched by this change. The migration: ### `FIXED` and `SMART_TURN` are removed -`end_of_utterance_silence_trigger` is off for this service: the service pins it to `0.0`, -the service consumes `EndOfUtterance` rather than forwarding it, and a non-forced end of utterance -does not close a segment. So there is nothing for `TurnDetectionMode.FIXED` to mean here and it +`end_of_utterance_silence_trigger` is off for this service: the service pins it to `0.0`, and a +non-forced end of utterance does not close a segment. So there is nothing for `TurnDetectionMode.FIXED` to mean here and it goes away rather than being aliased to another mode. `SMART_TURN` goes for the same reason: the service has no smart-turn endpoint yet (planned for a diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index 9bfd4fce..66ac7172 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -93,8 +93,7 @@ Emitted by the service: | `StartOfTurn` / `EndOfTurn` | `metadata.start_time` / `metadata.end_time` (service turn detection) | Passed through from the RT engine: `RecognitionStarted`, `AudioAdded`, `EndOfTranscript`, -`SpeakersResult`, `Info`, `Warning`, `Error`. `EndOfUtterance` is consumed by the service and -not forwarded. +`SpeakersResult`, `Info`, `Warning`, `Error`. Anything else the engine sends - the word-level `AddTranscript`/`AddPartialTranscript`, audio events - is not modelled here, but still reaches `client.events` and any handler registered @@ -123,9 +122,9 @@ point, so the transcriber never sees a name it has no notion of. The RT models ( `standard`) are not Agent STT models and are not accepted here; the deprecated `operating_point` still passes through, and suppresses the `model` default so the two never arrive together. -Engine silence-based end of utterance is off for this service, and `EndOfUtterance` is not -forwarded, so `conversation_config.end_of_utterance_silence_trigger` does not close segments. A -turn ends either because the service's VAD said so, or because you called `finalize()`. +Engine silence-based end of utterance is off for this service, so +`conversation_config.end_of_utterance_silence_trigger` does not close segments. A turn ends +either because the service's VAD said so, or because you called `finalize()`. ## Endpoint diff --git a/sdk/agent_stt/speechmatics/agent_stt/_models.py b/sdk/agent_stt/speechmatics/agent_stt/_models.py index 4ba8516d..03551991 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/_models.py +++ b/sdk/agent_stt/speechmatics/agent_stt/_models.py @@ -66,8 +66,6 @@ class ServerMessageType(str, Enum): SPEECH_ENDED: The service's VAD detected speech offset. START_OF_TURN: The service's turn detection opened a turn. END_OF_TURN: The service's turn detection closed a turn. - END_OF_UTTERANCE: Consumed by the service for segmentation and not forwarded; listed - so handlers stay valid against a direct RT endpoint. END_OF_TRANSCRIPT: The service has finished sending messages. SPEAKERS_RESULT: Response to GetSpeakers. INFO: Informational message. @@ -88,7 +86,6 @@ class ServerMessageType(str, Enum): SPEECH_ENDED = "SpeechEnded" START_OF_TURN = "StartOfTurn" END_OF_TURN = "EndOfTurn" - END_OF_UTTERANCE = "EndOfUtterance" END_OF_TRANSCRIPT = "EndOfTranscript" SPEAKERS_RESULT = "SpeakersResult" INFO = "Info" From a3094ff3768e4bedb6adae78152b47f4edff3524 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 14:19:51 +0100 Subject: [PATCH 18/19] remove not real time streaming file --- examples/agent_stt/file/main.py | 59 --------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 examples/agent_stt/file/main.py diff --git a/examples/agent_stt/file/main.py b/examples/agent_stt/file/main.py deleted file mode 100644 index d7a510e5..00000000 --- a/examples/agent_stt/file/main.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Transcribe a 16 kHz WAV file with the Agent STT service. - -The service runs its own VAD here, so it reports speech and turn events and closes each -segment itself. The whole transcript is on the client when the session ends. - -Run with: python examples/agent_stt/file/main.py [path/to/16kHz.wav] -""" - -import asyncio -import sys -import wave - -from speechmatics.agent_stt import AgentSttAsyncClient -from speechmatics.agent_stt import ServerMessageType -from speechmatics.agent_stt import TranscriptionConfig - -DEFAULT_AUDIO_FILE = "./tests/voice/assets/audio_01_16kHz.wav" - - -class WavSource: - """Reads raw PCM frames out of a WAV file, leaving the header behind.""" - - def __init__(self, wav: wave.Wave_read) -> None: - self._wav = wav - - def read(self, size: int) -> bytes: - return self._wav.readframes(size // self._wav.getsampwidth()) - - -async def main(path: str) -> None: - # Uses SPEECHMATICS_API_KEY from the environment - client = AgentSttAsyncClient(config=TranscriptionConfig(language="en", enable_partials=True)) - - @client.on(ServerMessageType.ADD_PARTIAL_SEGMENT) - def handle_partial_segment(message): - print(f"[partial] {message['segment']['transcript']}") - - @client.on(ServerMessageType.ADD_SEGMENT) - def handle_segment(message): - print(f"[final] {message['segment']['transcript']}") - - @client.on(ServerMessageType.START_OF_TURN) - def handle_start_of_turn(message): - print(f"[turn] start at {message['metadata']['start_time']}s") - - @client.on(ServerMessageType.END_OF_TURN) - def handle_end_of_turn(message): - print(f"[turn] end at {message['metadata']['end_time']}s") - - with wave.open(path, "rb") as wav: - if wav.getframerate() != 16000: - print(f"{path} is {wav.getframerate()} Hz; the Agent STT service needs 16 kHz audio") - return - await client.transcribe(WavSource(wav)) - - print(f"\nTranscript: {client.transcript}") - - -asyncio.run(main(sys.argv[1] if len(sys.argv) > 1 else DEFAULT_AUDIO_FILE)) From a92c8fa09116937db3b58b4c1d9df95512780e27 Mon Sep 17 00:00:00 2001 From: Georgios Hadjiharalambous Date: Tue, 18 Aug 2026 14:24:12 +0100 Subject: [PATCH 19/19] remove ConversationConfig , as we wont offer end_of_utterance_silence_trigger option --- sdk/agent_stt/README.md | 5 ++--- sdk/agent_stt/speechmatics/agent_stt/__init__.py | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/sdk/agent_stt/README.md b/sdk/agent_stt/README.md index 66ac7172..32bb5bc8 100644 --- a/sdk/agent_stt/README.md +++ b/sdk/agent_stt/README.md @@ -122,9 +122,8 @@ point, so the transcriber never sees a name it has no notion of. The RT models ( `standard`) are not Agent STT models and are not accepted here; the deprecated `operating_point` still passes through, and suppresses the `model` default so the two never arrive together. -Engine silence-based end of utterance is off for this service, so -`conversation_config.end_of_utterance_silence_trigger` does not close segments. A turn ends -either because the service's VAD said so, or because you called `finalize()`. +Engine silence-based end of utterance is not offered here. A turn ends either because the +service's VAD said so, or because you called `finalize()`. ## Endpoint diff --git a/sdk/agent_stt/speechmatics/agent_stt/__init__.py b/sdk/agent_stt/speechmatics/agent_stt/__init__.py index f5d22888..82dfaeb9 100644 --- a/sdk/agent_stt/speechmatics/agent_stt/__init__.py +++ b/sdk/agent_stt/speechmatics/agent_stt/__init__.py @@ -21,7 +21,6 @@ from speechmatics.rt import ConfigurationError from speechmatics.rt import ConnectionConfig from speechmatics.rt import ConnectionError -from speechmatics.rt import ConversationConfig from speechmatics.rt import EventEmitter from speechmatics.rt import JWTAuth from speechmatics.rt import Microphone @@ -71,7 +70,6 @@ "AudioEncoding", "AudioFormat", "ConnectionConfig", - "ConversationConfig", "SpeakerDiarizationConfig", "SpeakerIdentifier", "TranscriptionConfig",