diff --git a/.fernignore b/.fernignore index f54bb59c..bf1bb094 100644 --- a/.fernignore +++ b/.fernignore @@ -14,6 +14,12 @@ src/deepgram/client.py # key / access token) in DEBUG handshake logs. No Fern-generated counterpart. src/deepgram/_secure_logging.py +# Hand-written opt-out telemetry module (SDK Observability). Builds an isolated +# Sentry client (never the global sentry_sdk.init()) wired into client.py via +# the telemetry_opt_out / telemetry_handler constructor params. No Fern-generated +# counterpart. Inert until a real DSN is embedded/supplied. +src/deepgram/telemetry + # WebSocket socket clients: # - except Exception broad catch (supports custom transports, generator narrows to WebSocketException) # - _sanitize_numeric_types in agent socket client (float→int for API) @@ -173,6 +179,7 @@ tests/custom/test_listen_v2_connect_wire.py tests/custom/test_listen_v2_regen_constraints.py tests/custom/test_query_encoder.py tests/custom/test_secure_logging.py +tests/custom/test_telemetry.py tests/custom/test_socket_client_shims.py tests/custom/test_speak_v2_connect_wire.py tests/custom/test_speak_v2_socket.py diff --git a/poetry.lock b/poetry.lock index a9b5c91a..55a035cd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -1338,6 +1338,71 @@ files = [ {file = "ruff-0.11.5.tar.gz", hash = "sha256:cae2e2439cb88853e421901ec040a758960b576126dab520fa08e9de431d1bef"}, ] +[[package]] +name = "sentry-sdk" +version = "2.67.1" +description = "Python client for Sentry (https://sentry.io)" +optional = true +python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"telemetry\"" +files = [ + {file = "sentry_sdk-2.67.1-py3-none-any.whl", hash = "sha256:a66bfbce1cd8a93c51c369d642ad85b46253ea7a6f7938141315b83e2823cda5"}, + {file = "sentry_sdk-2.67.1.tar.gz", hash = "sha256:f263d8c9aa4137750640de8fb0ed5404df6bb564e20e4b59cb16a6eeba18d4ed"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = ">=1.26.11" + +[package.extras] +aiohttp = ["aiohttp (>=3.5)"] +anthropic = ["anthropic (>=0.16)"] +arq = ["arq (>=0.23)"] +asyncio = ["httpcore[asyncio] (==1.*)"] +asyncpg = ["asyncpg (>=0.23)"] +beam = ["apache-beam (>=2.12)"] +bottle = ["bottle (>=0.12.13)"] +celery = ["celery (>=3)"] +celery-redbeat = ["celery-redbeat (>=2)"] +chalice = ["chalice (>=1.16.0)"] +clickhouse-driver = ["clickhouse-driver (>=0.2.0)"] +django = ["django (>=1.8)"] +falcon = ["falcon (>=1.4)"] +fastapi = ["fastapi (>=0.79.0)"] +flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"] +google-genai = ["google-genai (>=1.29.0)"] +grpcio = ["grpcio (>=1.21.1)", "protobuf (>=3.8.0)"] +http2 = ["httpcore[http2] (==1.*)"] +httpx = ["httpx (>=0.16.0)"] +huey = ["huey (>=2)"] +huggingface-hub = ["huggingface_hub (>=0.22)"] +langchain = ["langchain (>=0.0.210)"] +langgraph = ["langgraph (>=0.6.6)"] +launchdarkly = ["launchdarkly-server-sdk (>=9.8.0)"] +litellm = ["litellm (>=1.77.5,!=1.82.7,!=1.82.8)"] +litestar = ["litestar (>=2.0.0)"] +loguru = ["loguru (>=0.5)"] +mcp = ["mcp (>=1.15.0)"] +openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"] +openfeature = ["openfeature-sdk (>=0.7.1)"] +opentelemetry = ["opentelemetry-distro (>=0.35b0)"] +opentelemetry-experimental = ["opentelemetry-distro"] +opentelemetry-otlp = ["opentelemetry-distro[otlp] (>=0.35b0)"] +pure-eval = ["asttokens", "executing", "pure_eval"] +pydantic-ai = ["pydantic-ai (>=1.0.0)"] +pymongo = ["pymongo (>=3.1)"] +pyspark = ["pyspark (>=2.4.4)"] +quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] +rq = ["rq (>=0.6)"] +sanic = ["sanic (>=0.8)"] +sqlalchemy = ["sqlalchemy (>=1.2)"] +starlette = ["starlette (>=0.19.1)"] +starlite = ["starlite (>=1.48)"] +statsig = ["statsig (>=0.55.3)"] +tornado = ["tornado (>=6)"] +unleash = ["UnleashClient (>=6.0.1)"] + [[package]] name = "six" version = "1.17.0" @@ -1468,11 +1533,12 @@ version = "2.7.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, ] +markers = {main = "extra == \"telemetry\""} [package.extras] brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] @@ -1721,8 +1787,9 @@ propcache = ">=0.2.1" [extras] aiohttp = ["aiohttp", "httpx-aiohttp"] +telemetry = ["sentry-sdk"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "da81cd862a1fa855037469ff02370744d0fdb4d757a53d9bae377f971e6f0fd2" +content-hash = "86483759450d7ae7da1d178cc1e108d3c024e9dc6b00de6119f6ed3875524995" diff --git a/pyproject.toml b/pyproject.toml index 41703540..5a03d73c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ python = "^3.10" aiohttp = { version = ">=3.14.1,<4", optional = true, python = ">=3.10"} httpx = ">=0.21.2" httpx-aiohttp = { version = "^0.1.8", optional = true, python = ">=3.10"} +sentry-sdk = { version = ">=2.0.0", optional = true } pydantic = ">= 1.9.2" pydantic-core = ">=2.18.2,<3.0.0" typing_extensions = ">= 4.0.0" @@ -99,3 +100,4 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.extras] aiohttp=["aiohttp", "httpx-aiohttp"] +telemetry=["sentry-sdk"] diff --git a/src/deepgram/client.py b/src/deepgram/client.py index 5d0383f1..0949c00d 100644 --- a/src/deepgram/client.py +++ b/src/deepgram/client.py @@ -28,7 +28,9 @@ from ._secure_logging import install_websocket_log_redaction from .base_client import AsyncBaseClient, BaseClient +from .telemetry import init_telemetry, install_response_capture from .transport import install_transport +from .version import __version__ from deepgram.core.client_wrapper import BaseClientWrapper @@ -75,8 +77,14 @@ class DeepgramClient(BaseClient): - `redact_credentials_in_logs`: Mask the `Authorization` header (API key / access token) in the `websockets` library's DEBUG handshake logs. Defaults to `True`; set to `False` to opt out and manage credential redaction yourself. - - `telemetry_opt_out`: Telemetry opt-out flag (maintained for backwards compatibility, no-op). - - `telemetry_handler`: Telemetry handler (maintained for backwards compatibility, no-op). + - `telemetry_opt_out`: Opt out of isolated, error-only SDK telemetry. Defaults to + ``True`` (telemetry off). When ``False`` and a telemetry DSN is + configured, the SDK reports its *own* errors (never host-app errors, + never audio/keys/transcripts/PII) to an isolated Sentry client. Also + forced off by the ``DEEPGRAM_TELEMETRY_DISABLED`` env kill-switch. + - `telemetry_handler`: Receive the SDK's own errors in-process instead of phoning home. + A callable ``(exc, tags)`` invoked for SDK-originated exceptions; when + set it fully replaces Sentry and no data leaves the process. """ def __init__(self, *args, **kwargs) -> None: @@ -130,8 +138,22 @@ def __init__(self, *args, **kwargs) -> None: reconnect = False self.reconnect = reconnect - # Store telemetry handler for backwards compatibility (no-op, telemetry not implemented) - self._telemetry_handler = None + # Wire opt-out telemetry. `telemetry_opt_out` and `telemetry_handler` + # are resolved above; a custom handler takes precedence over Sentry, + # and with no opt-out + a resolvable DSN this arms isolated phone-home. + # When active, attach a response hook (HTTP 5xx reporting, tagged with + # session id + request id) to the httpx client the wrapper already + # built — no generated files are touched. + self._telemetry_handler = telemetry_handler + self._telemetry = init_telemetry( + opt_out=telemetry_opt_out, + handler=telemetry_handler, + version=__version__, + ) + if self._telemetry is not None: + install_response_capture( + self._telemetry, self._client_wrapper, final_session_id + ) class AsyncDeepgramClient(AsyncBaseClient): @@ -154,8 +176,14 @@ class AsyncDeepgramClient(AsyncBaseClient): - `redact_credentials_in_logs`: Mask the `Authorization` header (API key / access token) in the `websockets` library's DEBUG handshake logs. Defaults to `True`; set to `False` to opt out and manage credential redaction yourself. - - `telemetry_opt_out`: Telemetry opt-out flag (maintained for backwards compatibility, no-op). - - `telemetry_handler`: Telemetry handler (maintained for backwards compatibility, no-op). + - `telemetry_opt_out`: Opt out of isolated, error-only SDK telemetry. Defaults to + ``True`` (telemetry off). When ``False`` and a telemetry DSN is + configured, the SDK reports its *own* errors (never host-app errors, + never audio/keys/transcripts/PII) to an isolated Sentry client. Also + forced off by the ``DEEPGRAM_TELEMETRY_DISABLED`` env kill-switch. + - `telemetry_handler`: Receive the SDK's own errors in-process instead of phoning home. + A callable ``(exc, tags)`` invoked for SDK-originated exceptions; when + set it fully replaces Sentry and no data leaves the process. """ def __init__(self, *args, **kwargs) -> None: @@ -209,5 +237,19 @@ def __init__(self, *args, **kwargs) -> None: reconnect = False self.reconnect = reconnect - # Store telemetry handler for backwards compatibility (no-op, telemetry not implemented) - self._telemetry_handler = None + # Wire opt-out telemetry. `telemetry_opt_out` and `telemetry_handler` + # are resolved above; a custom handler takes precedence over Sentry, + # and with no opt-out + a resolvable DSN this arms isolated phone-home. + # When active, attach a response hook (HTTP 5xx reporting, tagged with + # session id + request id) to the httpx client the wrapper already + # built — no generated files are touched. + self._telemetry_handler = telemetry_handler + self._telemetry = init_telemetry( + opt_out=telemetry_opt_out, + handler=telemetry_handler, + version=__version__, + ) + if self._telemetry is not None: + install_response_capture( + self._telemetry, self._client_wrapper, final_session_id + ) diff --git a/src/deepgram/telemetry/__init__.py b/src/deepgram/telemetry/__init__.py new file mode 100644 index 00000000..df467c79 --- /dev/null +++ b/src/deepgram/telemetry/__init__.py @@ -0,0 +1,22 @@ +"""Opt-out, isolated Sentry telemetry for the Deepgram Python SDK. + +Hand-written module (not Fern-generated). Wired into ``DeepgramClient`` / +``AsyncDeepgramClient`` via the ``telemetry_opt_out`` and ``telemetry_handler`` +constructor parameters. +""" + +from .client import ( + TelemetryHandler, + TelemetrySink, + init_telemetry, + is_enabled, +) +from .http_hooks import install_response_capture + +__all__ = [ + "TelemetryHandler", + "TelemetrySink", + "init_telemetry", + "install_response_capture", + "is_enabled", +] diff --git a/src/deepgram/telemetry/client.py b/src/deepgram/telemetry/client.py new file mode 100644 index 00000000..0c947734 --- /dev/null +++ b/src/deepgram/telemetry/client.py @@ -0,0 +1,277 @@ +"""Isolated, opt-out Sentry telemetry for the Deepgram Python SDK. + +Design constraints specific to running inside a *library* (not the CLI, which +owns its whole process): + +1. NEVER call the global ``sentry_sdk.init()``. That installs process-wide + integrations (excepthook, atexit, logging) and hijacks the host + application's own Sentry configuration. We instead build a private + ``sentry_sdk.Client`` and capture events by calling ``client.capture_event`` + directly — the customer's global Sentry scope is left completely untouched. + (A bound ``Scope`` is deliberately avoided: in sentry-sdk 2.x a manually + constructed ``Scope`` does not treat its client as active, so + ``Scope.capture_*`` silently no-ops. Calling the client directly is the + reliable isolated path.) +2. Disable all default/auto integrations so we never capture the host app's + unrelated errors. +3. Never capture local variables or request bodies (audio, keys, transcripts). + +Capture is routed through a per-client :class:`TelemetrySink`, not module +globals. Each ``DeepgramClient`` / ``AsyncDeepgramClient`` owns its own sink, so +one client's opt-out never suppresses (or arms) another's, and a custom handler +set on one client never receives another client's events. The heavyweight +Sentry ``Client`` *is* shared per-DSN across sinks in a process — that is a +genuine process resource, not per-client policy. + +The public opt-out surface lives on ``DeepgramClient``/``AsyncDeepgramClient`` +(``telemetry_opt_out=`` and ``telemetry_handler=``); this module implements +what those parameters actually do. +""" + +from __future__ import annotations + +import atexit +import os +import platform +import sys +from typing import Any, Callable, Dict, Optional + +from .scrub import scrub_event + +# Environment overrides. +# DEEPGRAM_TELEMETRY_DISABLED=1 -> hard off, wins over everything +# DEEPGRAM_SDK_TELEMETRY_DSN -> supply/override the Sentry DSN +# DEEPGRAM_TELEMETRY_ENVIRONMENT -> tag events with a deployment environment +DISABLE_ENV_VAR = "DEEPGRAM_TELEMETRY_DISABLED" +DSN_ENV_VAR = "DEEPGRAM_SDK_TELEMETRY_DSN" +ENVIRONMENT_ENV_VAR = "DEEPGRAM_TELEMETRY_ENVIRONMENT" + +# No real DSN is embedded yet. Until a public data-collection policy is +# published AND product signs off on phoning home by default, telemetry stays +# inert unless a DSN is supplied via DEEPGRAM_SDK_TELEMETRY_DSN: with no +# resolvable DSN, `init_telemetry` no-ops regardless of the opt-out flag. Drop +# the real public (ingest-only) DSN here to arm it by default. See the SDK +# Observability task. +_EMBEDDED_DSN = "" + +# Custom handler set by the caller (telemetry_handler=). When present it fully +# replaces Sentry: we hand it the exception and never phone home ourselves. +TelemetryHandler = Callable[[BaseException, Dict[str, str]], None] + +# Process-wide resources (NOT per-client policy): +# _client_cache: one Sentry Client per DSN, reused across sinks in a process. +# _live_clients: every built client, for a single atexit flush. +_client_cache: Dict[str, Any] = {} +_live_clients: list = [] +_flush_registered = False + + +def _resolve_dsn() -> Optional[str]: + dsn = os.environ.get(DSN_ENV_VAR) or _EMBEDDED_DSN + if not dsn or "PLACEHOLDER" in dsn: + return None + return dsn + + +def _resolve_environment() -> Optional[str]: + """The Sentry ``environment`` tag, or ``None`` to leave it unset. + + Deliberately not hard-coded to ``"production"``: this SDK runs inside + customer applications — dev boxes, CI, staging — so a fixed "production" + tag mislabels every event. We honour an explicit + ``DEEPGRAM_TELEMETRY_ENVIRONMENT`` override and otherwise assert nothing. + """ + return os.environ.get(ENVIRONMENT_ENV_VAR) or None + + +def is_enabled(opt_out: bool) -> bool: + """Whether Sentry phone-home should run for a client with this opt-out. + + Order: env kill-switch, then the caller's opt-out flag, then whether a real + DSN is resolvable. A custom handler is considered "enabled" separately (it + doesn't need a DSN) and is wired up in ``init_telemetry``. + """ + if os.environ.get(DISABLE_ENV_VAR, "").lower() in {"1", "true", "yes"}: + return False + if opt_out: + return False + return _resolve_dsn() is not None + + +def _build_base_tags(version: str) -> Dict[str, str]: + return { + "sdk.language": "python", + "sdk.version": version, + "sdk.os": platform.system().lower(), + "sdk.arch": platform.machine().lower(), + "sdk.python": f"{sys.version_info.major}.{sys.version_info.minor}", + } + + +def _get_or_build_client(dsn: str, version: str) -> Any: + """Return the process's Sentry client for ``dsn``, building it once. + + Returns ``None`` if ``sentry-sdk`` is not installed or construction fails. + Never raises. + """ + global _flush_registered + + if dsn in _client_cache: + return _client_cache[dsn] + + try: + from sentry_sdk import Client # type: ignore[import-not-found] + except ImportError: + # `sentry-sdk` is an optional extra; absence just means no phone-home. + return None + + try: + client = Client( + dsn=dsn, + release=f"deepgram-sdk@{version}", + environment=_resolve_environment(), + # Library isolation: no process-wide patching, no host-app capture. + default_integrations=False, + auto_enabling_integrations=False, + integrations=[], + # Treat the SDK's own package as first-party so Sentry highlights + # these frames and stack-trace linking (code mappings) resolves them. + in_app_include=["deepgram"], + # PII / payload safety. + send_default_pii=False, + include_local_variables=False, + max_request_body_size="never", + attach_stacktrace=True, + before_send=scrub_event, + ) + except Exception: + return None + + _client_cache[dsn] = client + _live_clients.append(client) + if not _flush_registered: + atexit.register(_flush_on_exit) + _flush_registered = True + return client + + +class TelemetrySink: + """A single client's capture target: either a Sentry client or a handler. + + Owned by the ``DeepgramClient`` that created it. Keeping capture state here + rather than in module globals is what isolates clients from each other: a + handler bound to this sink only ever sees this client's events, and a sink + that was never armed (no client, no handler) is a total no-op. + """ + + def __init__( + self, + *, + client: Any = None, + handler: Optional[TelemetryHandler] = None, + base_tags: Optional[Dict[str, str]] = None, + ) -> None: + self._client = client + self._handler = handler + self._base_tags = base_tags or {} + + def _merge_tags(self, tags: Optional[Dict[str, str]]) -> Dict[str, str]: + merged = dict(self._base_tags) + if tags: + merged.update(tags) + return merged + + def capture_exception( + self, exc: BaseException, *, tags: Optional[Dict[str, str]] = None + ) -> None: + """Report an SDK-originated exception. Best-effort; never raises. + + Only call this for errors that originate in Deepgram SDK code (API + errors, transport failures) — never arbitrary host-application errors. + """ + if self._handler is not None: + try: + self._handler(exc, tags or {}) + except Exception: + pass + return + + if self._client is None: + return + + try: + from sentry_sdk.utils import ( # type: ignore[import-not-found] + event_from_exception, + ) + + event, hint = event_from_exception(exc, client_options=self._client.options) + event["tags"] = self._merge_tags(tags) + self._client.capture_event(event, hint=hint) + except Exception: + pass + + def capture_message( + self, message: str, *, level: str = "error", tags: Optional[Dict[str, str]] = None + ) -> None: + """Report a non-exception signal (e.g. an HTTP 5xx). Never raises. + + Custom handlers are exception-oriented, so messages only go to Sentry; + a handler-backed sink (no Sentry client) drops these silently. + """ + if self._client is None: + return + try: + event = {"message": message, "level": level, "tags": self._merge_tags(tags)} + self._client.capture_event(event) + except Exception: + pass + + +def init_telemetry( + *, + opt_out: bool, + handler: Optional[TelemetryHandler] = None, + version: str = "unknown", +) -> Optional[TelemetrySink]: + """Build this client's telemetry sink, or ``None`` if telemetry is inert. + + Per-client by construction: the returned sink is owned by the caller, so a + handler set on one client never receives another client's events, and one + client's opt-out never suppresses (or arms) another's. Never raises. + """ + base_tags = _build_base_tags(version) + + # A custom handler fully replaces Sentry, needs no DSN, and is scoped to + # this sink alone. Checked before opt-out/DSN because it is the caller's + # explicit request to receive their own SDK's errors in-process. + if handler is not None: + return TelemetrySink(handler=handler, base_tags=base_tags) + + if not is_enabled(opt_out): + return None + + dsn = _resolve_dsn() + if dsn is None: # pragma: no cover - is_enabled already guaranteed this + return None + + client = _get_or_build_client(dsn, version) + if client is None: + return None + return TelemetrySink(client=client, base_tags=base_tags) + + +def _flush_on_exit() -> None: + """Flush queued events before process exit (best-effort, 2s per client).""" + for client in list(_live_clients): + try: + client.flush(timeout=2.0) + except Exception: + pass + + +def _reset_for_tests() -> None: + """Reset process-wide telemetry state. Test-only.""" + global _flush_registered + _client_cache.clear() + _live_clients.clear() + _flush_registered = False diff --git a/src/deepgram/telemetry/http_hooks.py b/src/deepgram/telemetry/http_hooks.py new file mode 100644 index 00000000..288a7ae8 --- /dev/null +++ b/src/deepgram/telemetry/http_hooks.py @@ -0,0 +1,104 @@ +"""Attach error telemetry to the SDK's httpx client without touching generated code. + +The Fern-generated wrapper builds its own httpx client and stores it at +``client_wrapper.httpx_client.httpx_client``. Rather than replace that client +(which would risk changing timeout/pool/proxy behaviour), we append a +``response`` event hook to the instance the wrapper already built. The hook +reports server-side failures (HTTP 5xx) with the request id and session id, via +the caller's :class:`~deepgram.telemetry.client.TelemetrySink`. + +Transport-level exceptions (connection resets, timeouts) are not visible to a +response hook and are a deliberate follow-up (they need a transport/send wrap). +""" + +from __future__ import annotations + +from typing import Any, Callable, Mapping, Optional + +import httpx + +# Server-side / SDK-actionable failures. 4xx (bad key, bad params) are user +# errors and are intentionally not reported to avoid noise. +HTTP_CAPTURE_MIN_STATUS = 500 + +# Deepgram returns a request id under one of these response headers. +_REQUEST_ID_HEADERS = ("dg-request-id", "x-dg-request-id", "request-id", "x-request-id") + +# Marker set on a raw httpx client once instrumented, so repeat installs are a +# no-op instead of stacking duplicate hooks (see ``install_response_capture``). +_INSTALLED_FLAG = "_dg_telemetry_installed" + + +def _extract_request_id(headers: Mapping[str, str]) -> Optional[str]: + for name in _REQUEST_ID_HEADERS: + value = headers.get(name) + if value: + return value + return None + + +def _make_recorder(sink: Any, session_id: str) -> Callable[[httpx.Response], None]: + def record(response: httpx.Response) -> None: + status = response.status_code + if status < HTTP_CAPTURE_MIN_STATUS: + return + tags = {"http.status": str(status), "session_id": session_id} + request_id = _extract_request_id(response.headers) + if request_id: + tags["request_id"] = request_id + try: + # Path only — the query string can carry keys/params. + tags["http.path"] = response.request.url.path + except Exception: + pass + sink.capture_message(f"HTTP {status} from Deepgram API", tags=tags) + + return record + + +def install_response_capture(sink: Any, client_wrapper: Any, session_id: str) -> bool: + """Append the response hook to the wrapper's raw httpx client. + + Sync vs async is detected from the client type. Truly idempotent (a second + call on an already-instrumented client is a no-op, not a duplicate hook) and + total: returns whether a hook is now in place, and never raises. + """ + try: + wrapper = getattr(client_wrapper, "httpx_client", None) + raw = getattr(wrapper, "httpx_client", None) + if not isinstance(raw, (httpx.Client, httpx.AsyncClient)): + return False + + # Idempotency guard: never stack a second hook on the same client. + if getattr(raw, _INSTALLED_FLAG, False): + return True + + record = _make_recorder(sink, session_id) + + if isinstance(raw, httpx.AsyncClient): + + async def hook(response: httpx.Response) -> None: + try: + record(response) + except Exception: + pass + else: + + def hook(response: httpx.Response) -> None: # type: ignore[misc] + try: + record(response) + except Exception: + pass + + hooks = raw.event_hooks + hooks.setdefault("response", []) + hooks["response"].append(hook) + try: + setattr(raw, _INSTALLED_FLAG, True) + except Exception: + # If the client rejects the marker attribute we still installed the + # hook; worst case a later call adds a second one. Don't fail here. + pass + return True + except Exception: + return False diff --git a/src/deepgram/telemetry/scrub.py b/src/deepgram/telemetry/scrub.py new file mode 100644 index 00000000..df7e9aec --- /dev/null +++ b/src/deepgram/telemetry/scrub.py @@ -0,0 +1,121 @@ +"""PII/secret scrubbing for SDK telemetry events. + +An SDK runs inside customer applications, so the scrubbing bar is far higher +than for a first-party CLI: audio bytes, API keys / access tokens, request and +response bodies, and file paths must never leave the customer's process. This +module is a defense-in-depth ``before_send`` hook layered on top of the Sentry +client options set in ``client.py`` (``send_default_pii=False``, +``include_local_variables=False``, ``max_request_body_size="never"``). + +Mirrors the redaction discipline already established in ``_secure_logging.py``. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any, Dict + +if TYPE_CHECKING: + from sentry_sdk.types import Event, Hint # type: ignore[import-not-found] + +# Header names that may carry credentials or identifying data. Compared +# case-insensitively. +_SENSITIVE_HEADERS = frozenset( + { + "authorization", + "x-api-key", + "api-key", + "cookie", + "set-cookie", + "proxy-authorization", + } +) + +# The user-identifying home-directory segment of a filesystem path: +# /Users//... (macOS) +# /home//... (Linux) +# C:\Users\\... (Windows) +# We keep the rest of the path (useful for debugging and for Sentry code +# mappings, which key off the in-package suffix) and redact only the username, +# which is the personal-data bit. +_HOME_DIR_RE = re.compile(r"([\\/](?:Users|home)[\\/])[^\\/]+", re.IGNORECASE) + + +def _redact_path(path: Any) -> Any: + if not isinstance(path, str): + return path + return _HOME_DIR_RE.sub(r"\1", path) + + +def _scrub_frame_paths(event: "Event") -> None: + """Redact the home-dir username from every stack-frame path in the event. + + Stack frames carry ``abs_path``/``filename`` like + ``/Users/jane.doe/src/acme/main.py``. Client-level scrubbing + (``include_local_variables=False``) does nothing about these, because the + path is part of the frame metadata, not a local variable. + """ + stacktraces = [] + for group in ("exception", "threads"): + container = event.get(group) + if isinstance(container, dict): + for value in container.get("values") or []: + if isinstance(value, dict) and isinstance(value.get("stacktrace"), dict): + stacktraces.append(value["stacktrace"]) + if isinstance(event.get("stacktrace"), dict): + stacktraces.append(event["stacktrace"]) + + for stacktrace in stacktraces: + for frame in stacktrace.get("frames") or []: + if not isinstance(frame, dict): + continue + for key in ("abs_path", "filename"): + if key in frame: + frame[key] = _redact_path(frame[key]) + + +def scrub_event(event: "Event", _hint: "Hint") -> "Event | None": + """Strip request bodies, headers, query strings, and user-identifying data. + + Returns the mutated event, or ``None`` to drop it entirely. Kept total and + exception-free: a scrubber that raises would take telemetry (and any code + path that flushes it) down with it, so callers must never see an error here. + """ + try: + request: Dict[str, Any] = event.get("request") or {} + # Bodies and query strings can contain audio, transcripts, keys, or URLs. + if "data" in request: + request["data"] = "[Filtered]" + if "query_string" in request: + request["query_string"] = "[Filtered]" + if "cookies" in request: + request["cookies"] = {} + # Drop credential-bearing headers; keep the rest for debugging value. + headers = request.get("headers") + if isinstance(headers, dict): + request["headers"] = { + k: ("[Filtered]" if k.lower() in _SENSITIVE_HEADERS else v) + for k, v in headers.items() + } + + # Never attribute events to an end user. Drop geo too — it is inferred + # from the sending IP and identifies the customer's location. + user: Dict[str, Any] = event.get("user") or {} + for key in ("email", "ip_address", "username", "id", "geo"): + user.pop(key, None) + + # Sentry auto-attaches the machine hostname; on a customer's machine + # that leaks their host/developer name, so strip it. + event.pop("server_name", None) + + # Extra/context blobs are developer-controlled and may echo payloads. + event.pop("extra", None) + + # Stack-frame paths embed the OS home directory, which carries the + # developer's username (e.g. /Users/jane.doe/...). Redact that segment. + _scrub_frame_paths(event) + except Exception: + # If scrubbing itself fails, drop the event rather than risk a leak. + return None + + return event diff --git a/tests/custom/test_telemetry.py b/tests/custom/test_telemetry.py new file mode 100644 index 00000000..06c3d278 --- /dev/null +++ b/tests/custom/test_telemetry.py @@ -0,0 +1,291 @@ +"""Tests for the hand-written opt-out telemetry module. + +Covers the SDK-specific risk areas: +- opt-out resolution (env kill-switch, flag, no-DSN inertness), +- PII/secret scrubbing, +- library isolation (a custom handler fully replaces Sentry), +- per-client isolation (one client's opt-out or handler never leaks to another). + +Most of these do not require ``sentry-sdk`` to be installed; the no-DSN default +keeps Sentry inert, so the module no-ops cleanly on the core paths. +""" + +import httpx +import pytest + +from deepgram.telemetry import client as telemetry +from deepgram.telemetry import http_hooks +from deepgram.telemetry.client import TelemetrySink +from deepgram.telemetry.scrub import scrub_event + + +@pytest.fixture(autouse=True) +def _reset(): + telemetry._reset_for_tests() + yield + telemetry._reset_for_tests() + + +class _RecordingSink: + """Minimal sink double that records capture_message calls.""" + + def __init__(self): + self.messages = [] + + def capture_message(self, message, **kwargs): + self.messages.append((message, kwargs)) + + +class TestOptOutResolution: + def test_disabled_by_default_without_dsn(self): + # No DSN embedded/supplied -> inert even when not opted out. + assert telemetry.is_enabled(opt_out=False) is False + + def test_opt_out_flag_disables(self, monkeypatch): + monkeypatch.setenv(telemetry.DSN_ENV_VAR, "https://k@o1.ingest.us.sentry.io/1") + assert telemetry.is_enabled(opt_out=True) is False + + def test_env_kill_switch_wins(self, monkeypatch): + monkeypatch.setenv(telemetry.DSN_ENV_VAR, "https://k@o1.ingest.us.sentry.io/1") + monkeypatch.setenv(telemetry.DISABLE_ENV_VAR, "1") + assert telemetry.is_enabled(opt_out=False) is False + + def test_enabled_with_real_dsn_and_no_opt_out(self, monkeypatch): + monkeypatch.setenv(telemetry.DSN_ENV_VAR, "https://k@o1.ingest.us.sentry.io/1") + assert telemetry.is_enabled(opt_out=False) is True + + def test_placeholder_dsn_is_not_real(self, monkeypatch): + monkeypatch.setenv( + telemetry.DSN_ENV_VAR, "https://PLACEHOLDER@x.ingest.us.sentry.io/0" + ) + assert telemetry.is_enabled(opt_out=False) is False + + +class TestEnvironmentTag: + def test_environment_defaults_to_unset(self, monkeypatch): + # Never hard-code "production": the SDK runs in customer dev/CI/staging. + monkeypatch.delenv(telemetry.ENVIRONMENT_ENV_VAR, raising=False) + assert telemetry._resolve_environment() is None + + def test_environment_honours_override(self, monkeypatch): + monkeypatch.setenv(telemetry.ENVIRONMENT_ENV_VAR, "staging") + assert telemetry._resolve_environment() == "staging" + + +class TestScrubbing: + def test_strips_secrets_and_pii(self): + event = { + "request": { + "headers": {"Authorization": "Bearer secret", "X-Trace": "keep"}, + "data": "raw-audio-bytes", + "query_string": "key=abc123", + "cookies": {"sid": "x"}, + }, + "user": { + "email": "a@b.com", + "id": "u1", + "ip_address": "1.2.3.4", + "geo": {"country_code": "GB"}, + }, + "server_name": "customers-laptop", + "extra": {"payload": "transcript"}, + } + out = scrub_event(event, {}) + assert out is not None + req = out["request"] + assert req["headers"]["Authorization"] == "[Filtered]" + assert req["headers"]["X-Trace"] == "keep" + assert req["data"] == "[Filtered]" + assert req["query_string"] == "[Filtered]" + assert req["cookies"] == {} + assert out["user"] == {} + assert "server_name" not in out + assert "extra" not in out + + def test_never_raises_on_malformed_event(self): + # Non-dict headers must not blow up the scrubber. + assert scrub_event({"request": {"headers": "oops"}}, {}) is not None + + def test_redacts_home_dir_username_from_stack_frames(self): + event = { + "exception": { + "values": [ + { + "stacktrace": { + "frames": [ + { + "abs_path": "/Users/jane.doe/src/acme/main.py", + "filename": "acme/main.py", + }, + {"abs_path": "/home/bob/.venv/lib/deepgram/client.py"}, + {"abs_path": "C:\\Users\\jane\\app.py"}, + {"abs_path": "/opt/app/no_home_here.py"}, + ] + } + } + ] + }, + "threads": { + "values": [ + {"stacktrace": {"frames": [{"abs_path": "/Users/carol/t.py"}]}} + ] + }, + } + out = scrub_event(event, {}) + frames = out["exception"]["values"][0]["stacktrace"]["frames"] + assert frames[0]["abs_path"] == "/Users//src/acme/main.py" + assert frames[0]["filename"] == "acme/main.py" # no home segment, untouched + assert frames[1]["abs_path"] == "/home//.venv/lib/deepgram/client.py" + assert frames[2]["abs_path"] == "C:\\Users\\\\app.py" + assert frames[3]["abs_path"] == "/opt/app/no_home_here.py" # unchanged + thread_frame = out["threads"]["values"][0]["stacktrace"]["frames"][0] + assert thread_frame["abs_path"] == "/Users//t.py" + + +class TestIsolation: + def test_custom_handler_replaces_sentry(self): + received = [] + + def handler(exc, tags): + received.append((exc, tags)) + + sink = telemetry.init_telemetry( + opt_out=False, handler=handler, version="9.9.9" + ) + assert sink is not None + + err = ValueError("boom") + sink.capture_exception(err, tags={"request_id": "r1"}) + + assert len(received) == 1 + assert received[0][0] is err + assert received[0][1]["request_id"] == "r1" + + def test_no_sink_is_safe(self): + # No DSN, no handler: init returns None and an empty sink no-ops. + assert telemetry.init_telemetry(opt_out=False, version="9.9.9") is None + TelemetrySink().capture_exception(ValueError("boom")) # must not raise + + def test_handler_errors_are_swallowed(self): + def bad_handler(exc, tags): + raise RuntimeError("handler blew up") + + sink = telemetry.init_telemetry( + opt_out=False, handler=bad_handler, version="9.9.9" + ) + assert sink is not None + sink.capture_exception(ValueError("boom")) # must not propagate + + +class TestPerClientIsolation: + """Regression tests for the module-global state bugs. + + Historically ``init_telemetry`` short-circuited on a module-global + ``_initialized`` flag and stored the handler in a module global, so a + second client's opt-out was ignored and one client's handler caught every + client's events. Sinks are now per-client, so neither can happen. + """ + + def test_opt_out_is_honoured_per_client_regardless_of_order( + self, monkeypatch + ): + monkeypatch.setenv(telemetry.DSN_ENV_VAR, "https://k@o1.ingest.us.sentry.io/1") + # Avoid needing sentry-sdk installed: stub the shared client builder. + monkeypatch.setattr( + telemetry, "_get_or_build_client", lambda dsn, version: object() + ) + + # An opted-in client arms; a later opted-out client stays inert... + armed_first = telemetry.init_telemetry(opt_out=False, version="1.0.0") + opted_out_second = telemetry.init_telemetry(opt_out=True, version="1.0.0") + assert armed_first is not None + assert opted_out_second is None + + # ...and the reverse order behaves the same (no sticky init state). + opted_out_first = telemetry.init_telemetry(opt_out=True, version="1.0.0") + armed_second = telemetry.init_telemetry(opt_out=False, version="1.0.0") + assert opted_out_first is None + assert armed_second is not None + + def test_handler_does_not_leak_across_clients(self): + received_a, received_b = [], [] + sink_a = telemetry.init_telemetry( + opt_out=False, handler=lambda exc, tags: received_a.append(exc), version="1" + ) + sink_b = telemetry.init_telemetry( + opt_out=False, handler=lambda exc, tags: received_b.append(exc), version="1" + ) + assert sink_a is not None and sink_b is not None + + err = ValueError("only for B") + sink_b.capture_exception(err) + + assert received_b == [err] + assert received_a == [] # A's handler must not see B's events + + +class TestResponseHook: + def _response(self, status, headers=None): + request = httpx.Request("GET", "https://api.deepgram.com/v1/listen?model=nova") + return httpx.Response(status, headers=headers or {}, request=request) + + def test_5xx_captured_with_request_id_and_no_query(self): + sink = _RecordingSink() + record = http_hooks._make_recorder(sink, "sess-1") + record(self._response(503, {"dg-request-id": "req-42"})) + + assert len(sink.messages) == 1 + tags = sink.messages[0][1]["tags"] + assert tags["http.status"] == "503" + assert tags["request_id"] == "req-42" + assert tags["session_id"] == "sess-1" + assert tags["http.path"] == "/v1/listen" # query stripped + + def test_4xx_not_captured(self): + sink = _RecordingSink() + http_hooks._make_recorder(sink, "s")(self._response(401)) + assert sink.messages == [] + + def test_install_appends_hook_to_httpx_client(self): + raw = httpx.Client() + wrapper = type("W", (), {"httpx_client": type("H", (), {"httpx_client": raw})()})() + try: + assert http_hooks.install_response_capture(_RecordingSink(), wrapper, "s") is True + assert len(raw.event_hooks.get("response", [])) == 1 + finally: + raw.close() + + def test_install_is_idempotent(self): + # A second install must not stack a duplicate response hook. + raw = httpx.Client() + wrapper = type("W", (), {"httpx_client": type("H", (), {"httpx_client": raw})()})() + try: + assert http_hooks.install_response_capture(_RecordingSink(), wrapper, "s") is True + assert http_hooks.install_response_capture(_RecordingSink(), wrapper, "s") is True + assert len(raw.event_hooks.get("response", [])) == 1 + finally: + raw.close() + + def test_install_is_safe_when_no_client(self): + wrapper = type("W", (), {"httpx_client": None})() + assert http_hooks.install_response_capture(_RecordingSink(), wrapper, "s") is False + + +class TestSentryIsolation: + """Requires sentry-sdk; validates we never touch the host app's global Sentry.""" + + def test_isolated_client_leaves_global_untouched(self, monkeypatch): + sentry_sdk = pytest.importorskip("sentry_sdk") + monkeypatch.setenv( + telemetry.DSN_ENV_VAR, "https://public@o0.ingest.us.sentry.io/1" + ) + + sink = telemetry.init_telemetry(opt_out=False, version="7.5.1") + assert sink is not None + # Our private client is configured and active (sends events)... + assert sink._client is not None + assert sink._client.is_active() is True + # ...but the process-global Sentry client is never configured by us. + assert sentry_sdk.get_client().dsn is None + # And capturing through the isolated client must not raise. + sink.capture_exception(ValueError("boom"), tags={"request_id": "r1"})