From c1493503e67c8ec849f6d31889c434e6e495721d Mon Sep 17 00:00:00 2001 From: Deborah Jacob Date: Sun, 19 Apr 2026 11:38:14 -0700 Subject: [PATCH 1/3] Docs: content capture + set_correlation API --- .vscode/settings.json | 35 ++ docs/api/configuration.md | 8 +- docs/api/decorators.md | 2 + docs/concepts/architecture.md | 7 +- docs/concepts/context-propagation.md | 57 ++-- docs/concepts/run-context.md | 20 +- docs/getting-started/configuration.md | 237 ++++++------- docs/getting-started/installation.md | 4 +- docs/getting-started/quickstart.md | 49 ++- docs/index.md | 20 +- docs/integration/collector.md | 16 +- docs/integration/existing-otel.md | 331 ++++++++++++++---- docs/patterns/best-practices.md | 23 +- docs/tracking/content-capture.md | 146 ++++++++ docs/tracking/llm-tracking.md | 29 ++ docs/tracking/outcomes.md | 463 +++++++------------------- 16 files changed, 836 insertions(+), 611 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 docs/tracking/content-capture.md diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a296890 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,35 @@ +{ + "// PURPOSE": "Makes Cursor / VS Code show ruff + mypy warnings inline while you edit. Install the 'Ruff' and 'Python' extensions from the Cursor extension panel for full effect.", + + "python.analysis.typeCheckingMode": "basic", + "python.analysis.autoImportCompletions": true, + "python.analysis.diagnosticMode": "workspace", + + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.ruff": "explicit", + "source.organizeImports.ruff": "explicit" + } + }, + + "ruff.enable": true, + "ruff.importStrategy": "fromEnvironment", + "ruff.lint.run": "onType", + "ruff.nativeServer": "on", + + "files.exclude": { + "**/__pycache__": true, + "**/.mypy_cache": true, + "**/.ruff_cache": true, + "**/.pytest_cache": true + }, + + "files.associations": { + "*.toml": "toml" + }, + "cursorpyright.analysis.autoImportCompletions": true, + "cursorpyright.analysis.diagnosticMode": "workspace", + "cursorpyright.analysis.typeCheckingMode": "basic" +} diff --git a/docs/api/configuration.md b/docs/api/configuration.md index b5f8901..c9acb48 100644 --- a/docs/api/configuration.md +++ b/docs/api/configuration.md @@ -17,13 +17,15 @@ from botanu.sdk.config import BotanuConfig | `service_namespace` | `str` | From env | Service namespace | | `deployment_environment` | `str` | From env / `"production"` | Deployment environment | | `auto_detect_resources` | `bool` | `True` | Auto-detect cloud resources | -| `otlp_endpoint` | `str` | From env / `"http://localhost:4318"` | OTLP endpoint | -| `otlp_headers` | `dict` | `None` | Custom headers for OTLP exporter | +| `api_key` | `str` | From env (`BOTANU_API_KEY`) | Auto-configures the endpoint to `https://ingest.botanu.ai` and attaches a bearer token on botanu-trusted hosts only | +| `otlp_endpoint` | `str` | From env / auto-configured from `api_key` / `"http://localhost:4318"` | OTLP endpoint | +| `otlp_headers` | `dict` | `None` | Custom headers for OTLP exporter — always honored | +| `content_capture_rate` | `float` | `0.0` | Prompt/response capture rate (0.0–1.0). See the [Content Capture doc](../tracking/content-capture.md). | | `max_export_batch_size` | `int` | `512` | Max spans per batch | | `max_queue_size` | `int` | `65536` | Max spans in queue (~64 MB at ~1 KB/span) | | `schedule_delay_millis` | `int` | `5000` | Delay between batch exports | | `export_timeout_millis` | `int` | `30000` | Timeout for export operations | -| `propagation_mode` | `str` | `"lean"` | `"lean"` or `"full"` | +| `propagation_mode` | `str` | `"lean"` | `"full"` (recommended) or `"lean"` (deprecated — will be removed) | | `auto_instrument_packages` | `list` | See below | Packages to auto-instrument | ### Constructor diff --git a/docs/api/decorators.md b/docs/api/decorators.md index e88c971..31591be 100644 --- a/docs/api/decorators.md +++ b/docs/api/decorators.md @@ -12,6 +12,7 @@ from botanu import botanu_workflow *, event_id: Union[str, Callable[..., str]], customer_id: Union[str, Callable[..., str]], + step: Optional[str] = None, environment: Optional[str] = None, tenant_id: Optional[str] = None, auto_outcome_on_success: bool = True, @@ -26,6 +27,7 @@ from botanu import botanu_workflow | `name` | `str` | Required | Workflow name (low cardinality, e.g. `"Customer Support"`) | | `event_id` | `str \| Callable` | Required | Business transaction identifier (e.g. ticket ID). Can be a static string or a callable that receives the same `(*args, **kwargs)` as the decorated function. | | `customer_id` | `str \| Callable` | Required | End-customer being served (e.g. org ID). Same static/callable rules as `event_id`. | +| `step` | `str` | `None` | Step name within a multi-step workflow (e.g. `"classify"`, `"research"`). ⚠ Accepted and stored on `RunContext` but **not yet emitted as a span attribute** — the collector servicegraph work that consumes it has not landed. Forward-compatible. | | `environment` | `str` | From env | Deployment environment | | `tenant_id` | `str` | `None` | Tenant identifier for multi-tenant systems | | `auto_outcome_on_success` | `bool` | `True` | Emit `"success"` outcome if no exception | diff --git a/docs/concepts/architecture.md b/docs/concepts/architecture.md index 5c4e366..2e323af 100644 --- a/docs/concepts/architecture.md +++ b/docs/concepts/architecture.md @@ -43,10 +43,15 @@ class BotanuConfig: service_name: str deployment_environment: str otlp_endpoint: str - propagation_mode: str # "lean" or "full" + api_key: str # BOTANU_API_KEY; auto-configures endpoint + bearer + content_capture_rate: float # 0.0 default; see Content Capture + propagation_mode: str # "full" (recommended); "lean" is deprecated auto_instrument_packages: List[str] ``` +Content capture for eval (prompts/responses) is opt-in and disabled by +default. See [Content Capture](../tracking/content-capture.md). + ### RunContext Holds run metadata and provides serialization: diff --git a/docs/concepts/context-propagation.md b/docs/concepts/context-propagation.md index 9bea134..6187dc1 100644 --- a/docs/concepts/context-propagation.md +++ b/docs/concepts/context-propagation.md @@ -19,37 +19,45 @@ When you make an outbound HTTP request, the `botanu.run_id` travels in the `bagg ## Propagation Modes -### Lean Mode (Default) +### Full Mode (recommended) + +Full mode is the durable direction — every cross-service call carries the +complete baggage needed to reconstruct run context downstream. The SDK +propagates exactly these seven keys (defined as `BAGGAGE_KEYS_FULL` in +[`src/botanu/processors/enricher.py`](../../src/botanu/processors/enricher.py)): -Only propagates essential fields to minimize header size: - `botanu.run_id` - `botanu.workflow` - `botanu.event_id` - `botanu.customer_id` - -```python -# Lean mode baggage (~120 bytes) -baggage: botanu.run_id=019abc12-def3-7890-abcd-1234567890ab,botanu.workflow=process,botanu.event_id=evt-001,botanu.customer_id=cust-456 -``` - -### Full Mode - -Propagates all context fields. In addition to the lean fields, full mode adds: - `botanu.environment` - `botanu.tenant_id` - `botanu.parent_run_id` -- `botanu.root_run_id` -- `botanu.attempt` -- `botanu.retry_of_run_id` -- `botanu.deadline` -- `botanu.cancelled` + +Enable it explicitly: + +```bash +export BOTANU_PROPAGATION_MODE=full +``` ```python -# Enable full mode -import os -os.environ["BOTANU_PROPAGATION_MODE"] = "full" +# Full mode baggage (~250 bytes, values-dependent) +baggage: botanu.run_id=019abc12-...,botanu.workflow=process,botanu.event_id=evt-001,botanu.customer_id=cust-456,botanu.environment=production,botanu.tenant_id=tnt-abc,botanu.parent_run_id=019abc11-... ``` +Fields that live on `RunContext` but **not** in baggage — `root_run_id`, +`attempt`, `retry_of_run_id`, `deadline`, `cancelled` — are reconstructed +from local state, not carried on the wire. If you need them downstream, +propagate them yourself via your message envelope (see "Message Queue +Propagation" below). + +### Lean Mode (deprecated — will be removed) + +Lean mode propagates only the first four keys from the full list. It was +the default in early 0.x releases and is still accepted for backward +compatibility, but it is **deprecated** — full mode will become the only +mode in a future release. Do not build new services assuming lean mode. + ## In-Process Propagation Within a single process, context is propagated via Python's `contextvars`: @@ -181,12 +189,13 @@ The same `run_id` flows through all services, enabling: ## Baggage Size Limits -W3C Baggage has practical size limits. The SDK uses lean mode by default to stay well under these limits: +W3C Baggage has practical size limits (most intermediaries allow 8 KB, but +individual hops may clip earlier). Typical sizes for botanu baggage: -| Mode | Typical Size | Recommendation | -|------|--------------|----------------| -| Lean | ~120 bytes | Use for most cases | -| Full | ~350 bytes | Use when you need all context downstream | +| Mode | Typical size | Notes | +| --- | --- | --- | +| Full (recommended) | ~250 bytes | 7 keys, well under any limit | +| Lean (deprecated) | ~120 bytes | 4 keys, historical only | ## Propagation and Auto-Instrumentation diff --git a/docs/concepts/run-context.md b/docs/concepts/run-context.md index 8e752d1..249628d 100644 --- a/docs/concepts/run-context.md +++ b/docs/concepts/run-context.md @@ -197,16 +197,22 @@ emit_outcome( ### To Baggage (for HTTP propagation) ```python -# Lean mode (default): essential fields -baggage = ctx.to_baggage_dict() -# {"botanu.run_id": "...", "botanu.workflow": "...", "botanu.event_id": "...", "botanu.customer_id": "..."} - -# Full mode: all fields +# Full mode (recommended): all seven baggage keys baggage = ctx.to_baggage_dict(lean_mode=False) -# Adds: botanu.environment, botanu.tenant_id, botanu.parent_run_id, botanu.root_run_id, -# botanu.attempt, botanu.retry_of_run_id, botanu.deadline, botanu.cancelled +# {"botanu.run_id": "...", "botanu.workflow": "...", "botanu.event_id": "...", +# "botanu.customer_id": "...", "botanu.environment": "...", +# "botanu.tenant_id": "...", "botanu.parent_run_id": "..."} + +# Lean mode (deprecated — will be removed): first four keys only +baggage = ctx.to_baggage_dict(lean_mode=True) +# {"botanu.run_id": "...", "botanu.workflow": "...", +# "botanu.event_id": "...", "botanu.customer_id": "..."} ``` +Fields that live on `RunContext` but are **not** in baggage — +`root_run_id`, `attempt`, `retry_of_run_id`, `deadline`, `cancelled` — +are reconstructed from local state. + ### To Span Attributes ```python diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index cd1e4bd..6d8bd63 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -1,36 +1,69 @@ # Configuration -Botanu SDK can be configured through code, environment variables, or YAML files. +botanu SDK can be configured with a single environment variable (the common +case) or through code / YAML for everything else. -## Configuration Precedence +## Simplest config — just the API key -1. **Code arguments** (explicit values passed to `BotanuConfig`) -2. **Environment variables** (`BOTANU_*`, `OTEL_*`) -3. **YAML config file** (`botanu.yaml` or specified path) -4. **Built-in defaults** +For the Botanu Cloud SaaS, a single env var is enough: -## Quick Configuration +```bash +export BOTANU_API_KEY= +python your_app.py +``` + +When `BOTANU_API_KEY` is set, `enable()` auto-configures the OTLP endpoint +to `https://ingest.botanu.ai` and attaches the key as a bearer token on the +exporter. You do not need to set `OTEL_EXPORTER_OTLP_ENDPOINT` too. + +## ⚠ The API key is only sent to botanu-trusted endpoints + +If you override the OTLP endpoint to something that isn't owned by +botanu — e.g., you point OTLP at Datadog, Honeycomb, or a self-hosted +collector — **the SDK will not attach your botanu API key to the +exporter's Authorization header**. It is silently dropped. This is to +prevent a misconfigured `OTEL_EXPORTER_OTLP_ENDPOINT` from leaking your +tenant credentials to a third-party backend. + +The trusted endpoint list is hard-coded in +[`src/botanu/sdk/config.py`](../../src/botanu/sdk/config.py): + +- any host ending in `.botanu.ai` (e.g., `ingest.botanu.ai`) +- `localhost`, `127.0.0.1`, `::1`, `0.0.0.0` (local development) -### Code-Based +For any other endpoint the exporter runs without a bearer header. If you +are shipping OTLP to a non-botanu backend and want auth on that exporter, +set `OTEL_EXPORTER_OTLP_HEADERS` or pass `otlp_headers=` to `BotanuConfig` +explicitly — those headers are always honored. + +## Configuration precedence + +1. **Code arguments** passed to `enable()` or `BotanuConfig(...)`. +2. **Environment variables** (`BOTANU_*`, `OTEL_*`). +3. **YAML config file** (`botanu.yaml` or a path you pass). +4. **Built-in defaults.** + +## Code-based ```python from botanu import enable enable( service_name="my-service", - otlp_endpoint="http://collector:4318/v1/traces", + otlp_endpoint="https://ingest.botanu.ai", ) ``` -### Environment Variables +## Environment variables ```bash +export BOTANU_API_KEY= # enough on its own for Botanu Cloud export OTEL_SERVICE_NAME=my-service -export OTEL_EXPORTER_OTLP_ENDPOINT=http://collector:4318 export BOTANU_ENVIRONMENT=production +export BOTANU_CONTENT_CAPTURE_RATE=0.10 # see Content Capture docs ``` -### YAML File +## YAML file ```yaml # botanu.yaml @@ -40,10 +73,13 @@ service: environment: production otlp: - endpoint: http://collector:4318/v1/traces + endpoint: https://ingest.botanu.ai + +content: + capture_rate: 0.10 propagation: - mode: lean + mode: full ``` Load with: @@ -54,9 +90,7 @@ from botanu.sdk.config import BotanuConfig config = BotanuConfig.from_yaml("botanu.yaml") ``` -## Full Configuration Reference - -### BotanuConfig Fields +## Full `BotanuConfig` fields ```python from dataclasses import dataclass @@ -74,7 +108,10 @@ class BotanuConfig: # OTLP exporter otlp_endpoint: str = None # OTEL_EXPORTER_OTLP_ENDPOINT - otlp_headers: dict = None # Custom headers for auth + otlp_headers: dict = None # Custom headers (always honored) + + # API key (auto-configures endpoint + Authorization header on trusted hosts) + api_key: str = None # BOTANU_API_KEY # Span export max_export_batch_size: int = 512 @@ -82,134 +119,88 @@ class BotanuConfig: schedule_delay_millis: int = 5000 export_timeout_millis: int = 30000 - # Propagation mode - propagation_mode: str = "lean" # BOTANU_PROPAGATION_MODE + # Content capture for eval (0.0 disables; see Content Capture doc) + content_capture_rate: float = 0.0 # BOTANU_CONTENT_CAPTURE_RATE - # Auto-instrumentation - auto_instrument_packages: list = [...] + # Propagation mode — "full" is the target. "lean" is deprecated. + propagation_mode: str = "lean" # BOTANU_PROPAGATION_MODE ``` -## Environment Variables +## Environment variable reference -### OpenTelemetry Standard Variables +### OpenTelemetry standard | Variable | Description | Default | -|----------|-------------|---------| +| --- | --- | --- | | `OTEL_SERVICE_NAME` | Service name | `unknown_service` | | `OTEL_SERVICE_VERSION` | Service version | None | | `OTEL_SERVICE_NAMESPACE` | Service namespace | None | | `OTEL_DEPLOYMENT_ENVIRONMENT` | Environment name | `production` | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP collector base URL | `http://localhost:4318` | -| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | OTLP traces endpoint (full URL) | None | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP collector base URL | Auto-set to `https://ingest.botanu.ai` when `BOTANU_API_KEY` is set | +| `OTEL_EXPORTER_OTLP_HEADERS` | Extra OTLP headers | None | -### Botanu-Specific Variables +### botanu-specific | Variable | Description | Default | -|----------|-------------|---------| +| --- | --- | --- | +| `BOTANU_API_KEY` | API key for Botanu Cloud. Auto-configures the endpoint and bearer token on trusted hosts. | None | | `BOTANU_ENVIRONMENT` | Fallback for environment | `production` | -| `BOTANU_PROPAGATION_MODE` | `lean` or `full` | `lean` | +| `BOTANU_CONTENT_CAPTURE_RATE` | Content-capture sampling rate (0.0–1.0). See [Content Capture](../tracking/content-capture.md). | `0.0` | +| `BOTANU_PROPAGATION_MODE` | `full` (recommended) or `lean` (deprecated) | `lean` | | `BOTANU_AUTO_DETECT_RESOURCES` | Auto-detect cloud resources | `true` | | `BOTANU_CONFIG_FILE` | Path to YAML config | None | +| `BOTANU_COLLECTOR_ENDPOINT` | Override for OTLP endpoint (same behavior as `OTEL_EXPORTER_OTLP_ENDPOINT`) | None | -## YAML Configuration - -### Full Example - -```yaml -# botanu.yaml - Full configuration example -service: - name: ${OTEL_SERVICE_NAME:-my-service} - version: ${APP_VERSION:-1.0.0} - namespace: production - environment: ${ENVIRONMENT:-production} - -resource: - auto_detect: true - -otlp: - endpoint: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://localhost:4318}/v1/traces - headers: - Authorization: Bearer ${OTLP_AUTH_TOKEN} - -export: - batch_size: 512 - queue_size: 2048 - delay_ms: 5000 +## Content capture -propagation: - mode: lean - -auto_instrument_packages: - - requests - - httpx - - fastapi - - sqlalchemy - - openai_v2 -``` +Prompt / response capture for the evaluator is disabled by default. +Turn it on with `BOTANU_CONTENT_CAPTURE_RATE`: -### Environment Variable Interpolation - -The YAML loader supports two interpolation patterns: - -```yaml -# Simple interpolation -endpoint: ${COLLECTOR_URL} - -# With default value -endpoint: ${COLLECTOR_URL:-http://localhost:4318} +```bash +export BOTANU_CONTENT_CAPTURE_RATE=0.10 # 10% of calls captured ``` -### Loading Configuration +See [Content Capture](../tracking/content-capture.md) for the full +pipeline, the three capture points, and the PII-scrubbing chain. -```python -from botanu.sdk.config import BotanuConfig +## Propagation modes -# Explicit path -config = BotanuConfig.from_yaml("config/botanu.yaml") +botanu's durable direction is **full-mode only** — every cross-service +call carries the complete run context in W3C Baggage. `lean` mode is still +present in the SDK for backward compatibility but will be removed; do not +depend on it. -# Auto-discover (searches botanu.yaml, config/botanu.yaml) -config = BotanuConfig.from_file_or_env() +Set explicitly: -# Environment only -config = BotanuConfig() +```bash +export BOTANU_PROPAGATION_MODE=full ``` -## Propagation Modes - -### Lean Mode (Default) - -Propagates only essential fields to minimize header size: - -- `botanu.run_id` -- `botanu.workflow` -- `botanu.event_id` -- `botanu.customer_id` +See [Context Propagation](../concepts/context-propagation.md) for the +exact field list. -Best for high-traffic systems where header size matters. +## Zero-code initialization -### Full Mode +If you want `enable()` to run without a line of code, import the +`botanu.register` module at process start. It calls `enable()` under the +hood: -Propagates all context fields: - -- `botanu.run_id` -- `botanu.workflow` -- `botanu.event_id` -- `botanu.customer_id` -- `botanu.environment` -- `botanu.tenant_id` -- `botanu.parent_run_id` +```bash +python -c "import botanu.register" -m your_app +``` -Enable with: +Or, for containers, add `botanu.register` to your `PYTHONSTARTUP`: ```bash -export BOTANU_PROPAGATION_MODE=full +PYTHONSTARTUP=$(python -c "import botanu, os; print(os.path.dirname(botanu.__file__) + '/register.py')") ``` -## Auto-Instrumentation +This is useful when you cannot edit the entry point (e.g., a third-party +process runner). -### Default Packages +## Auto-instrumentation -By default, Botanu enables instrumentation for: +### Default packages ```python [ @@ -230,9 +221,7 @@ By default, Botanu enables instrumentation for: ] ``` -### Customizing Packages - -Override the default list via `BotanuConfig`: +### Customizing packages ```python from botanu import enable @@ -242,26 +231,16 @@ config = BotanuConfig(auto_instrument_packages=["requests", "fastapi", "openai_v enable(config=config) ``` -### Disabling Auto-Instrumentation +### Disabling ```python enable(auto_instrumentation=False) ``` -## Exporting Configuration - -```python -config = BotanuConfig( - service_name="my-service", - deployment_environment="production", -) - -# Export as dictionary -print(config.to_dict()) -``` - -## See Also +## See also -- [Architecture](../concepts/architecture.md) - SDK design principles -- [Collector Configuration](../integration/collector.md) - Collector setup -- [Existing OTel Setup](../integration/existing-otel.md) - Integration with existing OTel +- [Quickstart](quickstart.md) +- [Architecture](../concepts/architecture.md) +- [Collector](../integration/collector.md) +- [Existing OTel / Datadog setup](../integration/existing-otel.md) +- [Content Capture](../tracking/content-capture.md) diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 48837b7..8c070dd 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -22,7 +22,7 @@ Instrumentation packages are lightweight shims that silently no-op when the targ Set your API key as an environment variable. The SDK auto-configures the OTLP endpoint to `ingest.botanu.ai` — no other configuration needed. ```bash -export BOTANU_API_KEY="btnu_live_..." +export BOTANU_API_KEY="" ``` That's it. No collector to run, no infrastructure to deploy. Botanu hosts everything. @@ -56,7 +56,7 @@ FROM python:3.12-slim WORKDIR /app RUN pip install botanu COPY . . -ENV BOTANU_API_KEY="btnu_live_..." +ENV BOTANU_API_KEY="" CMD ["python", "app.py"] ``` diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index dad5081..c4efb96 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -5,7 +5,7 @@ Get event-level cost attribution working in 5 minutes. ## Prerequisites - Python 3.9+ -- OpenTelemetry Collector running (see [Collector Configuration](../integration/collector.md)) +- A botanu API key (sign up at [botanu.ai](https://botanu.ai)) ## Step 1: Install @@ -13,21 +13,29 @@ Get event-level cost attribution working in 5 minutes. pip install botanu ``` -## Step 2: Set Environment Variables +## Step 2: Set one environment variable ```bash -export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 -export OTEL_SERVICE_NAME=my-service +export BOTANU_API_KEY= ``` -Or in Docker / Kubernetes: +That's it for the Botanu Cloud SaaS. The SDK auto-configures the OTLP +endpoint to `https://ingest.botanu.ai` and attaches your API key as a +bearer token. + +### Alternative — self-hosted or local collector -```yaml -environment: - - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 - - OTEL_SERVICE_NAME=my-service +If you run your own OTel collector, point at it explicitly: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +export OTEL_SERVICE_NAME=my-service ``` +See [Collector](../integration/collector.md). Note: the SDK does not +attach your `BOTANU_API_KEY` to non-botanu endpoints — set +`OTEL_EXPORTER_OTLP_HEADERS` if your self-hosted collector needs auth. + ## Step 3: Enable SDK ```python @@ -38,6 +46,11 @@ enable() Call `enable()` once at application startup. It reads configuration from environment variables — no hardcoded values needed. +> **Already using Datadog or another OTel APM?** `enable()` auto-detects +> your existing TracerProvider and adds botanu alongside without +> disturbing your sampling ratio or APM bill. See [Using botanu with an +> existing OTel / APM setup](../integration/existing-otel.md). + ## Step 4: Define Entry Point ```python @@ -57,7 +70,7 @@ All LLM calls, database queries, and HTTP requests inside the function are autom **Entry service** (`entry/app.py`): ```python -from botanu import enable, botanu_workflow, emit_outcome +from botanu import enable, botanu_workflow enable() @@ -68,11 +81,12 @@ enable() ) async def handle_request(req): data = await fetch_data(req) - result = await process(data) - emit_outcome("success") - return result + return await process(data) ``` +No `emit_outcome("success")` call is needed — event outcome is resolved +server-side from eval verdict / HITL / SoR. See [Outcomes](../tracking/outcomes.md). + **Downstream service** (`intermediate/app.py`): ```python @@ -97,6 +111,9 @@ All spans across all services share the same `run_id`, enabling cost-per-event a ## Next Steps -- [Configuration](configuration.md) - Environment variables and YAML config -- [Kubernetes Deployment](../integration/kubernetes.md) - Zero-code instrumentation at scale -- [Context Propagation](../concepts/context-propagation.md) - How run_id flows across services +- [Configuration](configuration.md) — environment variables and YAML config +- [Using botanu with existing OTel / Datadog](../integration/existing-otel.md) — brownfield detection + sampling preservation +- [Content Capture](../tracking/content-capture.md) — enabling prompt/response capture for eval +- [Outcomes](../tracking/outcomes.md) — how event outcome is resolved +- [Kubernetes Deployment](../integration/kubernetes.md) — zero-code instrumentation at scale +- [Context Propagation](../concepts/context-propagation.md) — how run_id flows across services diff --git a/docs/index.md b/docs/index.md index ad62bb1..c4c78df 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,14 +32,15 @@ event cost?" and "What was the outcome?" - [LLM Tracking](tracking/llm-tracking.md) — Track AI model calls and token usage - [Data Tracking](tracking/data-tracking.md) — Track database, storage, and messaging operations -- [Outcomes](tracking/outcomes.md) — Record business outcomes for ROI calculation +- [Content Capture](tracking/content-capture.md) — Capture prompts and responses for eval (opt-in) +- [Outcomes](tracking/outcomes.md) — Record diagnostic context; how event outcome is actually resolved ### Integration - [Auto-Instrumentation](integration/auto-instrumentation.md) — Supported libraries and frameworks - [Kubernetes Deployment](integration/kubernetes.md) — Zero-code instrumentation at scale -- [Existing OTel Setup](integration/existing-otel.md) — Integrate with existing OpenTelemetry deployments -- [Collector Configuration](integration/collector.md) — Configure the OpenTelemetry Collector +- [Using botanu with existing OTel / Datadog](integration/existing-otel.md) — Brownfield detection, sampling preservation, ddtrace coexistence +- [Collector](integration/collector.md) — Botanu Cloud collector endpoints and auth ### Patterns @@ -55,17 +56,20 @@ event cost?" and "What was the outcome?" ## Quick Example ```python -from botanu import enable, botanu_workflow, emit_outcome +from botanu import enable, botanu_workflow -enable() +enable() # reads BOTANU_API_KEY from env; auto-configures endpoint @botanu_workflow("my-workflow", event_id="evt-001", customer_id="cust-42") async def do_work(): - result = await do_something() - emit_outcome("success") - return result + return await do_something() ``` +Outcome is resolved server-side from eval verdict / HITL / SoR — you do +not need to call `emit_outcome` to record success. See +[Outcomes](tracking/outcomes.md) for diagnostic annotations that are +still useful. + ## License [Apache License 2.0](https://github.com/botanu-ai/botanu-sdk-python/blob/main/LICENSE) diff --git a/docs/integration/collector.md b/docs/integration/collector.md index 6d1708d..46b88e0 100644 --- a/docs/integration/collector.md +++ b/docs/integration/collector.md @@ -27,7 +27,7 @@ The SDK defaults to HTTP (`ingest.botanu.ai:4318`) when `BOTANU_API_KEY` is set. No collector configuration is needed on your side. Just set the API key: ```bash -export BOTANU_API_KEY="btnu_live_..." +export BOTANU_API_KEY= ``` ```python @@ -50,13 +50,25 @@ Or via environment variable: export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 ``` +### ⚠ The API key is only sent to botanu-trusted endpoints + +If you override the endpoint to a non-botanu host (e.g., a self-hosted +collector, Datadog, or any third-party OTLP backend), the SDK **does not +attach your `BOTANU_API_KEY` as an Authorization header**. This prevents a +misconfigured `OTEL_EXPORTER_OTLP_ENDPOINT` from leaking tenant +credentials to another vendor. Trusted hosts are `*.botanu.ai` plus local +dev hosts (`localhost`, `127.0.0.1`, `::1`, `0.0.0.0`); everything else +runs unauthenticated unless you pass your own `otlp_headers=`. See the +[Configuration doc](../getting-started/configuration.md#️-the-api-key-is-only-sent-to-botanu-trusted-endpoints) +for the full list. + ## Data Flow ``` Your App (SDK) │ │ OTLP/HTTP (TLS) - │ Authorization: Bearer btnu_live_... + │ Authorization: Bearer ▼ ingest.botanu.ai (Botanu-hosted collector) │ diff --git a/docs/integration/existing-otel.md b/docs/integration/existing-otel.md index 72de805..c15689f 100644 --- a/docs/integration/existing-otel.md +++ b/docs/integration/existing-otel.md @@ -1,88 +1,177 @@ -# Existing OpenTelemetry Setup +# Using botanu with an existing OTel / APM setup -Integrate botanu with your existing OpenTelemetry configuration — Datadog, Jaeger, Grafana Tempo, Splunk, New Relic, or any OTel-compatible backend. +botanu is designed to sit alongside an OTel / Datadog / Jaeger / Honeycomb / +New Relic setup you already have. You do not have to migrate off your current +APM — you call `enable()`, and botanu detects what is already configured and +adds itself without stealing spans or changing your sampled volume. -## Automatic Detection (Recommended) +This page explains exactly what `enable()` does in each of the three +configurations it detects, so you can reason about the outcome before you +install. -As of SDK v0.1.0, `enable()` **automatically detects your existing TracerProvider** and adds botanu alongside it. No manual processor setup needed: +## TL;DR + +- **Already on the OTel SDK?** `enable()` keeps your span processors, preserves + your sampling ratio for them, and adds the botanu exporter at 100%. + Your existing APM bill does not change. +- **Already on ddtrace (Datadog's Python SDK)?** `enable()` creates a + separate, parallel TracerProvider for botanu. ddtrace is untouched. +- **No existing tracing?** `enable()` creates a fresh provider and wires + everything up for you. + +In all three cases the SDK is a single call: ```python from botanu import enable -enable() # Detects existing OTel, adds botanu alongside +enable() # reads config from env; no hard-coded values ``` -**What happens under the hood:** +--- + +## Detection — what `enable()` checks + +When you call `enable()`, the SDK calls `trace.get_tracer_provider()` and +branches on what it finds. The logic lives in +[`src/botanu/sdk/bootstrap.py`](../../src/botanu/sdk/bootstrap.py). + +| What `get_tracer_provider()` returns | What botanu does | +| --- | --- | +| `opentelemetry.sdk.trace.TracerProvider` (the real OTel SDK class) | Treats as **brownfield OTel**. Creates a new provider, migrates your span processors, wraps ratio-sampled processors in `SampledSpanProcessor`, adds botanu alongside, swaps the global provider. | +| `opentelemetry.trace.ProxyTracerProvider` (no real provider set) | Treats as **greenfield**. Creates a fresh provider with `ALWAYS_ON` sampling. | +| Anything else (e.g., `ddtrace.opentelemetry.TracerProvider`) | Treats as **unknown / parallel**. Creates a separate TracerProvider for botanu. Your existing tracer is untouched. | + +botanu never mutates your existing provider in place. It either creates a +new one and swaps the global, or leaves yours entirely alone. + +--- + +## Brownfield: existing OTel SDK + +### What botanu does + +1. Reads the sampling ratio off your provider using + `_extract_sampler_ratio()`. This recognises `AlwaysOn`, `AlwaysOff`, + `TraceIdRatioBased`, and `ParentBased(...)` wrappers around those. +2. Collects the list of processors already attached to your provider. +3. Creates a **new** `TracerProvider` with `ALWAYS_ON`, keeping your + `Resource`. +4. For each of your existing processors: + - If your ratio was `< 1.0`, wraps the processor in a + `SampledSpanProcessor(proc, original_ratio)` so it continues to see + only the fraction of spans it used to. + - Otherwise attaches it as-is. +5. Adds `RunContextEnricher` (run_id / workflow / event_id baggage → span + attributes), `ResourceEnricher`, and botanu's own `BatchSpanProcessor` + to the new provider — **unwrapped**, so botanu sees 100%. +6. `trace.set_tracer_provider(new_provider)` — this becomes the global. +7. Logs one of: + + ```text + Botanu SDK: existing TracerProvider detected with 10% sampling. + Preserved your sampling ratio for existing exporters. + botanu captures 100%. No impact on your existing observability bill. + ``` + + or, for 100% samplers: + + ```text + Botanu SDK: existing TracerProvider detected. + Added botanu exporter alongside your existing setup. + ``` + +### Why flip to AlwaysOn internally? + +botanu needs 100% of spans to produce accurate cost attribution — you can't +extrapolate token counts or per-span costs from a 10% sample without +distortion. So the new provider is `ALWAYS_ON`. The resulting diagram: + +```text +App (sampler = AlwaysOn → every span created) + │ + ├─ SampledSpanProcessor(0.10) → your Datadog BatchSpanProcessor → Datadog (sees 10%) + │ ↑ same volume as before + │ + └─ botanu BatchSpanProcessor → botanu collector (sees 100%) +``` -| Your setup | What `enable()` does | -|-----------|---------------------| -| OTel SDK with AlwaysOn sampling | Migrates your processors to a new provider, adds botanu exporter alongside | -| OTel SDK with ratio sampling (e.g., 10%) | Same, but wraps your processors in `SampledSpanProcessor` to preserve your ratio. Your Datadog/Jaeger bill is unchanged. | -| ddtrace (Datadog Python SDK) | Creates a parallel TracerProvider. ddtrace continues unchanged. | -| No existing tracing | Creates a fresh provider (standard greenfield path) | +`SampledSpanProcessor` is deterministic on `trace_id`, matching OTel's +`TraceIdRatioBasedSampler` algorithm — the same trace always gets the same +decision, so a trace that hits Datadog also hits 100% of botanu spans and +nothing is orphaned. -**Zero disruption guarantee:** Your existing dashboards, bills, and sampling are preserved exactly as they were. +### Unknown sampler — safety path -## How Sampling Is Preserved +If `_extract_sampler_ratio()` cannot identify your sampler (custom +subclass, third-party library), botanu **does not assume 100%**. Instead it: -If your existing provider uses ratio-based sampling (e.g., 10%), botanu needs to change the sampler to AlwaysOn (to capture 100% for cost attribution). But your existing exporter should still see only 10%. +1. Logs a warning with your sampler's class name. +2. Creates the new provider with your **original sampler** preserved. +3. Attaches your processors unwrapped — they see what they saw before. +4. Attaches botanu's processors also under your original sampler — meaning + botanu will see only the sampled subset too, not 100%. -botanu solves this with `SampledSpanProcessor`, which wraps your existing processors and applies your original ratio at the export level: +This is deliberate. Silently defaulting an unknown sampler to 1.0 would +inflate your existing exporter's volume 10× or 100× and potentially blow up +your observability bill. The cost of the unknown path is that botanu's cost +numbers are computed on the sampled subset; accept that or migrate your +sampler to a known one (`AlwaysOn` / `TraceIdRatioBased` / +`ParentBased(TraceIdRatioBased(...))`). -``` -App (AlwaysOn sampler — all spans created) - → SampledSpanProcessor(0.1) → Your Datadog exporter → Datadog (sees 10%) - → botanu exporter → botanu collector (sees 100%) -``` +--- -This is deterministic — the same trace_id always gets the same sampling decision. +## Parallel: ddtrace -## Manual Integration (Advanced) +`ddtrace` (Datadog's Python SDK) installs its own tracer that implements +the OTel API but extends a different base class than the OTel SDK. botanu +detects this via `isinstance(existing, TracerProvider)` returning `False` +and falls through to the parallel path. -If you prefer manual control or want to understand the internals: +In the parallel path: -```python -from opentelemetry import trace -from botanu.processors import RunContextEnricher, SampledSpanProcessor -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.trace.export import BatchSpanProcessor +- botanu creates its own `TracerProvider` and does **not** call + `trace.set_tracer_provider(...)`. +- ddtrace keeps handling spans for ddtrace decorators and for Datadog + auto-instrumentation. +- botanu's decorators (`@botanu_workflow`, `track_llm_call`, etc.) get + their spans from the botanu provider, which forwards to the botanu + collector. -# Get your existing TracerProvider -provider = trace.get_tracer_provider() +The two tracing systems coexist. Nothing is stolen, nothing is wrapped. -# 1. Add RunContextEnricher (propagates run_id, workflow, event_id to all spans) -provider.add_span_processor(RunContextEnricher()) +A span will appear in Datadog if it was created inside ddtrace +instrumentation, and in botanu if it was created inside botanu +instrumentation. To cross-reference a trace between the two dashboards, use +`botanu.run_id` — it is set via W3C Baggage on every botanu span, and you +can write a small Datadog tag mapper to surface it on ddtrace spans too if +you want. -# 2. Add botanu OTLP exporter (sends traces to botanu collector) -botanu_exporter = OTLPSpanExporter( - endpoint="https://ingest.botanu.ai:4318/v1/traces", - headers={"Authorization": "Bearer btnu_live_..."}, -) -provider.add_span_processor(BatchSpanProcessor(botanu_exporter)) -``` +### Longer-term option -## With Datadog (ddtrace) +If you eventually want a single tracing layer, the migration path is: -ddtrace uses its own tracing system (not OTel SDK). `enable()` detects this and creates a separate TracerProvider for botanu: +1. Today: dual tracing — ddtrace + botanu running in parallel. +2. Later: switch ddtrace off, move to the OTel SDK, configure the OTel + Datadog exporter. Now botanu's brownfield path kicks in and you're back + to one provider with two exporters. -```python -# ddtrace continues working unchanged -from ddtrace import tracer # noqa — ddtrace auto-patches +We do not require this and there is no deadline — the parallel setup is +supported indefinitely. -# botanu creates its own provider alongside ddtrace -from botanu import enable -enable() -``` +--- + +## Greenfield: no existing tracing -Both tracing systems run in parallel. No conflicts. +If `trace.get_tracer_provider()` returns a `ProxyTracerProvider`, nothing +is configured yet. botanu creates a fresh `TracerProvider` with +`ALWAYS_ON`, adds `RunContextEnricher` / `ResourceEnricher` / botanu's +exporter, and sets it as the global. This is the standard path for +first-time users. -**Migration path** (optional, for simplification): -1. **Phase A** (now): Dual tracing — ddtrace + botanu -2. **Phase C** (later): Configure ddtrace OTLP export, remove botanu auto-instrumentation -3. **Phase D** (long-term): Migrate to OTel SDK + Datadog exporter — single tracing layer +--- -## Using botanu Decorators +## Using botanu decorators -With either automatic or manual integration, use botanu decorators for cost attribution: +Regardless of which path `enable()` takes, the decorator API is the same: ```python from botanu import botanu_workflow, emit_outcome @@ -98,21 +187,123 @@ async def handle_ticket(req): return result ``` -All child spans (auto-instrumented OpenAI, database, HTTP calls) inherit the run context automatically via W3C Baggage. +Auto-instrumented spans (OpenAI SDK, HTTP clients, DB drivers) inside the +decorated call inherit the run context through W3C Baggage, so cost +attribution works even for spans your code never directly creates. See +[Auto-Instrumentation](auto-instrumentation.md). -## Troubleshooting +--- + +## Manual integration (advanced, OTel SDK only) + +If you want to wire botanu into an existing OTel SDK provider without +calling `enable()` at all, you can attach the processors yourself: + +```python +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor -### run_id not appearing on spans -1. Verify `enable()` was called (or `RunContextEnricher` was added manually) -2. Check `@botanu_workflow` is on your entry point functions -3. Verify W3C Baggage propagator is active: `propagate.get_global_textmap()` +from botanu.processors import RunContextEnricher, SampledSpanProcessor + +provider = trace.get_tracer_provider() +assert isinstance(provider, TracerProvider), "manual integration requires the OTel SDK provider" -### Existing traces missing after adding botanu -This should not happen — `enable()` preserves your existing processors. If it does: -1. Check `enable()` was called ONCE (not multiple times) -2. Check your existing provider was created BEFORE `enable()` runs +# 1. Enrich all spans with run_id / workflow / event_id from baggage. +provider.add_span_processor(RunContextEnricher()) + +# 2. Send a copy of every span to the botanu collector. +botanu_exporter = OTLPSpanExporter( + endpoint="https://ingest.botanu.ai:4318/v1/traces", + headers={"Authorization": "Bearer "}, +) +provider.add_span_processor(BatchSpanProcessor(botanu_exporter)) +``` + +**This does not work for ddtrace.** ddtrace's `TracerProvider` does not +expose `add_span_processor()`. If you are on ddtrace, use `enable()` and +let the SDK take the parallel path. + +Manual integration also skips the `SampledSpanProcessor` preservation +logic — if your provider uses ratio sampling and you need botanu to still +see 100%, you'd have to duplicate the bootstrap logic. Just call +`enable()`; that's what it's for. + +--- + +## Verifying it worked + +After installing and calling `enable()`, you should see exactly one of +these log lines: + +| Log line | Means | +| --- | --- | +| `existing TracerProvider detected with N% sampling. Preserved your sampling ratio` | Brownfield OTel SDK with known sampler. Your APM gets the same volume, botanu gets 100%. | +| `existing TracerProvider detected. Added botanu exporter alongside your existing setup` | Brownfield OTel SDK with AlwaysOn. Both exporters get 100%. | +| `could not identify the sampling ratio of X. Preserving the original sampler` | Brownfield OTel SDK with unknown sampler. Both exporters see the sampled subset. | +| (no brownfield line) | Greenfield or ddtrace parallel — check the `enable()` return value. | + +Sanity checks in order: + +1. Open your existing APM — confirm span volume is unchanged. +2. Open botanu — confirm spans are arriving. A run created by + `@botanu_workflow` carries `botanu.run_id`, `botanu.workflow`, + `botanu.event_id`. +3. If both arrive, you're done. + +--- + +## Troubleshooting -### Sampling concerns -If you use ratio sampling and see unexpected volume changes in your APM: -1. Check botanu logs for "Preserved your sampling ratio" message -2. Verify `SampledSpanProcessor` is wrapping your exporter (not replacing it) +### My spans disappeared from Datadog after I installed botanu + +Should not happen. Check, in order: + +1. `enable()` was called exactly once at startup — if called twice, both + calls no-op after the first, so duplicate calls are safe but indicate a + confused startup order. +2. Your existing OTel provider was created **before** `enable()` runs. If + `enable()` runs first, brownfield detection doesn't see you and you'll + end up on the greenfield path, which blows away an OTel provider set + afterwards. +3. Your existing exporter was actually attached to the provider botanu + detected. If you have multiple providers (per-module, per-service), the + one `trace.get_tracer_provider()` returns is the one botanu wraps — + processors attached to others are unaffected (and therefore invisible + to botanu too). + +### botanu shows 100% of spans but Datadog only shows 10% + +That's the expected brownfield behavior with a ratio sampler. botanu +captures 100% for cost attribution; your existing exporter stays on its +original ratio so your bill and dashboards are unchanged. Look for the log +line starting `Preserved your sampling ratio`. + +### `run_id` is missing on auto-instrumented spans + +1. Verify `enable()` was called (or `RunContextEnricher` was attached + manually). +2. Verify an entry-point function is wrapped in `@botanu_workflow` — the + baggage is set on entry and inherited by child spans from there. +3. Verify the W3C Baggage propagator is active: + `from opentelemetry import propagate; propagate.get_global_textmap()` + should include `baggage` in its composite. + +### `could not identify the sampling ratio` warning + +Your sampler is a type botanu doesn't recognise. Two options: + +1. Accept it — botanu sees only the sampled subset. Cost attribution is + still correct, just computed on fewer spans. +2. Switch to `TraceIdRatioBased(...)`, `ParentBased(TraceIdRatioBased(...))`, + `AlwaysOn`, or `AlwaysOff` on your `TracerProvider`. botanu will then + take the normal brownfield path and preserve your ratio for existing + processors while capturing 100% for itself. + +## See also + +- [Collector](collector.md) — where botanu's spans go next +- [Auto-Instrumentation](auto-instrumentation.md) — the span sources +- [Configuration](../getting-started/configuration.md) — env vars, endpoint trust +- Source of truth: [`src/botanu/sdk/bootstrap.py`](../../src/botanu/sdk/bootstrap.py) and [`src/botanu/processors/sampled.py`](../../src/botanu/processors/sampled.py) diff --git a/docs/patterns/best-practices.md b/docs/patterns/best-practices.md index ce22b2d..f60367b 100644 --- a/docs/patterns/best-practices.md +++ b/docs/patterns/best-practices.md @@ -46,25 +46,38 @@ Workflow names appear in dashboards and queries. Choose names carefully: ## Outcome Recording -### Always Record Outcomes +### Outcome is derived, not reported -Every run should have an explicit outcome: +Event outcome is computed server-side from eval verdict rollup / HITL / SoR +connector — not from `emit_outcome(status=...)`. The `status` argument is +now a diagnostic helper only (it raises a `DeprecationWarning` on every +call). You do **not** need to call `emit_outcome` to record success or +failure; `@botanu_workflow` already creates the run and the platform will +resolve its outcome. + +`emit_outcome` is still useful when you want to annotate the run with +*diagnostic* context the dashboard can show alongside outcome: ```python @botanu_workflow("process_data", event_id=data_id, customer_id=customer_id) async def process_data(data_id: str, customer_id: str): try: result = await process(data_id) + # Stamp value_type / value_amount for cost-per-value math. emit_outcome("success", value_type="records_processed", value_amount=result.count) return result - except ValidationError: - emit_outcome("failed", reason="validation_error") + except ValidationError as exc: + # Stamp the reason/error_type so the dashboard can group failures. + emit_outcome("failed", reason="validation_error", error_type=type(exc).__name__) raise except TimeoutError: - emit_outcome("failed", reason="timeout") + emit_outcome("failed", reason="timeout", error_type="TimeoutError") raise ``` +See [Outcomes](../tracking/outcomes.md) for the full list of diagnostic +fields that still stamp. + ### Quantify Value When Possible Include value amounts for better ROI analysis: diff --git a/docs/tracking/content-capture.md b/docs/tracking/content-capture.md new file mode 100644 index 0000000..3762e69 --- /dev/null +++ b/docs/tracking/content-capture.md @@ -0,0 +1,146 @@ +# Content Capture + +Capture prompt and response text so the evaluator can judge your runs. +Disabled by default — you opt in with a sampling rate. + +## Why capture content? + +The botanu evaluator (LLM-as-judge, retrieval-quality checks, policy checks) +can only score what it can see. Without captured input/output text, the +evaluator falls back to a workflow-name placeholder and every verdict ends up +scoring the same empty string. Capture is the on-ramp to real eval verdicts +and, via the verdict rollup, to accurate event-level outcome determination. + +## The knob + +One config field turns the whole thing on: + +```python +from botanu import enable +from botanu.sdk.config import BotanuConfig + +enable(config=BotanuConfig(content_capture_rate=0.10)) +``` + +Or via environment: + +```bash +export BOTANU_CONTENT_CAPTURE_RATE=0.10 +``` + +Recommended settings: + +| Environment | Rate | Why | +| --- | --- | --- | +| Production | `0.10`–`0.20` | Enough samples for statistical eval without flooding storage | +| Staging / shadow | `1.0` | Capture everything while iterating on prompts | +| Sandbox / local | `1.0` | Capture everything | +| Unknown | `0.0` (default) | Capture nothing — privacy-safe default | + +The gate is a `random.random() < rate` check per call. It is independent for +each capture point — the SDK does not coordinate across processes. + +## Three capture points + +### 1. Workflow-level (automatic, once per run) + +`@botanu_workflow` will capture the decorated function's bound arguments as +input and its return value as output, **once per run**, when +`content_capture_rate` fires. + +```python +from botanu import botanu_workflow + +@botanu_workflow( + "summarize", + event_id=lambda req: req.id, + customer_id=lambda req: req.tenant, +) +def summarize(req): + return llm.summarize(req.text) +``` + +When the rate gate passes: + +- The arguments are bound against the signature (`inspect.signature(func).bind_partial`) + and written as `botanu.eval.input_content` on the root `botanu.run` span. +- The return value is written as `botanu.eval.output_content`. + +Both fields are JSON-serialized (with a `repr` fallback) and truncated to +4096 characters. The decision is made once per call so you never land a +half-captured pair. + +### 2. LLM-span-level (explicit, per model call) + +[`LLMTracker`](llm-tracking.md#set_input_content-set_output_content) exposes +`set_input_content()` and `set_output_content()` for per-call capture. Use +these when you want the *actual* prompt / response text on a specific LLM +span rather than the bound workflow arguments. + +```python +from botanu.tracking.llm import track_llm_call + +with track_llm_call(provider="openai", model="gpt-4") as tracker: + tracker.set_input_content(prompt) + response = openai.chat.completions.create(model="gpt-4", messages=[...]) + tracker.set_output_content(response.choices[0].message.content) + tracker.set_tokens( + input_tokens=response.usage.prompt_tokens, + output_tokens=response.usage.completion_tokens, + ) +``` + +These calls no-op when `content_capture_rate` is 0.0. Each call evaluates +the rate independently. + +### 3. Data/tool-span-level + +`track_tool_call()` and the data-tracking helpers follow the same pattern — +expose optional content setters that respect the same rate. See +[Data Tracking](data-tracking.md) for the specific signatures. + +## What gets written + +| Attribute | Written by | Source | +| --- | --- | --- | +| `botanu.eval.input_content` | `@botanu_workflow` | Bound function arguments (JSON) | +| `botanu.eval.output_content` | `@botanu_workflow` | Return value (JSON) | +| `botanu.eval.input_content` | `LLMTracker.set_input_content()` | Explicit prompt text | +| `botanu.eval.output_content` | `LLMTracker.set_output_content()` | Explicit response text | + +All values are truncated to 4096 characters before being stamped. + +## PII handling + +The SDK **does not scrub PII**. Scrubbing happens downstream: + +1. **Collector** — runs a regex redaction pass on `botanu.eval.*` attributes + (credit-card, email, phone, API-key patterns) before forwarding. +2. **Evaluator** — runs a Microsoft Presidio NER pass before storing captured + text against the eval record. + +If you have strict PII requirements, keep `content_capture_rate=0.0` and +drive eval off explicit tool/score annotations instead. The capture pipeline +is opt-in precisely so you can stay private by default. + +## Verifying capture is on + +After setting a non-zero rate, run a workflow and check the span attributes +with your normal OTel tooling. A captured span will carry +`botanu.eval.input_content` and `botanu.eval.output_content` as string +attributes. If they are absent, check in order: + +1. `BotanuConfig.content_capture_rate` is actually > 0.0 in the running + process (`BotanuConfig.from_yaml(...)` and env precedence can surprise + you — print `get_config().content_capture_rate` to be sure). +2. You are inside a span (`@botanu_workflow` or `track_llm_call` scope). +3. The random gate didn't miss — at `rate=0.1`, ~90% of calls will look + empty. Set the rate to `1.0` temporarily to confirm plumbing. + +## See also + +- [LLM Tracking → set_input_content / set_output_content](llm-tracking.md#set_input_content-set_output_content) +- [Configuration → content_capture_rate](../getting-started/configuration.md) +- `src/botanu/sampling/content_sampler.py` — the rate gate +- `src/botanu/sdk/decorators.py` — workflow-level auto-capture +- `src/botanu/tracking/llm.py` — LLM-span capture diff --git a/docs/tracking/llm-tracking.md b/docs/tracking/llm-tracking.md index e2053ba..f414fcf 100644 --- a/docs/tracking/llm-tracking.md +++ b/docs/tracking/llm-tracking.md @@ -72,6 +72,35 @@ When the response uses a different model than requested: tracker.set_response_model("gpt-4-0613") ``` +### set_input_content() / set_output_content() + +Capture the prompt text and response text for downstream evaluation. + +```python +tracker.set_input_content(prompt_text) +tracker.set_output_content(response_text) +``` + +Both methods are **gated by `BotanuConfig.content_capture_rate`**: + +- Default rate is `0.0` — both calls no-op. Nothing is written to the span. +- Set the rate to `0.10`–`0.20` in production (or `1.0` in a sandbox) to start + capturing. The gate is a simple `random.random() < rate` check, so the + decision is per-call. +- Text is truncated at `max_chars` (default 4096) before being stamped. + +When capture fires, the SDK writes: + +| Attribute | Source | +| --- | --- | +| `botanu.eval.input_content` | `set_input_content(text)` | +| `botanu.eval.output_content` | `set_output_content(text)` | + +**PII is not scrubbed in the SDK.** The collector runs a regex redaction pass +and the evaluator runs Presidio NER before any captured text is stored. +See [Content Capture](content-capture.md) for the full pipeline and for the +workflow-level auto-capture path that `@botanu_workflow` provides. + ### set_request_params() Record request parameters for analysis: diff --git a/docs/tracking/outcomes.md b/docs/tracking/outcomes.md index 40a4837..1a24a0b 100644 --- a/docs/tracking/outcomes.md +++ b/docs/tracking/outcomes.md @@ -1,274 +1,154 @@ # Outcomes -> **⚠️ DEPRECATED (2026-04-16)**: The `status` argument on `emit_outcome()` no longer -> stamps `botanu.outcome.status` on the span. Customer-reported outcome was removed -> because it was trivially fakeable — a misconfigured or adversarial SDK could -> claim every event succeeded and skew cost-per-outcome numbers. +> **⚠️ DEPRECATED (2026-04-16):** The `status` argument on `emit_outcome()` no +> longer stamps `botanu.outcome.status` on the span. Customer-reported outcome +> was removed because it was trivially fakeable — a misconfigured or +> adversarial SDK could claim every event succeeded and skew cost-per-outcome. > -> **What to do instead**: event outcome is now derived by botanu's evaluator -> (LLM-as-judge verdict), human review queue, or a system-of-record connector -> (coming later). You don't need to call `emit_outcome()` for outcome -> determination. Keep calls that pass diagnostic fields (`reason`, `error_type`, -> `value_type`, `value_amount`, `confidence`, `metadata`) — those still stamp. -> Expect a `DeprecationWarning` on every `emit_outcome(status=...)` call until -> you migrate. +> **What determines event outcome now:** botanu derives the outcome server-side +> from, in priority order: +> +> 1. A System-of-Record (SoR) connector (Zendesk, Stripe, your own webhook). +> 2. A human reviewer verdict from the HITL queue. +> 3. The evaluator's LLM-as-judge verdict rollup for the event's runs. +> 4. `pending` if nothing above fires yet. +> +> **What still works:** the other `emit_outcome(...)` fields (`reason`, +> `error_type`, `value_type`, `value_amount`, `confidence`, `metadata`) still +> stamp as *diagnostic* span attributes. They are useful for debugging and +> drill-down in the dashboard; they are not the authoritative outcome. +> +> Every call to `emit_outcome(status=...)` emits a `DeprecationWarning`. ## Overview -Outcomes connect infrastructure costs to business value. By recording diagnostic fields per event, you enrich the data the evaluator works with. +An **event** is one business transaction (a support ticket, an order, a report +generation). An event has one outcome that botanu determines from the signals +above. What you can do from the SDK is enrich the event with diagnostic +context — a reason string, an error classification, a value figure for +cost-per-value math, or arbitrary metadata. + +**Hierarchy refresher:** -**Terminology:** -- An **event** is one business transaction (e.g., a customer request, a pipeline trigger). -- A **run** is one execution attempt within an event. -- An event's **outcome** is derived by botanu (eval verdict rollup / HITL / SoR); you no longer set it yourself. +- **Event** — one business unit of work (has an `event_id`). Outcome lives here. +- **Run** — one execution attempt for an event. Retries replace the previous + attempt; see [Run Context](../concepts/run-context.md). +- **Span** — one LLM/DB/tool call within a run. -## Basic Usage (updated) +## The diagnostic helpers + +`emit_outcome` is now a thin helper that writes a fixed set of diagnostic +attributes onto the current span. Everything except `status` still stamps. ```python from botanu import botanu_workflow, emit_outcome -@botanu_workflow("process-items", event_id=request.id, customer_id=customer.id) -async def handle_request(): - result = await do_work() +@botanu_workflow("fulfill-order", event_id=order.id, customer_id=customer.id) +async def process_order(order): + result = await do_work(order) - # Optional: record diagnostic fields. The `status` argument is deprecated - # (no longer stamps outcome) but value_type / value_amount still stamp. + # Diagnostic fields — useful for dashboard drill-down and cost-per-value + # math, but not the authoritative outcome. emit_outcome( - "success", # accepted for backward compat; emits DeprecationWarning - value_type="items_processed", - value_amount=result.count, + "success", # accepted for back-compat, DeprecationWarning + value_type="orders_fulfilled", + value_amount=1, + metadata={"sku_count": len(order.items)}, ) + return result ``` -For the MVP eval flow, the simpler pattern is just `@botanu_workflow(...)` — no `emit_outcome()` call needed at all. +If you do not need any of the diagnostic fields, you can drop `emit_outcome` +entirely. `@botanu_workflow` already creates the run and span — outcome will +be filled in server-side from the signals above. -## emit_outcome() Parameters +## emit_outcome() reference ```python emit_outcome( - status: str, # Required: "success", "partial", "failed", "timeout", "canceled", "abandoned" + status: str, # Required for validation; does NOT stamp the outcome. *, - value_type: str = None, # What was achieved - value_amount: float = None, # How much - confidence: float = None, # Confidence score (0.0-1.0) - reason: str = None, # Why (especially for failures) - error_type: str = None, # Error classification - metadata: dict = None, # Additional key-value pairs + value_type: str | None = None, # Free-form business-value label. + value_amount: float | None = None, + confidence: float | None = None, # 0.0–1.0 + reason: str | None = None, # Free-form; especially for failures. + error_type: str | None = None, # Exception/classification name. + metadata: dict | None = None, # Arbitrary diagnostic kv. ) ``` -### status - -The outcome status: - -| Status | Description | Example | -|--------|-------------|---------| -| `success` | Fully achieved goal | All items processed | -| `partial` | Partially achieved | 3 of 5 items processed | -| `failed` | Did not achieve goal | Error during processing | -| `timeout` | Timed out before completing | Deadline exceeded | -| `canceled` | Canceled by user or system | User aborted the request | -| `abandoned` | Abandoned without completion | No response from upstream | +### status (required but diagnostic-only) -### value_type +`status` is still validated against the set below. It is accepted for +backward compatibility and its value is not written to the span. -A descriptive label for what was achieved: +| Value | Intended meaning | +| --- | --- | +| `success` | Event produced the intended result | +| `partial` | Event produced some of the intended result | +| `failed` | Event did not produce a result | +| `timeout` | Event did not finish in its deadline | +| `canceled` | Event was canceled by user or system | +| `abandoned` | Event was abandoned without completion | -```python -emit_outcome("success", value_type="items_processed", value_amount=1) -emit_outcome("success", value_type="documents_generated", value_amount=5) -emit_outcome("success", value_type="tasks_completed", value_amount=1) -emit_outcome("success", value_type="revenue_generated", value_amount=499.99) -``` +A value outside this set raises `ValueError`. `"failure"` is not valid — use +`"failed"`. -### value_amount +### Other fields -The quantified value: +Everything below stamps as a `botanu.outcome.*` diagnostic attribute: ```python -# Count -emit_outcome("success", value_type="records_written", value_amount=100) - -# Revenue -emit_outcome("success", value_type="order_value", value_amount=1299.99) - -# Score -emit_outcome("success", value_type="quality_score", value_amount=4.5) +emit_outcome("success", value_type="tickets_resolved", value_amount=1) +emit_outcome("success", value_type="revenue_generated", value_amount=1299.99) +emit_outcome("success", value_type="classifications_completed", + value_amount=1, confidence=0.92) +emit_outcome("failed", reason="upstream_unavailable", error_type="ServiceUnavailable") +emit_outcome("timeout", reason="model_took_too_long", error_type="DeadlineExceeded") +emit_outcome("partial", reason="processed_3_of_5", value_amount=3) +emit_outcome("success", value_type="items_processed", value_amount=10, + metadata={"batch_id": "abc-123", "retry_count": 2}) ``` -### confidence +## Span attributes that are still emitted -For probabilistic outcomes: - -```python -emit_outcome( - "success", - value_type="classifications_completed", - value_amount=1, - confidence=0.92, -) -``` - -### reason - -Explain the outcome (especially for failures): - -```python -emit_outcome("failed", reason="rate_limit_exceeded") -emit_outcome("failed", reason="invalid_input") -emit_outcome("partial", reason="timeout_partial_results", value_amount=3) -``` - -### error_type - -Classify the error for aggregation: - -```python -emit_outcome("failed", reason="upstream service unavailable", error_type="ServiceUnavailable") -emit_outcome("timeout", reason="model took too long", error_type="DeadlineExceeded") -``` - -### metadata - -Attach arbitrary key-value pairs: - -```python -emit_outcome( - "success", - value_type="items_processed", - value_amount=10, - metadata={"batch_id": "abc-123", "retry_count": 2}, -) -``` - -## Outcome Patterns - -### Success with Value - -```python -@botanu_workflow("fulfill-order", event_id=order.id, customer_id=customer.id) -async def process_order(): - result = await do_work() - - emit_outcome( - "success", - value_type="orders_fulfilled", - value_amount=1, - ) -``` - -### Success with Revenue - -```python -@botanu_workflow("handle-inquiry", event_id=inquiry.id, customer_id=customer.id) -async def handle_inquiry(): - result = await process() - - if result.completed: - emit_outcome( - "success", - value_type="revenue_generated", - value_amount=result.total, - ) - else: - emit_outcome( - "partial", - value_type="leads_qualified", - value_amount=1, - ) -``` - -### Partial Success - -```python -@botanu_workflow("batch-process", event_id=batch.id, customer_id=customer.id) -async def process_batch(items: list): - processed = 0 - for item in items: - try: - await do_something(item) - processed += 1 - except Exception: - continue - - if processed == len(items): - emit_outcome("success", value_type="items_processed", value_amount=processed) - elif processed > 0: - emit_outcome( - "partial", - value_type="items_processed", - value_amount=processed, - reason=f"processed_{processed}_of_{len(items)}", - ) - else: - emit_outcome("failed", reason="no_items_processed") -``` - -### Failure with Reason - -```python -@botanu_workflow("analyze", event_id=job.id, customer_id=customer.id) -async def analyze(doc_id: str): - try: - data = await do_work(doc_id) - if not data: - emit_outcome("failed", reason="not_found", error_type="NotFound") - return None - - result = await process(data) - emit_outcome("success", value_type="items_analyzed", value_amount=1) - return result - - except RateLimitError: - emit_outcome("failed", reason="rate_limit_exceeded", error_type="RateLimitError") - raise - except TimeoutError: - emit_outcome("timeout", reason="analysis_timeout", error_type="TimeoutError") - raise -``` - -### Classification with Confidence - -```python -@botanu_workflow("classify", event_id=request.id, customer_id=customer.id) -async def classify(message: str): - result = await do_work(message) +| Attribute | Description | +| --- | --- | +| `botanu.outcome.value_type` | What was achieved (free-form label) | +| `botanu.outcome.value_amount` | Quantified value | +| `botanu.outcome.confidence` | Confidence score (0.0–1.0) | +| `botanu.outcome.reason` | Reason string (especially for failures) | +| `botanu.outcome.error_type` | Error classification | +| `botanu.outcome.metadata.*` | Flattened metadata dict | - emit_outcome( - "success", - value_type="classifications_completed", - value_amount=1, - confidence=result.confidence, - ) +> `botanu.outcome.status` is **not** emitted. Dashboards that read from +> `runs.outcome_status` are reading a legacy physical column kept only for +> backward compatibility; the authoritative field is `events.final_outcome`, +> which is written by the platform, not the SDK. - return result.label -``` +## Automatic outcome (convenience) -## Automatic Outcomes +`@botanu_workflow(..., auto_outcome_on_success=True)` (default) automatically +calls `emit_outcome("success")` at the end of a successful call, and +`emit_outcome("failed", reason=type(exc).__name__)` on exception. Since +`status` no longer stamps the outcome, this is pure convenience — it still +writes `reason` and `error_type` for failures, which is useful diagnostic +context. -The `@botanu_workflow` decorator automatically emits outcomes: +Disable if you prefer explicit calls: ```python -@botanu_workflow("my-workflow", event_id=event_id, customer_id=customer_id, auto_outcome_on_success=True) # Default +@botanu_workflow("my-workflow", event_id=event_id, customer_id=customer_id, + auto_outcome_on_success=False) async def my_function(): - # If no exception and no explicit emit_outcome, emits "success" + result = await do_work() + emit_outcome("success", value_type="items", value_amount=1) return result ``` -If an exception is raised, it automatically emits `"failed"` with the exception class as the reason. - -To disable: - -```python -@botanu_workflow("my-workflow", event_id=event_id, customer_id=customer_id, auto_outcome_on_success=False) -async def my_function(): - # Must call emit_outcome explicitly - emit_outcome("success") -``` - -## Context Manager Alternative +## Context manager form -Use `run_botanu` when you need workflow tracking without a decorator: +When you can't use the decorator: ```python from botanu import run_botanu, emit_outcome @@ -276,130 +156,25 @@ from botanu import run_botanu, emit_outcome async def my_function(event_id: str, customer_id: str): async with run_botanu("my-workflow", event_id=event_id, customer_id=customer_id): result = await do_work() - emit_outcome("success", value_type="items_processed", value_amount=result.count) - return result -``` - -## Span Attributes - -Outcomes are recorded as span attributes: - -| Attribute | Description | -|-----------|-------------| -| `botanu.outcome.status` | Status (success/partial/failed/timeout/canceled/abandoned) | -| `botanu.outcome.value_type` | What was achieved | -| `botanu.outcome.value_amount` | Quantified value | -| `botanu.outcome.confidence` | Confidence score | -| `botanu.outcome.reason` | Reason for outcome | -| `botanu.outcome.error_type` | Error classification | - -## Span Events - -An event is also emitted for timeline visibility: - -```python -# Event: botanu.outcome_emitted -# Attributes: -# status: "success" -# value_type: "items_processed" -# value_amount: 1 -``` - -## Cost-Per-Outcome Analysis - -With outcomes recorded, you can calculate: - -```sql --- Cost per successful outcome -SELECT - AVG(total_cost) as avg_cost_per_success -FROM runs -WHERE workflow = 'fulfill-order' - AND outcome_status = 'success' - AND outcome_value_type = 'orders_fulfilled'; - --- ROI by workflow -SELECT - workflow, - SUM(outcome_value_amount * value_per_unit) as total_value, - SUM(total_cost) as total_cost, - (SUM(outcome_value_amount * value_per_unit) - SUM(total_cost)) / SUM(total_cost) as roi -FROM runs -GROUP BY workflow; -``` - -## Best Practices - -### 1. Always Record Outcomes - -Every workflow should emit an outcome: - -```python -@botanu_workflow("my-workflow", event_id=event_id, customer_id=customer_id) -async def my_function(): - try: - result = await do_work() - emit_outcome("success", value_type="items_processed", value_amount=result.count) + emit_outcome("success", value_type="items_processed", + value_amount=result.count) return result - except Exception as e: - emit_outcome("failed", reason=type(e).__name__, error_type=type(e).__name__) - raise -``` - -### 2. Use Consistent Value Types - -Define standard value types for your organization: - -```python -# Good - consistent naming -emit_outcome("success", value_type="items_processed", value_amount=1) -emit_outcome("success", value_type="documents_generated", value_amount=1) - -# Bad - inconsistent -emit_outcome("success", value_type="item_done", value_amount=1) -emit_outcome("success", value_type="doc processed", value_amount=1) ``` -### 3. Quantify When Possible - -Include amounts for better analysis: - -```python -# Good - quantified -emit_outcome("success", value_type="records_written", value_amount=50) - -# Less useful - no amount -emit_outcome("success") -``` +## Cost-per-outcome math -### 4. Include Reasons for Failures +Cost-per-outcome is computed by the platform from: -Always explain why something failed: +- the `runs.cost_total_usd` column populated by the cost engine, and +- the `events.final_outcome` column populated by the outcome resolver. -```python -emit_outcome("failed", reason="api_rate_limit", error_type="RateLimitError") -emit_outcome("failed", reason="invalid_input_format", error_type="ValidationError") -emit_outcome("timeout", reason="model_unavailable", error_type="TimeoutError") -``` - -### 5. One Outcome Per Run - -Emit only one outcome per workflow execution: - -```python -@botanu_workflow("process-items", event_id=event_id, customer_id=customer_id) -async def process_items(items): - successful = 0 - for item in items: - if await process(item): - successful += 1 - - # One outcome at the end - emit_outcome("success", value_type="items_processed", value_amount=successful) -``` +You don't query these yourself — open the dashboard. What you *can* do from +the SDK is annotate with `value_type` / `value_amount` so a business-value +column appears alongside cost-per-outcome in the dashboard. -## See Also +## See also -- [Run Context](../concepts/run-context.md) - Understanding runs -- [LLM Tracking](llm-tracking.md) - Tracking LLM costs -- [Best Practices](../patterns/best-practices.md) - More patterns +- [Run Context](../concepts/run-context.md) — the event/run/span hierarchy +- [LLM Tracking](llm-tracking.md) — per-call attribution +- [Content Capture](content-capture.md) — capturing prompts/responses for eval +- [Best Practices](../patterns/best-practices.md) From 9ebc4c69fd7d60c001a3571fae13a8c80157705b Mon Sep 17 00:00:00 2001 From: Deborah Jacob Date: Thu, 23 Apr 2026 18:02:24 -0700 Subject: [PATCH 2/3] feat: unified event/step API + docs rewrite Replace @botanu_workflow / run_botanu / @botanu_outcome with botanu.event() (context manager + async CM + decorator) and botanu.step() for multi-phase events. event() lazy-calls enable() so customers don't need a separate init step. - Remove lean mode (full baggage is only mode; drop BAGGAGE_KEYS_LEAN, lean_mode param, propagation_mode field, BOTANU_PROPAGATION_MODE env) - Remove emit_outcome(status=...) positional arg (diagnostic fields only) - Drop ~440 LOC of legacy decorators + dead outcome.status checks - Add lazy _ensure_enabled() on first event() call - Rewrite all 20 SDK docs (Langfuse-style, each example self-contained) - Fix to_baggage_dict docs: 5 always + 7 conditional keys - Drop stale api_key field claim from BotanuConfig docs 417 tests pass, ruff clean. --- docs/api/configuration.md | 61 +-- docs/api/decorators.md | 118 ----- docs/api/event.md | 88 ++++ docs/api/tracking.md | 29 +- docs/concepts/architecture.md | 216 +-------- docs/concepts/context-propagation.md | 256 ++--------- docs/concepts/run-context.md | 252 +++-------- docs/getting-started/configuration.md | 154 ++----- docs/getting-started/quickstart.md | 120 ++--- docs/index.md | 86 ++-- docs/integration/collector.md | 18 +- docs/integration/existing-otel.md | 26 +- docs/integration/kubernetes.md | 35 +- docs/patterns/anti-patterns.md | 488 +++----------------- docs/patterns/best-practices.md | 423 +++--------------- docs/tracking/content-capture.md | 91 +++- docs/tracking/data-tracking.md | 116 ++--- docs/tracking/llm-tracking.md | 60 ++- docs/tracking/outcomes.md | 204 +++------ pyproject.toml | 8 + src/botanu/__init__.py | 35 +- src/botanu/models/run_context.py | 28 +- src/botanu/processors/enricher.py | 37 +- src/botanu/register.py | 2 +- src/botanu/sampling/content_sampler.py | 2 +- src/botanu/sdk/__init__.py | 8 +- src/botanu/sdk/bootstrap.py | 11 +- src/botanu/sdk/config.py | 86 +++- src/botanu/sdk/decorators.py | 596 ++++++++++++------------- src/botanu/sdk/middleware.py | 1 - src/botanu/sdk/pii.py | 179 ++++++++ src/botanu/sdk/pii_presidio.py | 98 ++++ src/botanu/sdk/span_helpers.py | 65 +-- src/botanu/tracking/data.py | 11 +- src/botanu/tracking/llm.py | 25 +- tests/conftest.py | 2 +- tests/unit/test_bootstrap.py | 35 +- tests/unit/test_config.py | 85 +++- tests/unit/test_data_tracking.py | 50 +++ tests/unit/test_decorators.py | 460 ------------------- tests/unit/test_enricher.py | 66 +-- tests/unit/test_event.py | 393 ++++++++++++++++ tests/unit/test_llm_tracking.py | 61 +++ tests/unit/test_pii.py | 223 +++++++++ tests/unit/test_run_context.py | 48 +- tests/unit/test_span_helpers.py | 69 +-- 46 files changed, 2363 insertions(+), 3162 deletions(-) delete mode 100644 docs/api/decorators.md create mode 100644 docs/api/event.md create mode 100644 src/botanu/sdk/pii.py create mode 100644 src/botanu/sdk/pii_presidio.py delete mode 100644 tests/unit/test_decorators.py create mode 100644 tests/unit/test_event.py create mode 100644 tests/unit/test_pii.py diff --git a/docs/api/configuration.md b/docs/api/configuration.md index c9acb48..27c83a6 100644 --- a/docs/api/configuration.md +++ b/docs/api/configuration.md @@ -11,23 +11,25 @@ from botanu.sdk.config import BotanuConfig ### Fields | Field | Type | Default | Description | -|-------|------|---------|-------------| +| --- | --- | --- | --- | | `service_name` | `str` | From env / `"unknown_service"` | Service name | | `service_version` | `str` | From env | Service version | | `service_namespace` | `str` | From env | Service namespace | | `deployment_environment` | `str` | From env / `"production"` | Deployment environment | | `auto_detect_resources` | `bool` | `True` | Auto-detect cloud resources | -| `api_key` | `str` | From env (`BOTANU_API_KEY`) | Auto-configures the endpoint to `https://ingest.botanu.ai` and attaches a bearer token on botanu-trusted hosts only | -| `otlp_endpoint` | `str` | From env / auto-configured from `api_key` / `"http://localhost:4318"` | OTLP endpoint | +| `otlp_endpoint` | `str` | From env / auto-configured when `BOTANU_API_KEY` is set / `"http://localhost:4318"` | OTLP endpoint | | `otlp_headers` | `dict` | `None` | Custom headers for OTLP exporter — always honored | -| `content_capture_rate` | `float` | `0.0` | Prompt/response capture rate (0.0–1.0). See the [Content Capture doc](../tracking/content-capture.md). | +| `content_capture_rate` | `float` | `0.0` | Prompt/response capture rate (0.0–1.0). See [Content Capture](../tracking/content-capture.md). | +| `pii_scrub_enabled` | `bool` | `True` | In-process PII scrub of captured content | +| `pii_scrub_use_presidio` | `bool` | `False` | Add Microsoft Presidio NER to the scrub pipeline | | `max_export_batch_size` | `int` | `512` | Max spans per batch | | `max_queue_size` | `int` | `65536` | Max spans in queue (~64 MB at ~1 KB/span) | | `schedule_delay_millis` | `int` | `5000` | Delay between batch exports | | `export_timeout_millis` | `int` | `30000` | Timeout for export operations | -| `propagation_mode` | `str` | `"lean"` | `"full"` (recommended) or `"lean"` (deprecated — will be removed) | | `auto_instrument_packages` | `list` | See below | Packages to auto-instrument | +`BOTANU_API_KEY` is not a field on the dataclass. When the env var is set, `BotanuConfig` auto-configures `otlp_endpoint` to `https://ingest.botanu.ai` and injects the bearer token into `otlp_headers` — but only for botanu-trusted hosts (any `*.botanu.ai` plus `localhost`). + ### Constructor ```python @@ -121,29 +123,34 @@ print(config.to_dict()) ```yaml service: - name: string # Service name - version: string # Service version - namespace: string # Service namespace - environment: string # Deployment environment + name: string + version: string + namespace: string + environment: string resource: - auto_detect: boolean # Auto-detect cloud resources + auto_detect: boolean otlp: - endpoint: string # OTLP endpoint URL - headers: # Custom headers + endpoint: string + headers: header-name: value export: - batch_size: integer # Max spans per batch - queue_size: integer # Max spans in queue - delay_ms: integer # Delay between exports - export_timeout_ms: integer # Export timeout - -propagation: - mode: string # "lean" or "full" - -auto_instrument_packages: # List of packages to instrument + batch_size: integer + queue_size: integer + delay_ms: integer + export_timeout_ms: integer + +eval: + content_capture_rate: float + pii: + enabled: boolean + use_presidio: boolean + replacement: string + disable_patterns: [string] + +auto_instrument_packages: - package_name ``` @@ -297,7 +304,6 @@ if not is_enabled(): | Variable | Description | Default | |----------|-------------|---------| | `BOTANU_ENVIRONMENT` | Fallback for environment | `"production"` | -| `BOTANU_PROPAGATION_MODE` | `"lean"` or `"full"` | `"lean"` | | `BOTANU_AUTO_DETECT_RESOURCES` | Auto-detect cloud resources | `"true"` | | `BOTANU_CONFIG_FILE` | Path to YAML config file | None | | `BOTANU_COLLECTOR_ENDPOINT` | Override for OTLP endpoint | None | @@ -309,8 +315,7 @@ if not is_enabled(): ## RunContext -Model for run metadata. Created automatically by `@botanu_workflow` and -`run_botanu`. +Model for run metadata. Created automatically by `botanu.event(...)`. ```python from botanu.models.run_context import RunContext @@ -365,9 +370,15 @@ def from_baggage(cls, baggage: Dict[str, str]) -> Optional[RunContext] Serialise to baggage format. ```python -def to_baggage_dict(self, lean_mode: Optional[bool] = None) -> Dict[str, str] +def to_baggage_dict(self) -> Dict[str, str] ``` +Always included: `botanu.run_id`, `botanu.workflow`, `botanu.event_id`, `botanu.customer_id`, `botanu.environment`. + +Included when set: `botanu.tenant_id`, `botanu.parent_run_id`, `botanu.root_run_id`, `botanu.attempt`, `botanu.retry_of_run_id`, `botanu.deadline`, `botanu.cancelled`. + +The `RunContextEnricher` stamps the first seven on downstream spans; the remaining five are for `from_baggage` to reconstruct retry and deadline state across process boundaries. + #### to_span_attributes() Serialise to span attributes. diff --git a/docs/api/decorators.md b/docs/api/decorators.md deleted file mode 100644 index 31591be..0000000 --- a/docs/api/decorators.md +++ /dev/null @@ -1,118 +0,0 @@ -# Decorators API Reference - -## @botanu_workflow - -The primary decorator for creating workflow runs with automatic context propagation. - -```python -from botanu import botanu_workflow - -@botanu_workflow( - name: str, - *, - event_id: Union[str, Callable[..., str]], - customer_id: Union[str, Callable[..., str]], - step: Optional[str] = None, - environment: Optional[str] = None, - tenant_id: Optional[str] = None, - auto_outcome_on_success: bool = True, - span_kind: SpanKind = SpanKind.SERVER, -) -``` - -### Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `name` | `str` | Required | Workflow name (low cardinality, e.g. `"Customer Support"`) | -| `event_id` | `str \| Callable` | Required | Business transaction identifier (e.g. ticket ID). Can be a static string or a callable that receives the same `(*args, **kwargs)` as the decorated function. | -| `customer_id` | `str \| Callable` | Required | End-customer being served (e.g. org ID). Same static/callable rules as `event_id`. | -| `step` | `str` | `None` | Step name within a multi-step workflow (e.g. `"classify"`, `"research"`). ⚠ Accepted and stored on `RunContext` but **not yet emitted as a span attribute** — the collector servicegraph work that consumes it has not landed. Forward-compatible. | -| `environment` | `str` | From env | Deployment environment | -| `tenant_id` | `str` | `None` | Tenant identifier for multi-tenant systems | -| `auto_outcome_on_success` | `bool` | `True` | Emit `"success"` outcome if no exception | -| `span_kind` | `SpanKind` | `SERVER` | OpenTelemetry span kind | - -### Example - -```python -from botanu import botanu_workflow - -# Static values: -@botanu_workflow("my-workflow", event_id="evt-001", customer_id="cust-42") -def do_work(): - result = do_something() - return result - -# Dynamic values extracted from function arguments: -@botanu_workflow( - "my-workflow", - event_id=lambda request: request.event_id, - customer_id=lambda request: request.customer_id, -) -async def handle_request(request): - ... -``` - -### Span Attributes - -| Attribute | Description | -|-----------|-------------| -| `botanu.run_id` | Generated UUIDv7 | -| `botanu.workflow` | `name` parameter | -| `botanu.event_id` | Resolved `event_id` | -| `botanu.customer_id` | Resolved `customer_id` | -| `botanu.environment` | Deployment environment | -| `botanu.tenant_id` | Tenant identifier (if provided) | - -### Alias - -`workflow` is an alias for `botanu_workflow`: - -```python -from botanu import workflow - -@workflow("my-workflow", event_id="evt-001", customer_id="cust-42") -def do_work(): - ... -``` - ---- - -## run_botanu - -Context manager alternative to `@botanu_workflow` for cases where you cannot -use a decorator (dynamic workflows, scripts, runtime-determined names). - -```python -from botanu import run_botanu - -with run_botanu( - name: str, - *, - event_id: str, - customer_id: str, - environment: Optional[str] = None, - tenant_id: Optional[str] = None, - auto_outcome_on_success: bool = True, - span_kind: SpanKind = SpanKind.SERVER, -) as run_ctx: RunContext -``` - -### Example - -```python -from botanu import run_botanu, emit_outcome - -with run_botanu("my-workflow", event_id="evt-001", customer_id="cust-42") as run: - result = do_something() - emit_outcome("success") -``` - -The yielded `RunContext` contains `run_id`, `workflow`, `event_id`, and other -metadata. Parameters are identical to `@botanu_workflow`. - -## See Also - -- [Quick Start](../getting-started/quickstart.md) -- [Run Context](../concepts/run-context.md) diff --git a/docs/api/event.md b/docs/api/event.md new file mode 100644 index 0000000..fc70dfb --- /dev/null +++ b/docs/api/event.md @@ -0,0 +1,88 @@ +# `event` and `step` + +## `botanu.event(...)` + +Primary integration point. Works as a [context manager](https://docs.python.org/3/library/contextlib.html), an [async context manager](https://docs.python.org/3/reference/datamodel.html#async-context-managers), or a decorator. + +```python +def event( + *, + event_id: str | Callable, + customer_id: str | Callable, + workflow: str, + environment: str | None = None, + tenant_id: str | None = None, + auto_outcome_on_success: bool = True, + capture_input: bool | None = None, + span_kind: SpanKind = SpanKind.SERVER, +) -> _Event +``` + +### Parameters + +| Parameter | Description | +| --- | --- | +| `event_id` | Business event identifier — the join key for outcome correlation. String, or a callable taking the decorated function's args. In context-manager form must be a resolved string. | +| `customer_id` | Customer being served. String or callable with the same rules as `event_id`. | +| `workflow` | Workflow name (low cardinality). Required. | +| `environment` | Deployment environment override. Falls back to `BOTANU_ENVIRONMENT` or `OTEL_DEPLOYMENT_ENVIRONMENT`. | +| `tenant_id` | Tenant identifier for multi-tenant apps. | +| `auto_outcome_on_success` | Mark the run `SUCCESS` on clean exit. Default `True`. | +| `capture_input` | Force content capture on/off. `None` (default) uses the sampled `content_capture_rate`. | +| `span_kind` | [OTel span kind](https://opentelemetry.io/docs/specs/otel/trace/api/#spankind). Default `SERVER`. | + +### Context manager + +```python +import botanu + +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + agent.run(ticket) +``` + +### Async context manager + +```python +async with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + await agent.arun(ticket) +``` + +### Decorator + +Supports callables for `event_id` and `customer_id`: + +```python +@botanu.event( + workflow="Support", + event_id=lambda ticket: ticket.id, + customer_id=lambda ticket: ticket.user_id, +) +def handle_ticket(ticket): + ... +``` + +Works for both sync and `async def` functions. + +## `botanu.step(name)` + +Context manager for multi-phase workflows. Nests inside an `event` scope and emits a span with `kind=INTERNAL`. + +```python +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + with botanu.step("retrieval"): + docs = vector_db.query(ticket.query) + with botanu.step("generation"): + response = llm.complete(docs) +``` + +Each step stamps `botanu.step=` on its span and propagates it via baggage. + +## Auto-enable + +`event()` calls `botanu.enable()` implicitly on first use. Explicit `botanu.enable(...)` is only needed to override config (custom endpoint, API key, content-capture rate). + +## See also + +- [Run Context](../concepts/run-context.md) +- [Context Propagation](../concepts/context-propagation.md) +- [Outcomes](../tracking/outcomes.md) diff --git a/docs/api/tracking.md b/docs/api/tracking.md index b3041e6..3d2df79 100644 --- a/docs/api/tracking.md +++ b/docs/api/tracking.md @@ -223,9 +223,12 @@ with track_db_operation( #### Example ```python -with track_db_operation(system="postgresql", operation="SELECT") as db: - result = await cursor.execute(query) - db.set_result(rows_returned=len(result)) +import asyncpg + +async with asyncpg.connect(dsn) as conn: + with track_db_operation(system="postgresql", operation="SELECT") as db: + rows = await conn.fetch("SELECT id FROM orders WHERE user_id = $1", user_id) + db.set_result(rows_returned=len(rows)) ``` --- @@ -390,13 +393,12 @@ def add_metadata(**kwargs: Any) -> MessagingTracker ### emit_outcome() -Emit a business outcome for the current span. +Stamp diagnostic fields on the current span. Authoritative event outcome is resolved server-side — see [Outcomes](../tracking/outcomes.md). ```python -from botanu import emit_outcome +import botanu -emit_outcome( - status: str, +botanu.emit_outcome( *, value_type: Optional[str] = None, value_amount: Optional[float] = None, @@ -410,20 +412,19 @@ emit_outcome( #### Parameters | Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `status` | `str` | Required | Outcome status: `"success"`, `"partial"`, `"failed"`, `"timeout"`, `"canceled"`, `"abandoned"` | -| `value_type` | `str` | `None` | Type of business value achieved | +| --- | --- | --- | --- | +| `value_type` | `str` | `None` | Type of business value (e.g. `"tickets_resolved"`) | | `value_amount` | `float` | `None` | Quantified value amount | -| `confidence` | `float` | `None` | Confidence score (0.0-1.0) | -| `reason` | `str` | `None` | Reason for the outcome | +| `confidence` | `float` | `None` | Confidence score, `0.0`–`1.0` | +| `reason` | `str` | `None` | Diagnostic reason | | `error_type` | `str` | `None` | Error classification (e.g. `"TimeoutError"`) | | `metadata` | `dict[str, str]` | `None` | Additional key-value metadata | #### Example ```python -emit_outcome("success", value_type="items_processed", value_amount=1) -emit_outcome("failed", error_type="TimeoutError", reason="LLM took >30s") +botanu.emit_outcome(value_type="tickets_resolved", value_amount=1) +botanu.emit_outcome(error_type="TimeoutError", reason="LLM took >30s") ``` --- diff --git a/docs/concepts/architecture.md b/docs/concepts/architecture.md index 2e323af..cda94c5 100644 --- a/docs/concepts/architecture.md +++ b/docs/concepts/architecture.md @@ -1,221 +1,45 @@ # Architecture -Botanu SDK follows a "thin SDK, smart collector" architecture. The SDK does minimal work in your application's hot path, delegating heavy processing to the OpenTelemetry Collector. +Thin SDK, smart collector. The SDK does minimal work in the hot path; everything heavy runs in the OpenTelemetry [Collector](https://opentelemetry.io/docs/collector/). -## Design Principles +## What the SDK does -### 1. Minimal Hot-Path Overhead +- Generates a UUIDv7 `run_id` per event +- Sets seven [W3C Baggage](https://www.w3.org/TR/baggage/) keys on the active [OTel context](https://opentelemetry.io/docs/specs/otel/context/) +- Records token counts as span attributes +- Exports spans via [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/) -The SDK only performs lightweight operations during request processing: -- Generate UUIDv7 `run_id` -- Read/write W3C Baggage -- Record token counts as span attributes +Target overhead: under 0.5 ms per request. -**Target overhead**: < 0.5ms per request +## What the collector does -### 2. OTel-Native - -Built on OpenTelemetry primitives, not alongside them: -- Uses standard `TracerProvider` -- Standard `SpanProcessor` for enrichment -- Standard OTLP export -- W3C Baggage for propagation - -### 3. Collector-Side Processing - -Heavy operations happen in the OTel Collector: -- PII redaction - Cost calculation from token counts - Vendor normalization - Cardinality management - Aggregation and sampling +- Belt-and-suspenders PII regex (SDK already scrubs captured content in-process — see [`botanu/sdk/pii.py`](../../src/botanu/sdk/pii.py)) +## SDK components -## SDK Components - -### BotanuConfig - -Central configuration for the SDK: - -```python -@dataclass -class BotanuConfig: - service_name: str - deployment_environment: str - otlp_endpoint: str - api_key: str # BOTANU_API_KEY; auto-configures endpoint + bearer - content_capture_rate: float # 0.0 default; see Content Capture - propagation_mode: str # "full" (recommended); "lean" is deprecated - auto_instrument_packages: List[str] -``` - -Content capture for eval (prompts/responses) is opt-in and disabled by -default. See [Content Capture](../tracking/content-capture.md). - -### RunContext - -Holds run metadata and provides serialization: - -```python -@dataclass -class RunContext: - run_id: str - root_run_id: str - workflow: str - event_id: str - customer_id: str - attempt: int - # ... -``` +- `BotanuConfig` — configuration dataclass. See [Configuration](../getting-started/configuration.md). +- `RunContext` — run metadata + serialization. See [Run Context](run-context.md). +- `RunContextEnricher` — [OTel SpanProcessor](https://opentelemetry.io/docs/specs/otel/trace/sdk/#span-processor) that reads baggage and stamps span attributes. +- Tracking helpers: `track_llm_call`, `track_db_operation`, `track_storage_operation`, `track_messaging_operation`. -### RunContextEnricher - -The only span processor in the SDK. Reads baggage, writes to spans: - -```python -class RunContextEnricher(SpanProcessor): - def on_start(self, span, parent_context): - for key in self._baggage_keys: - value = baggage.get_baggage(key, parent_context) - if value: - span.set_attribute(key, value) -``` - -### Tracking Helpers - -Context managers for manual instrumentation: - -- `track_llm_call()` - LLM/model operations -- `track_db_operation()` - Database operations -- `track_storage_operation()` - Object storage operations -- `track_messaging_operation()` - Message queue operations - -## Data Flow - -### 1. Run Initiation - -```python -@botanu_workflow("process", event_id="evt-001", customer_id="cust-42") -def do_work(): - pass -``` - -1. Generate UUIDv7 `run_id` -2. Create `RunContext` -3. Set baggage in current context -4. Start root span with run attributes - -### 2. Context Propagation - -```python -# Within the run -response = requests.get("https://api.example.com") -``` - -1. HTTP instrumentation reads current context -2. Baggage is injected into request headers -3. Downstream service extracts baggage -4. Context continues propagating - -### 3. Span Enrichment - -Every span (including auto-instrumented): - -1. `RunContextEnricher.on_start()` is called -2. Reads `botanu.run_id` from baggage -3. Writes to span attributes -4. Span is exported with run context - -### 4. Export and Processing - -1. `BatchSpanProcessor` batches spans -2. `OTLPSpanExporter` sends to collector -3. Collector processes (cost calc, PII redaction) -4. Spans written to backend - -## Why This Architecture? - -### SDK Stays Thin - -| Operation | Location | Reason | -|-----------|----------|--------| -| run_id generation | SDK | Must be synchronous | -| Baggage propagation | SDK | Process-local | -| Token counting | SDK | Available at call site | -| Cost calculation | Collector | Pricing tables change | -| PII redaction | Collector | Consistent policy | -| Aggregation | Collector | Reduces data volume | - -### No Vendor Lock-in - -- Standard OTel export format -- Any OTel-compatible backend works -- Collector processors are configurable - -### Minimal Dependencies - -Core SDK only requires `opentelemetry-api`: - -```toml -dependencies = [ - "opentelemetry-api >= 1.20.0", -] -``` - -Full SDK adds export capabilities: - -```toml -[project.optional-dependencies] -sdk = [ - "opentelemetry-sdk >= 1.20.0", - "opentelemetry-exporter-otlp-proto-http >= 1.20.0", -] -``` - -## Integration Points - -### Existing TracerProvider - -If you already have OTel configured: +## Integration ```python from opentelemetry import trace from botanu.processors.enricher import RunContextEnricher -# Add our processor to your existing provider provider = trace.get_tracer_provider() provider.add_span_processor(RunContextEnricher()) ``` -### Existing Instrumentation - -Botanu works alongside existing instrumentation: - -```python -# Your existing setup -from opentelemetry.instrumentation.requests import RequestsInstrumentor -RequestsInstrumentor().instrument() - -# Add Botanu -from botanu import enable -enable(service_name="my-service") - -# Both work together - requests are instrumented AND get run_id -``` - -## Performance Characteristics - -| Operation | Typical Latency | -|-----------|-----------------| -| `generate_run_id()` | < 0.01ms | -| `RunContextEnricher.on_start()` | < 0.05ms | -| `track_llm_call()` overhead | < 0.1ms | -| Baggage injection | < 0.01ms | - -Total SDK overhead per request: **< 0.5ms** +If you already have an OTel setup, the SDK detects it and adds itself alongside. See [Coexisting with existing OTel / Datadog](../integration/existing-otel.md). -## See Also +## See also -- [Run Context](run-context.md) - RunContext model details -- [Context Propagation](context-propagation.md) - How context flows -- [Collector Configuration](../integration/collector.md) - Collector setup +- [Run Context](run-context.md) +- [Context Propagation](context-propagation.md) +- [Collector](../integration/collector.md) diff --git a/docs/concepts/context-propagation.md b/docs/concepts/context-propagation.md index 6187dc1..22fdef9 100644 --- a/docs/concepts/context-propagation.md +++ b/docs/concepts/context-propagation.md @@ -1,264 +1,90 @@ # Context Propagation -Context propagation ensures that the `run_id` and other metadata flow through your entire application -- across function calls, HTTP requests, message queues, and async workers. +The SDK uses [W3C Baggage](https://www.w3.org/TR/baggage/) to carry business context across function calls, HTTP requests, message queues, and async workers. -## How It Works +## Keys -Botanu uses **W3C Baggage** for context propagation, the same standard used by OpenTelemetry for distributed tracing. - -``` -+-----------------------------------------------------------------+ -| HTTP Request Headers | -+-----------------------------------------------------------------+ -| traceparent: 00-{trace_id}-{span_id}-01 | -| baggage: botanu.run_id=019abc12...,botanu.workflow=process | -+-----------------------------------------------------------------+ -``` - -When you make an outbound HTTP request, the `botanu.run_id` travels in the `baggage` header alongside the trace context. - -## Propagation Modes - -### Full Mode (recommended) - -Full mode is the durable direction — every cross-service call carries the -complete baggage needed to reconstruct run context downstream. The SDK -propagates exactly these seven keys (defined as `BAGGAGE_KEYS_FULL` in -[`src/botanu/processors/enricher.py`](../../src/botanu/processors/enricher.py)): +The `RunContextEnricher` stamps these seven keys on every span that starts inside an event scope, by reading them from [W3C Baggage](https://www.w3.org/TR/baggage/) on the active [OTel context](https://opentelemetry.io/docs/specs/otel/context/): - `botanu.run_id` - `botanu.workflow` - `botanu.event_id` - `botanu.customer_id` - `botanu.environment` -- `botanu.tenant_id` -- `botanu.parent_run_id` - -Enable it explicitly: - -```bash -export BOTANU_PROPAGATION_MODE=full -``` - -```python -# Full mode baggage (~250 bytes, values-dependent) -baggage: botanu.run_id=019abc12-...,botanu.workflow=process,botanu.event_id=evt-001,botanu.customer_id=cust-456,botanu.environment=production,botanu.tenant_id=tnt-abc,botanu.parent_run_id=019abc11-... -``` - -Fields that live on `RunContext` but **not** in baggage — `root_run_id`, -`attempt`, `retry_of_run_id`, `deadline`, `cancelled` — are reconstructed -from local state, not carried on the wire. If you need them downstream, -propagate them yourself via your message envelope (see "Message Queue -Propagation" below). - -### Lean Mode (deprecated — will be removed) +- `botanu.tenant_id` (when set) +- `botanu.parent_run_id` (when nested inside a parent event) -Lean mode propagates only the first four keys from the full list. It was -the default in early 0.x releases and is still accepted for backward -compatibility, but it is **deprecated** — full mode will become the only -mode in a future release. Do not build new services assuming lean mode. +Additional keys (`botanu.root_run_id`, `botanu.attempt`, `botanu.retry_of_run_id`, `botanu.deadline`, `botanu.cancelled`) ride in the baggage dict too when they have non-default values, so that `RunContext.from_baggage(...)` on the receiving side of cross-process propagation can reconstruct retry and deadline state. -## In-Process Propagation +## In-process -Within a single process, context is propagated via Python's `contextvars`: +Context flows via Python [`contextvars`](https://docs.python.org/3/library/contextvars.html): ```python -from botanu import botanu_workflow +import botanu -@botanu_workflow("process", event_id="evt-001", customer_id="cust-456") -def do_work(): - # Context is set here - - do_something() # Inherits context - do_more_work() # Inherits context - save_result() # Inherits context +with botanu.event(event_id="ticket-42", customer_id="acme", workflow="Support"): + fetch_from_db() + call_llm() + publish_result() ``` -The `RunContextEnricher` span processor automatically reads baggage and writes to span attributes: - -```python -class RunContextEnricher(SpanProcessor): - def on_start(self, span, parent_context): - for key in ["botanu.run_id", "botanu.workflow"]: - value = baggage.get_baggage(key, parent_context) - if value: - span.set_attribute(key, value) -``` - -This ensures **every span** -- including auto-instrumented ones -- gets the `run_id`. - -## HTTP Propagation - -### Outbound Requests +The `RunContextEnricher` span processor stamps the seven keys on every span that starts inside the scope, including auto-instrumented ones. -When using instrumented HTTP clients (`requests`, `httpx`, `urllib3`), baggage is automatically propagated: +## HTTP -```python -import requests - -@botanu_workflow("process", event_id="evt-001", customer_id="cust-456") -def do_work(): - # Baggage is automatically added to headers - response = requests.get("https://api.example.com/data") -``` - -### Inbound Requests (Frameworks) +[OTel HTTP instrumentors](https://opentelemetry.io/docs/languages/python/instrumentation/) (requests, httpx, urllib3, aiohttp) propagate baggage automatically on outbound calls. -For web frameworks (`FastAPI`, `Flask`, `Django`), use the middleware to extract context: +For inbound, add the middleware to your web framework: ```python -# FastAPI +from fastapi import FastAPI from botanu.sdk.middleware import BotanuMiddleware app = FastAPI() app.add_middleware(BotanuMiddleware) - -@app.post("/tasks") -def process(request: Request): - # RunContext is extracted from incoming baggage - # or created if not present - pass ``` -## Message Queue Propagation - -For async messaging systems, you need to manually inject and extract context. +## Message queues -### Injecting Context (Producer) +Inject on the producer side: ```python -from botanu.sdk.context import get_run_id, get_baggage - -def publish_message(queue, payload): - run_id = get_run_id() +from botanu.sdk.context import get_baggage, get_run_id - message = { - "payload": payload, - "metadata": { - "run_id": run_id, - "workflow": get_baggage("botanu.workflow"), - "event_id": get_baggage("botanu.event_id"), - "customer_id": get_baggage("botanu.customer_id"), - } - } - queue.publish(message) +message = { + "payload": payload, + "metadata": { + "botanu.run_id": get_run_id(), + "botanu.workflow": get_baggage("botanu.workflow"), + "botanu.event_id": get_baggage("botanu.event_id"), + "botanu.customer_id": get_baggage("botanu.customer_id"), + }, +} ``` -### Extracting Context (Consumer) +Extract on the consumer side: ```python +import botanu from botanu.models.run_context import RunContext -from botanu import run_botanu - -def process_message(message): - baggage = message.get("metadata", {}) - ctx = RunContext.from_baggage(baggage) - - if ctx: - # Continue with existing context using context manager - with run_botanu( - ctx.workflow, - event_id=ctx.event_id, - customer_id=ctx.customer_id, - ): - do_work(message["payload"]) - else: - # Create new context - with run_botanu( - "process_message", - event_id="evt-fallback", - customer_id="unknown", - ): - do_work(message["payload"]) -``` - -## Cross-Service Propagation - -``` -+--------------+ HTTP +--------------+ Kafka +--------------+ -| Service A | ------------> | Service B | ------------> | Service C | -| | baggage: | | message | | -| run_id=X | run_id=X | run_id=X | run_id=X | run_id=X | -+--------------+ +--------------+ +--------------+ -``` - -The same `run_id` flows through all services, enabling: -- End-to-end cost attribution -- Cross-service trace correlation -- Distributed debugging - -## Baggage Size Limits - -W3C Baggage has practical size limits (most intermediaries allow 8 KB, but -individual hops may clip earlier). Typical sizes for botanu baggage: - -| Mode | Typical size | Notes | -| --- | --- | --- | -| Full (recommended) | ~250 bytes | 7 keys, well under any limit | -| Lean (deprecated) | ~120 bytes | 4 keys, historical only | - -## Propagation and Auto-Instrumentation - -The SDK works seamlessly with OTel auto-instrumentation: -```python -from botanu import enable +ctx = RunContext.from_baggage(message["metadata"]) -enable( - service_name="my-service", - auto_instrumentation=True, # Enable auto-instrumentation -) +with botanu.event(event_id=ctx.event_id, customer_id=ctx.customer_id, workflow=ctx.workflow): + do_work(message["payload"]) ``` -Auto-instrumented libraries will automatically propagate baggage: -- `requests`, `httpx`, `urllib3` (HTTP clients) -- `fastapi`, `flask`, `django` (Web frameworks) -- `celery` (Task queues) -- `grpc` (gRPC) - -## Debugging Propagation - -### Check Current Context +## Debugging ```python from botanu.sdk.context import get_baggage, get_run_id -run_id = get_run_id() -print(f"Current run_id: {run_id}") - -workflow = get_baggage("botanu.workflow") -print(f"Current workflow: {workflow}") -``` - -### Verify Header Propagation - -```python -# In your HTTP client -import httpx - -def debug_request(): - with httpx.Client() as client: - response = client.get( - "https://httpbin.org/headers", - ) - print(response.json()) - # Check for 'baggage' header in response +print(get_run_id()) +print(get_baggage("botanu.event_id")) ``` -## Common Issues - -### Context Not Propagating - -1. **Missing initialization**: Ensure `enable()` is called at startup -2. **Missing middleware**: Add `BotanuMiddleware` to your web framework -3. **Async context loss**: Use `contextvars`-aware async patterns - -### Duplicate run_ids - -1. **Multiple decorators**: Only use `@botanu_workflow` at the entry point -2. **Middleware + decorator**: Choose one, not both - -## See Also +## See also -- [Run Context](run-context.md) - Understanding the RunContext model -- [Architecture](architecture.md) - Overall SDK architecture +- [Run Context](run-context.md) +- [Architecture](architecture.md) diff --git a/docs/concepts/run-context.md b/docs/concepts/run-context.md index 249628d..08f6e7d 100644 --- a/docs/concepts/run-context.md +++ b/docs/concepts/run-context.md @@ -1,250 +1,104 @@ # Run Context -The Run Context is the core concept in Botanu SDK. It represents a single execution attempt of a business event that you want to track for cost attribution. +The run context is the data the SDK carries through one business event, connecting the root span, every auto-instrumented child span, and downstream services via [W3C Baggage](https://www.w3.org/TR/baggage/). -## Events and Runs +## Events and runs -An **event** is one business transaction -- a logical unit of work that produces a business outcome. Examples: +- **Event**: one business transaction (resolving a ticket, processing an order). Identified by `event_id`. +- **Run**: one execution attempt within an event. Retries share `event_id`, get new `run_id`. +- **Outcome**: resolved server-side from SoR connectors, HITL reviews, or eval verdicts. The SDK does not set it. -- Processing an incoming request -- Handling a scheduled job -- Executing a pipeline step -- Responding to a webhook +## `run_id` -A **run** is one execution attempt within an event. Each retry of the same event gets a new `run_id` but shares the same `event_id`. A single run may involve: +[UUIDv7](https://datatracker.ietf.org/doc/html/draft-ietf-uuidrev-rfc4122bis), generated per run. Time-sortable, globally unique. -- Multiple LLM calls (possibly to different providers) -- Database queries -- Storage operations -- External API calls -- Message queue operations +## `RunContext` -An event will have an **outcome** -- the business result of the work (success, failure, partial, etc.). - -## The run_id - -Every run is identified by a unique `run_id` -- a UUIDv7 that is: - -- **Time-sortable**: IDs generated later sort after earlier ones -- **Globally unique**: No collisions across services -- **Propagated automatically**: Flows through your entire application via W3C Baggage +Most code doesn't touch this directly. `botanu.event(...)` creates it: ```python -from botanu.models.run_context import generate_run_id +import botanu -run_id = generate_run_id() -# "019abc12-def3-7890-abcd-1234567890ab" +with botanu.event(event_id="ticket-42", customer_id="acme", workflow="Support"): + agent.run(ticket) ``` -## RunContext Model - -The `RunContext` dataclass holds all metadata for a run: +Decorator form: ```python -from botanu.models.run_context import RunContext - -ctx = RunContext.create( - workflow="process", - event_id="evt-001", - customer_id="cust-456", - environment="production", - tenant_id="tenant-123", -) - -print(ctx.run_id) # "019abc12-def3-7890-..." -print(ctx.root_run_id) # Same as run_id for top-level runs -print(ctx.attempt) # 1 (first attempt) +@botanu.event(workflow="Support", event_id=lambda t: t.id, customer_id=lambda t: t.user_id) +def handle_ticket(ticket): + ... ``` -### Key Fields +### Fields | Field | Description | -|-------|-------------| -| `run_id` | Unique identifier for this run (UUIDv7) | -| `root_run_id` | ID of the original run (for retries, same as `run_id` for first attempt) | -| `event_id` | Identifier for the business event (same across retries) | -| `customer_id` | Identifier for the customer this event belongs to | -| `workflow` | Workflow/function name | -| `environment` | Deployment environment (production, staging, etc.) | -| `attempt` | Attempt number (1 for first, 2+ for retries) | -| `tenant_id` | Optional tenant identifier for multi-tenant systems | - -## Creating Runs - -### Using the Decorator (Recommended) +| --- | --- | +| `run_id` | This run (UUIDv7) | +| `root_run_id` | First attempt's `run_id` | +| `event_id` | Business event identifier | +| `customer_id` | Customer this event belongs to | +| `workflow` | Workflow name | +| `environment` | Deployment environment | +| `attempt` | 1 for first, 2+ for retries | +| `tenant_id` | Optional, for multi-tenant apps | +| `parent_run_id` | If nested inside a parent event | -```python -from botanu import botanu_workflow - -@botanu_workflow("process", event_id="evt-001", customer_id="cust-456") -def do_work(): - # RunContext is automatically created and propagated - # All operations inside inherit the same run_id - pass -``` +### Manual creation -The `workflow` alias also works: - -```python -from botanu import workflow - -@workflow("process", event_id="evt-001", customer_id="cust-456") -def do_work(): - pass -``` - -### Using the Context Manager - -```python -from botanu import run_botanu - -def do_work(): - with run_botanu("process", event_id="evt-001", customer_id="cust-456"): - # RunContext is active within this block - pass -``` - -### Manual Creation +For custom propagation paths (message queues, cross-process handoffs): ```python from botanu.models.run_context import RunContext -ctx = RunContext.create( - workflow="process", - event_id="evt-001", - customer_id="cust-456", - tenant_id="acme-corp", -) - -# Use ctx.to_baggage_dict() to propagate via HTTP headers -# Use ctx.to_span_attributes() to add to spans +ctx = RunContext.create(workflow="Support", event_id="ticket-42", customer_id="acme") +baggage = ctx.to_baggage_dict() ``` -## Retry Handling - -When a run fails and is retried, use `create_retry()` to maintain lineage: +## Retries ```python -previous = RunContext.create( - workflow="process", - event_id="evt-001", - customer_id="cust-456", -) +retry = RunContext.create_retry(previous_ctx) -# First attempt fails... - -retry = RunContext.create_retry(previous) -print(retry.attempt) # 2 -print(retry.retry_of_run_id) # Previous run_id -print(retry.root_run_id) # Same as previous.run_id -print(retry.run_id) # New unique ID +retry.attempt # 2 +retry.retry_of_run_id # previous_ctx.run_id +retry.root_run_id # previous_ctx.run_id +retry.run_id # fresh UUIDv7 ``` -This enables: -- Tracking total attempts for a business event -- Correlating retries back to the previous request -- Calculating aggregate cost across all attempts - -## Deadlines and Cancellation - -RunContext supports deadline and cancellation tracking: +## Deadlines and cancellation ```python ctx = RunContext.create( - workflow="process", - event_id="evt-001", - customer_id="cust-456", - deadline_seconds=30.0, # 30 second deadline + workflow="Support", + event_id="ticket-42", + customer_id="acme", + deadline_seconds=30.0, ) -# Check deadline if ctx.is_past_deadline(): - raise TimeoutError("Deadline exceeded") - -# Check remaining time -remaining = ctx.remaining_time_seconds() + raise TimeoutError -# Request cancellation ctx.request_cancellation(reason="user") if ctx.is_cancelled(): - # Clean up and exit - pass + ... ``` -## Outcomes - -Record the business outcome of a run using `emit_outcome`: - -```python -from botanu import emit_outcome -from botanu.models.run_context import RunStatus - -emit_outcome( - RunStatus.SUCCESS, - value_type="task_completed", - value_amount=1.0, - confidence=0.95, - reason="Completed successfully", -) -``` - -`RunStatus` values: `SUCCESS`, `FAILURE`, `PARTIAL`, `TIMEOUT`, `CANCELED`. - -`emit_outcome` accepts these keyword arguments: `value_type`, `value_amount`, `confidence`, `reason`, `error_type`, `metadata`. - ## Serialization -### To Baggage (for HTTP propagation) - -```python -# Full mode (recommended): all seven baggage keys -baggage = ctx.to_baggage_dict(lean_mode=False) -# {"botanu.run_id": "...", "botanu.workflow": "...", "botanu.event_id": "...", -# "botanu.customer_id": "...", "botanu.environment": "...", -# "botanu.tenant_id": "...", "botanu.parent_run_id": "..."} - -# Lean mode (deprecated — will be removed): first four keys only -baggage = ctx.to_baggage_dict(lean_mode=True) -# {"botanu.run_id": "...", "botanu.workflow": "...", -# "botanu.event_id": "...", "botanu.customer_id": "..."} -``` - -Fields that live on `RunContext` but are **not** in baggage — -`root_run_id`, `attempt`, `retry_of_run_id`, `deadline`, `cancelled` — -are reconstructed from local state. - -### To Span Attributes - ```python -attrs = ctx.to_span_attributes() -# {"botanu.run_id": "...", "botanu.workflow": "...", ...} +ctx.to_baggage_dict() +ctx.to_span_attributes() +RunContext.from_baggage(baggage_dict) ``` -### From Baggage (receiving side) - -```python -ctx = RunContext.from_baggage(baggage_dict) -if ctx is None: - # Required fields missing, create new context - ctx = RunContext.create(workflow="unknown", event_id="evt-fallback", customer_id="unknown") -``` - -## Environment Variables - -| Variable | Description | Default | -|----------|-------------|---------| -| `BOTANU_ENVIRONMENT` | Default environment | `"production"` | -| `BOTANU_PROPAGATION_MODE` | `"lean"` or `"full"` | `"lean"` | - -## Best Practices +`to_baggage_dict()` always includes `botanu.run_id`, `botanu.workflow`, `botanu.event_id`, `botanu.customer_id`, `botanu.environment`. It adds `tenant_id`, `parent_run_id`, `root_run_id`, `attempt`, `retry_of_run_id`, `deadline`, `cancelled` when those are set on the context. -1. **One event per business outcome**: Don't create events for internal operations -2. **Use descriptive workflow names**: They appear in dashboards and queries -3. **Leverage tenant_id**: Essential for multi-tenant cost attribution -4. **Handle retries properly**: Always use `create_retry()` for retry attempts -5. **Always provide event_id and customer_id**: They are required for proper cost attribution +The `RunContextEnricher` stamps the first seven of those on every downstream span. The remaining five (`root_run_id`, `attempt`, `retry_of_run_id`, `deadline`, `cancelled`) are included for `from_baggage()` to reconstruct retry and deadline state when context crosses a process boundary (e.g. a queue worker). -## See Also +## See also -- [Context Propagation](context-propagation.md) - How context flows through your application -- [Outcomes](../tracking/outcomes.md) - Recording business outcomes +- [Context Propagation](context-propagation.md) +- [Outcomes](../tracking/outcomes.md) +- [event API reference](../api/event.md) diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 6d8bd63..ffe2968 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -1,54 +1,40 @@ # Configuration -botanu SDK can be configured with a single environment variable (the common -case) or through code / YAML for everything else. +One environment variable is enough for most installations. Code and YAML are available for anything more complex. -## Simplest config — just the API key - -For the Botanu Cloud SaaS, a single env var is enough: +## Minimal ```bash export BOTANU_API_KEY= python your_app.py ``` -When `BOTANU_API_KEY` is set, `enable()` auto-configures the OTLP endpoint -to `https://ingest.botanu.ai` and attaches the key as a bearer token on the -exporter. You do not need to set `OTEL_EXPORTER_OTLP_ENDPOINT` too. +When `BOTANU_API_KEY` is set, the SDK auto-configures the OTLP endpoint to `https://ingest.botanu.ai` and sends the key as a bearer token on every export. No other variables are required. -## ⚠ The API key is only sent to botanu-trusted endpoints +## The API key is only sent to botanu-trusted endpoints -If you override the OTLP endpoint to something that isn't owned by -botanu — e.g., you point OTLP at Datadog, Honeycomb, or a self-hosted -collector — **the SDK will not attach your botanu API key to the -exporter's Authorization header**. It is silently dropped. This is to -prevent a misconfigured `OTEL_EXPORTER_OTLP_ENDPOINT` from leaking your -tenant credentials to a third-party backend. +If you override the OTLP endpoint to anything outside the botanu-trusted list, the SDK will not attach your API key. This prevents a misconfigured `OTEL_EXPORTER_OTLP_ENDPOINT` from leaking tenant credentials to a third-party backend. -The trusted endpoint list is hard-coded in -[`src/botanu/sdk/config.py`](../../src/botanu/sdk/config.py): +Trusted hosts: -- any host ending in `.botanu.ai` (e.g., `ingest.botanu.ai`) -- `localhost`, `127.0.0.1`, `::1`, `0.0.0.0` (local development) +- Any host ending in `.botanu.ai` +- `localhost`, `127.0.0.1`, `::1`, `0.0.0.0` -For any other endpoint the exporter runs without a bearer header. If you -are shipping OTLP to a non-botanu backend and want auth on that exporter, -set `OTEL_EXPORTER_OTLP_HEADERS` or pass `otlp_headers=` to `BotanuConfig` -explicitly — those headers are always honored. +For any other endpoint, the exporter ships without a bearer header. If your self-hosted collector needs auth, set `OTEL_EXPORTER_OTLP_HEADERS` or pass `otlp_headers=` to `BotanuConfig` explicitly — those headers are always honored. ## Configuration precedence -1. **Code arguments** passed to `enable()` or `BotanuConfig(...)`. -2. **Environment variables** (`BOTANU_*`, `OTEL_*`). -3. **YAML config file** (`botanu.yaml` or a path you pass). -4. **Built-in defaults.** +1. Code arguments to `enable()` or `BotanuConfig(...)` +2. Environment variables (`BOTANU_*`, `OTEL_*`) +3. YAML config file (`botanu.yaml` or a path you pass) +4. Built-in defaults -## Code-based +## Code ```python -from botanu import enable +import botanu -enable( +botanu.enable( service_name="my-service", otlp_endpoint="https://ingest.botanu.ai", ) @@ -57,16 +43,15 @@ enable( ## Environment variables ```bash -export BOTANU_API_KEY= # enough on its own for Botanu Cloud +export BOTANU_API_KEY= export OTEL_SERVICE_NAME=my-service export BOTANU_ENVIRONMENT=production -export BOTANU_CONTENT_CAPTURE_RATE=0.10 # see Content Capture docs +export BOTANU_CONTENT_CAPTURE_RATE=0.10 ``` -## YAML file +## YAML ```yaml -# botanu.yaml service: name: my-service version: 1.0.0 @@ -75,57 +60,43 @@ service: otlp: endpoint: https://ingest.botanu.ai -content: - capture_rate: 0.10 - -propagation: - mode: full +eval: + content_capture_rate: 0.10 ``` -Load with: - ```python from botanu.sdk.config import BotanuConfig config = BotanuConfig.from_yaml("botanu.yaml") ``` -## Full `BotanuConfig` fields +## `BotanuConfig` fields ```python from dataclasses import dataclass @dataclass class BotanuConfig: - # Service identification - service_name: str = None # OTEL_SERVICE_NAME - service_version: str = None # OTEL_SERVICE_VERSION - service_namespace: str = None # OTEL_SERVICE_NAMESPACE - deployment_environment: str = None # OTEL_DEPLOYMENT_ENVIRONMENT - - # Resource detection - auto_detect_resources: bool = True # BOTANU_AUTO_DETECT_RESOURCES + service_name: str = None + service_version: str = None + service_namespace: str = None + deployment_environment: str = None - # OTLP exporter - otlp_endpoint: str = None # OTEL_EXPORTER_OTLP_ENDPOINT - otlp_headers: dict = None # Custom headers (always honored) + auto_detect_resources: bool = True - # API key (auto-configures endpoint + Authorization header on trusted hosts) - api_key: str = None # BOTANU_API_KEY + otlp_endpoint: str = None + otlp_headers: dict = None - # Span export max_export_batch_size: int = 512 max_queue_size: int = 65536 schedule_delay_millis: int = 5000 export_timeout_millis: int = 30000 - # Content capture for eval (0.0 disables; see Content Capture doc) - content_capture_rate: float = 0.0 # BOTANU_CONTENT_CAPTURE_RATE - - # Propagation mode — "full" is the target. "lean" is deprecated. - propagation_mode: str = "lean" # BOTANU_PROPAGATION_MODE + content_capture_rate: float = 0.0 ``` +`BOTANU_API_KEY` is not a field on the dataclass — when the env var is set, `BotanuConfig` auto-configures `otlp_endpoint` + `otlp_headers` for the botanu-trusted endpoint. + ## Environment variable reference ### OpenTelemetry standard @@ -143,60 +114,32 @@ class BotanuConfig: | Variable | Description | Default | | --- | --- | --- | -| `BOTANU_API_KEY` | API key for Botanu Cloud. Auto-configures the endpoint and bearer token on trusted hosts. | None | +| `BOTANU_API_KEY` | API key for Botanu Cloud. Auto-configures endpoint and bearer token on trusted hosts. | None | | `BOTANU_ENVIRONMENT` | Fallback for environment | `production` | -| `BOTANU_CONTENT_CAPTURE_RATE` | Content-capture sampling rate (0.0–1.0). See [Content Capture](../tracking/content-capture.md). | `0.0` | -| `BOTANU_PROPAGATION_MODE` | `full` (recommended) or `lean` (deprecated) | `lean` | +| `BOTANU_CONTENT_CAPTURE_RATE` | Content-capture sampling rate, `0.0`–`1.0`. See [Content Capture](../tracking/content-capture.md). | `0.0` | | `BOTANU_AUTO_DETECT_RESOURCES` | Auto-detect cloud resources | `true` | | `BOTANU_CONFIG_FILE` | Path to YAML config | None | -| `BOTANU_COLLECTOR_ENDPOINT` | Override for OTLP endpoint (same behavior as `OTEL_EXPORTER_OTLP_ENDPOINT`) | None | +| `BOTANU_COLLECTOR_ENDPOINT` | OTLP endpoint override (same behavior as `OTEL_EXPORTER_OTLP_ENDPOINT`) | None | ## Content capture -Prompt / response capture for the evaluator is disabled by default. -Turn it on with `BOTANU_CONTENT_CAPTURE_RATE`: - -```bash -export BOTANU_CONTENT_CAPTURE_RATE=0.10 # 10% of calls captured -``` - -See [Content Capture](../tracking/content-capture.md) for the full -pipeline, the three capture points, and the PII-scrubbing chain. - -## Propagation modes - -botanu's durable direction is **full-mode only** — every cross-service -call carries the complete run context in W3C Baggage. `lean` mode is still -present in the SDK for backward compatibility but will be removed; do not -depend on it. - -Set explicitly: +Prompt and response capture for the evaluator is disabled by default. Turn it on with: ```bash -export BOTANU_PROPAGATION_MODE=full +export BOTANU_CONTENT_CAPTURE_RATE=0.10 ``` -See [Context Propagation](../concepts/context-propagation.md) for the -exact field list. +The SDK scrubs PII in-process before writing the captured content. See [Content Capture](../tracking/content-capture.md) for the scrub pipeline, custom patterns, and opt-out flags. ## Zero-code initialization -If you want `enable()` to run without a line of code, import the -`botanu.register` module at process start. It calls `enable()` under the -hood: +If you can't edit the entry point (third-party process runner, gunicorn preload), import `botanu.register` at process start: ```bash python -c "import botanu.register" -m your_app ``` -Or, for containers, add `botanu.register` to your `PYTHONSTARTUP`: - -```bash -PYTHONSTARTUP=$(python -c "import botanu, os; print(os.path.dirname(botanu.__file__) + '/register.py')") -``` - -This is useful when you cannot edit the entry point (e.g., a third-party -process runner). +It calls `enable()` under the hood. ## Auto-instrumentation @@ -204,37 +147,30 @@ process runner). ```python [ - # HTTP clients "requests", "httpx", "urllib3", "aiohttp_client", - # Web frameworks "fastapi", "flask", "django", "starlette", - # Databases "sqlalchemy", "psycopg2", "asyncpg", "pymongo", "redis", - # Messaging "celery", "kafka_python", - # gRPC "grpc", - # GenAI "openai_v2", "anthropic", "vertexai", "google_genai", "langchain", - # Runtime "logging", ] ``` -### Customizing packages +### Customizing ```python -from botanu import enable +import botanu from botanu.sdk.config import BotanuConfig config = BotanuConfig(auto_instrument_packages=["requests", "fastapi", "openai_v2"]) -enable(config=config) +botanu.enable(config=config) ``` ### Disabling ```python -enable(auto_instrumentation=False) +botanu.enable(auto_instrumentation=False) ``` ## See also @@ -242,5 +178,5 @@ enable(auto_instrumentation=False) - [Quickstart](quickstart.md) - [Architecture](../concepts/architecture.md) - [Collector](../integration/collector.md) -- [Existing OTel / Datadog setup](../integration/existing-otel.md) +- [Coexisting with existing OTel / Datadog](../integration/existing-otel.md) - [Content Capture](../tracking/content-capture.md) diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index c4efb96..8ba0742 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -1,119 +1,97 @@ # Quickstart -Get event-level cost attribution working in 5 minutes. +Get event-level cost attribution working in five minutes. ## Prerequisites -- Python 3.9+ -- A botanu API key (sign up at [botanu.ai](https://botanu.ai)) +- Python 3.9 or newer +- A botanu API key from [app.botanu.ai](https://app.botanu.ai) -## Step 1: Install +## 1. Install ```bash pip install botanu ``` -## Step 2: Set one environment variable +## 2. Set the API key ```bash export BOTANU_API_KEY= ``` -That's it for the Botanu Cloud SaaS. The SDK auto-configures the OTLP -endpoint to `https://ingest.botanu.ai` and attaches your API key as a -bearer token. +The SDK auto-configures the OTLP endpoint to `https://ingest.botanu.ai` and sends the key as a bearer token. -### Alternative — self-hosted or local collector - -If you run your own OTel collector, point at it explicitly: +Running your own collector instead? Point at it directly and skip the bearer header: ```bash export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 export OTEL_SERVICE_NAME=my-service ``` -See [Collector](../integration/collector.md). Note: the SDK does not -attach your `BOTANU_API_KEY` to non-botanu endpoints — set -`OTEL_EXPORTER_OTLP_HEADERS` if your self-hosted collector needs auth. - -## Step 3: Enable SDK +## 3. Wrap your agent ```python -from botanu import enable +import botanu -enable() +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + agent.run(ticket) ``` -Call `enable()` once at application startup. It reads configuration from environment variables — no hardcoded values needed. +Every LLM call, HTTP call, and database query inside the block is captured and stamped with `event_id`, `customer_id`, and `workflow`. There's no separate `enable()` call — the SDK initializes itself on the first `event` call. -> **Already using Datadog or another OTel APM?** `enable()` auto-detects -> your existing TracerProvider and adds botanu alongside without -> disturbing your sampling ratio or APM bill. See [Using botanu with an -> existing OTel / APM setup](../integration/existing-otel.md). +## Decorator form -## Step 4: Define Entry Point +For long-lived handlers, a decorator reads cleaner: ```python -from botanu import botanu_workflow +import botanu -@botanu_workflow("my-workflow", event_id="evt-001", customer_id="cust-42") -async def do_work(): - data = await db.query(...) - result = await llm.complete(data) - return result +@botanu.event( + workflow="Support", + event_id=lambda ticket: ticket.id, + customer_id=lambda ticket: ticket.user_id, +) +def handle_ticket(ticket): + return agent.run(ticket) ``` -All LLM calls, database queries, and HTTP requests inside the function are automatically tracked with the same `run_id` tied to the `event_id`. +Works for both sync and `async def` functions. -## Complete Example +## Multi-phase workflows -**Entry service** (`entry/app.py`): +Break a multi-step event into phases with `step`: ```python -from botanu import enable, botanu_workflow - -enable() - -@botanu_workflow( - "my-workflow", - event_id=lambda req: req.event_id, - customer_id=lambda req: req.customer_id, -) -async def handle_request(req): - data = await fetch_data(req) - return await process(data) +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + with botanu.step("retrieval"): + docs = vector_db.query(ticket.query) + with botanu.step("generation"): + response = llm.complete(docs) ``` -No `emit_outcome("success")` call is needed — event outcome is resolved -server-side from eval verdict / HITL / SoR. See [Outcomes](../tracking/outcomes.md). - -**Downstream service** (`intermediate/app.py`): +Each phase produces its own span and inherits the event's business context. -```python -from botanu import enable +## What gets stamped on every span -enable() # propagates run_id from incoming request — no decorator needed -``` +| Attribute | Example | +| --- | --- | +| `botanu.run_id` | `019abc12-3f4d-7...` | +| `botanu.event_id` | `ticket-42` | +| `botanu.customer_id` | `acme-corp` | +| `botanu.workflow` | `Support` | +| `botanu.environment` | `production` | +| `gen_ai.usage.input_tokens` | `150` | +| `gen_ai.usage.output_tokens` | `200` | -## What Gets Tracked +All spans produced by auto-instrumentation (OpenAI, Anthropic, LangChain, httpx, SQLAlchemy, Redis, ~25 others) inherit these attributes automatically. -| Attribute | Example | Description | -|-----------|---------|-------------| -| `botanu.run_id` | `019abc12-...` | Unique run identifier (UUIDv7) | -| `botanu.workflow` | `my-workflow` | Workflow name | -| `botanu.event_id` | `evt-001` | Business event identifier | -| `botanu.customer_id` | `cust-42` | Customer identifier | -| `gen_ai.usage.input_tokens` | `150` | LLM input tokens | -| `gen_ai.usage.output_tokens` | `200` | LLM output tokens | -| `db.system` | `postgresql` | Database system | +## Already using Datadog or another OTel APM? -All spans across all services share the same `run_id`, enabling cost-per-event analytics. +The SDK detects existing `TracerProvider` setups and adds itself alongside without disturbing your sampling ratio. See [Coexisting with existing OTel / Datadog](../integration/existing-otel.md). -## Next Steps +## Next -- [Configuration](configuration.md) — environment variables and YAML config -- [Using botanu with existing OTel / Datadog](../integration/existing-otel.md) — brownfield detection + sampling preservation -- [Content Capture](../tracking/content-capture.md) — enabling prompt/response capture for eval -- [Outcomes](../tracking/outcomes.md) — how event outcome is resolved -- [Kubernetes Deployment](../integration/kubernetes.md) — zero-code instrumentation at scale -- [Context Propagation](../concepts/context-propagation.md) — how run_id flows across services +- [Configuration](configuration.md) — env vars, YAML, and advanced options +- [Concepts: Run Context](../concepts/run-context.md) — what `event_id` buys you +- [Outcomes](../tracking/outcomes.md) — how success/failure is resolved +- [Kubernetes Deployment](../integration/kubernetes.md) diff --git a/docs/index.md b/docs/index.md index c4c78df..15652fe 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,74 +1,64 @@ -# Botanu SDK Documentation +# Botanu Python SDK -Botanu SDK provides OpenTelemetry-native event-level cost attribution for AI -workflows. +OpenTelemetry-native event-level cost attribution for AI workflows. -## Overview +## What it does -Traditional observability tools trace individual requests. But AI workflows are -different — a single business event (resolving a support ticket, processing an -order) might involve multiple runs spanning LLM calls, retries, tool executions, -and data operations across different services and vendors. +Traditional observability tools trace individual requests. AI workflows are different — one business event (resolving a support ticket, processing an order) usually spans multiple LLM calls, retries, tool executions, and data operations across services and vendors. -Botanu introduces **event-level attribution**: a stable `event_id` that follows -your entire business transaction, enabling you to answer "How much did this -event cost?" and "What was the outcome?" +The SDK stamps a stable `event_id` on every span produced inside one business event, so cost and outcome can be joined server-side. One wrap around the agent captures everything that happens inside. + +## Quickstart + +```python +import botanu + +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + agent.run(ticket) +``` + +That single wrap captures every LLM call, HTTP call, and DB call the agent makes, and ties them to `event_id`. Authentication and endpoint setup come from `BOTANU_API_KEY` in the environment — no explicit `enable()` needed. + +See [Quick Start](getting-started/quickstart.md) for the full five-minute walkthrough. ## Documentation ### Getting Started -- [Installation](getting-started/installation.md) — Install and configure the SDK -- [Quick Start](getting-started/quickstart.md) — Get up and running in 5 minutes -- [Configuration](getting-started/configuration.md) — Environment variables and options +- [Installation](getting-started/installation.md) +- [Quick Start](getting-started/quickstart.md) +- [Configuration](getting-started/configuration.md) -### Core Concepts +### Concepts -- [Run Context](concepts/run-context.md) — Events, runs, and context propagation -- [Context Propagation](concepts/context-propagation.md) — How context flows across services -- [Architecture](concepts/architecture.md) — SDK design and component overview +- [Run Context](concepts/run-context.md) — events, runs, and how context propagates +- [Context Propagation](concepts/context-propagation.md) — how `event_id` flows across services +- [Architecture](concepts/architecture.md) ### Tracking -- [LLM Tracking](tracking/llm-tracking.md) — Track AI model calls and token usage -- [Data Tracking](tracking/data-tracking.md) — Track database, storage, and messaging operations -- [Content Capture](tracking/content-capture.md) — Capture prompts and responses for eval (opt-in) -- [Outcomes](tracking/outcomes.md) — Record diagnostic context; how event outcome is actually resolved +- [LLM Tracking](tracking/llm-tracking.md) +- [Data Tracking](tracking/data-tracking.md) +- [Content Capture](tracking/content-capture.md) — prompts and responses for eval (opt-in) +- [Outcomes](tracking/outcomes.md) — diagnostic annotations; authoritative outcome resolution ### Integration -- [Auto-Instrumentation](integration/auto-instrumentation.md) — Supported libraries and frameworks -- [Kubernetes Deployment](integration/kubernetes.md) — Zero-code instrumentation at scale -- [Using botanu with existing OTel / Datadog](integration/existing-otel.md) — Brownfield detection, sampling preservation, ddtrace coexistence -- [Collector](integration/collector.md) — Botanu Cloud collector endpoints and auth +- [Auto-Instrumentation](integration/auto-instrumentation.md) — supported libraries +- [Kubernetes Deployment](integration/kubernetes.md) +- [Coexisting with existing OTel / Datadog](integration/existing-otel.md) +- [Collector](integration/collector.md) — endpoints and auth ### Patterns -- [Best Practices](patterns/best-practices.md) — Recommended patterns for production use -- [Anti-Patterns](patterns/anti-patterns.md) — Common mistakes to avoid +- [Best Practices](patterns/best-practices.md) +- [Anti-Patterns](patterns/anti-patterns.md) ### API Reference -- [Decorators](api/decorators.md) — `@botanu_workflow` and related decorators -- [Tracking API](api/tracking.md) — Manual tracking context managers -- [Configuration API](api/configuration.md) — `BotanuConfig` and initialization - -## Quick Example - -```python -from botanu import enable, botanu_workflow - -enable() # reads BOTANU_API_KEY from env; auto-configures endpoint - -@botanu_workflow("my-workflow", event_id="evt-001", customer_id="cust-42") -async def do_work(): - return await do_something() -``` - -Outcome is resolved server-side from eval verdict / HITL / SoR — you do -not need to call `emit_outcome` to record success. See -[Outcomes](tracking/outcomes.md) for diagnostic annotations that are -still useful. +- [event & step](api/event.md) — the primary API +- [Tracking](api/tracking.md) — manual tracking context managers +- [Configuration](api/configuration.md) — `BotanuConfig` and initialization ## License diff --git a/docs/integration/collector.md b/docs/integration/collector.md index 46b88e0..2154555 100644 --- a/docs/integration/collector.md +++ b/docs/integration/collector.md @@ -7,7 +7,7 @@ Botanu hosts a multi-tenant OpenTelemetry Collector — you don't need to deploy The SDK sends telemetry to Botanu's hosted collector via OTLP over HTTPS. The collector handles: - **Tenant isolation** — API key in the OTLP Authorization header identifies your tenant -- **PII scrubbing** — Configurable redaction of sensitive data patterns +- **PII scrubbing (belt-and-suspenders)** — Regex pass over `botanu.eval.*` attributes; the SDK already scrubs captured content in-process before it leaves your application - **Enrichment** — Vendor normalization, span classification - **Aggregation** — Event-level accumulation (spans → run summaries) - **Cost computation** — Token-to-dollar conversion using the pricing rate card @@ -86,13 +86,23 @@ Dashboard (app.botanu.ai) ## PII Handling -The collector applies PII scrubbing rules before data is stored. By default: +PII scrubbing runs in **two layers** — the SDK strips captured content +in-process (your application sees only `[REDACTED]` in exported spans), +and the collector runs a second regex pass as belt-and-suspenders. + +**SDK (in-process, first line of defense):** + +- Runs on text passed to `set_input_content` / `set_output_content` / + `set_retrieval_content` and on `botanu.event(...)` auto-captured payloads +- On by default — opt-out via `BOTANU_PII_SCRUB_ENABLED=false` +- See [Content Capture → PII handling](../tracking/content-capture.md#pii-handling) + +**Collector (belt-and-suspenders):** - Email addresses, phone numbers, SSNs, and credit card numbers are redacted - Raw prompt/completion content is stripped (token counts are preserved for cost) - Only aggregated summaries (cost, latency, token counts, outcome status) are stored - -Configure additional scrubbing rules via the dashboard at **Settings → Data Privacy**. +- Configure additional scrubbing rules via the dashboard at **Settings → Data Privacy** ## Sampling diff --git a/docs/integration/existing-otel.md b/docs/integration/existing-otel.md index c15689f..e74f49f 100644 --- a/docs/integration/existing-otel.md +++ b/docs/integration/existing-otel.md @@ -132,9 +132,8 @@ In the parallel path: `trace.set_tracer_provider(...)`. - ddtrace keeps handling spans for ddtrace decorators and for Datadog auto-instrumentation. -- botanu's decorators (`@botanu_workflow`, `track_llm_call`, etc.) get - their spans from the botanu provider, which forwards to the botanu - collector. +- botanu's API (`botanu.event`, `track_llm_call`, etc.) gets its spans + from the botanu provider, which forwards to the botanu collector. The two tracing systems coexist. Nothing is stolen, nothing is wrapped. @@ -169,28 +168,25 @@ first-time users. --- -## Using botanu decorators +## Using the botanu API -Regardless of which path `enable()` takes, the decorator API is the same: +Regardless of which path `enable()` takes, the API is the same: ```python -from botanu import botanu_workflow, emit_outcome +import botanu -@botanu_workflow( - name="Customer Support", +@botanu.event( + workflow="Customer Support", event_id=lambda req: req.ticket_id, customer_id=lambda req: req.org_id, ) async def handle_ticket(req): result = await process(req) - emit_outcome("success", value_type="tickets_resolved", value_amount=1) + botanu.emit_outcome(value_type="tickets_resolved", value_amount=1) return result ``` -Auto-instrumented spans (OpenAI SDK, HTTP clients, DB drivers) inside the -decorated call inherit the run context through W3C Baggage, so cost -attribution works even for spans your code never directly creates. See -[Auto-Instrumentation](auto-instrumentation.md). +Auto-instrumented spans (OpenAI SDK, HTTP clients, DB drivers) inside the event scope inherit the run context through [W3C Baggage](https://www.w3.org/TR/baggage/), so cost attribution works even for spans your code never directly creates. See [Auto-Instrumentation](auto-instrumentation.md). --- @@ -248,7 +244,7 @@ Sanity checks in order: 1. Open your existing APM — confirm span volume is unchanged. 2. Open botanu — confirm spans are arriving. A run created by - `@botanu_workflow` carries `botanu.run_id`, `botanu.workflow`, + `botanu.event(...)` carries `botanu.run_id`, `botanu.workflow`, `botanu.event_id`. 3. If both arrive, you're done. @@ -284,7 +280,7 @@ line starting `Preserved your sampling ratio`. 1. Verify `enable()` was called (or `RunContextEnricher` was attached manually). -2. Verify an entry-point function is wrapped in `@botanu_workflow` — the +2. Verify an entry-point function or block uses `botanu.event(...)` — the baggage is set on entry and inherited by child spans from there. 3. Verify the W3C Baggage propagator is active: `from opentelemetry import propagate; propagate.get_global_textmap()` diff --git a/docs/integration/kubernetes.md b/docs/integration/kubernetes.md index 765c736..01b797f 100644 --- a/docs/integration/kubernetes.md +++ b/docs/integration/kubernetes.md @@ -9,8 +9,8 @@ For organizations with thousands of applications, modifying code in every repo i ## What Requires Code Changes | Service Type | Code Change | Config Change | -|--------------|-------------|---------------| -| **Entry point** | `@botanu_workflow` decorator (generates `run_id`) | K8s annotation | +| --- | --- | --- | +| **Entry point** | `botanu.event(...)` wrap (generates `run_id`) | K8s annotation | | **Intermediate services** | None | K8s annotation only | **Entry point** = The service where the business transaction starts (API gateway, webhook handler, queue consumer). @@ -38,8 +38,8 @@ With zero-code instrumentation, the following are automatically traced: │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │ │ App A │ │ App B │ │ App C │ │ │ │ (entry) │ │ (no change) │ │ (no change) │ │ -│ │ @botanu_ │ │ │ │ │ │ -│ │ workflow │ │ │ │ │ │ +│ │ botanu. │ │ │ │ │ │ +│ │ event(...) │ │ │ │ │ │ │ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ │ │ │ │ │ │ │ OTel auto-injected via Operator │ @@ -247,23 +247,24 @@ spec: ## Entry Point Service (Code Change Required) -The entry point service is the **only** service that needs a code change. It must use `@botanu_workflow` to generate the `run_id`: +The entry point service is the **only** service that needs a code change. It must wrap the request handler with `botanu.event(...)` so the `run_id` and business context are generated: ```python -from botanu import enable, botanu_workflow - -enable(service_name="entry-service") - -@botanu_workflow("do_work", event_id=event_id, customer_id=customer_id) -def do_work(event_id: str, customer_id: str): - data = do_something() - result = process(data) - return result +import botanu + +@botanu.event( + workflow="OrderFulfillment", + event_id=lambda req: req.order_id, + customer_id=lambda req: req.customer_id, +) +def handle(req): + data = do_something(req) + return process(data) ``` -The `@botanu_workflow` decorator generates a `run_id` and propagates it via W3C Baggage to all downstream calls. +`botanu.event(...)` generates the `run_id` and propagates it via [W3C Baggage](https://www.w3.org/TR/baggage/) to every downstream call made inside the scope. -**Downstream services (B, C, D, etc.) need zero code changes** -- they just need the K8s annotation. +**Downstream services (B, C, D, etc.) need zero code changes** — they just need the K8s annotation to enable OTel auto-instrumentation, which picks up the baggage automatically. ## Helm Chart @@ -351,7 +352,7 @@ For 2000+ applications: 2. **Phase 2**: Install OTel Operator 3. **Phase 3**: Create Instrumentation resource 4. **Phase 4**: Add annotations via GitOps (batch by team/namespace) -5. **Phase 5**: Instrument entry points with `@botanu_workflow` +5. **Phase 5**: Instrument entry points with `botanu.event(...)` Each phase is independent. Annotations can be rolled out gradually. diff --git a/docs/patterns/anti-patterns.md b/docs/patterns/anti-patterns.md index 426e796..4b9f3af 100644 --- a/docs/patterns/anti-patterns.md +++ b/docs/patterns/anti-patterns.md @@ -1,485 +1,143 @@ # Anti-Patterns -Common mistakes to avoid when using Botanu SDK. +Patterns that look reasonable at first but break cost-per-outcome attribution or make your dashboards noisy. Each section shows the symptom and the cleaner alternative. -## Run Design Anti-Patterns +## Splitting one business event into multiple events -### Creating Runs for Internal Operations - -**Don't** create runs for internal functions: - -```python -# BAD - Too many runs -@botanu_workflow("fetch_data", event_id=event_id, customer_id=customer_id) # Don't do this -async def fetch_data(event_id, customer_id): - return await db.query(...) - -@botanu_workflow("do_work", event_id=event_id, customer_id=customer_id) # Or this -async def do_work(event_id, customer_id): - return await llm.complete(...) - -@botanu_workflow("handle_request", event_id=event_id, customer_id=customer_id) -async def handle_request(event_id, customer_id): - data = await fetch_data(event_id, customer_id) - result = await do_work(event_id, customer_id) - return result -``` - -**Do** use a single run at the entry point: - -```python -# GOOD - One run for the business outcome -@botanu_workflow("handle_request", event_id=event_id, customer_id=customer_id) -async def handle_request(event_id: str, customer_id: str): - data = await fetch_data(event_id) # Not decorated - result = await do_work(data) # Not decorated - emit_outcome("success", value_type="requests_processed", value_amount=1) - return result -``` - -### Nesting @botanu_workflow Decorators - -**Don't** nest workflow decorators: - -```python -# BAD - Nested runs create confusion -@botanu_workflow("outer", event_id=event_id, customer_id=customer_id) -async def outer(): - await inner() # Creates a second run - -@botanu_workflow("inner", event_id=event_id, customer_id=customer_id) # Don't do this -async def inner(): - ... -``` - -**Do** use @botanu_workflow only at entry points: - -```python -# GOOD - Only entry point is decorated -@botanu_workflow("main_flow", event_id=event_id, customer_id=customer_id) -async def main_flow(): - await step_one() # No decorator - await step_two() # No decorator -``` - -### Generic Workflow Names - -**Don't** use vague names: - -```python -# BAD - Meaningless in dashboards -@botanu_workflow("process", event_id=event_id, customer_id=customer_id) -@botanu_workflow("handle", event_id=event_id, customer_id=customer_id) -@botanu_workflow("main", event_id=event_id, customer_id=customer_id) -@botanu_workflow("do_work", event_id=event_id, customer_id=customer_id) -``` - -**Do** use descriptive business names: - -```python -# GOOD - Clear in reports -@botanu_workflow("support_resolution", event_id=event_id, customer_id=customer_id) -@botanu_workflow("invoice_processing", event_id=event_id, customer_id=customer_id) -@botanu_workflow("lead_scoring", event_id=event_id, customer_id=customer_id) -@botanu_workflow("document_analysis", event_id=event_id, customer_id=customer_id) -``` - -## Outcome Anti-Patterns - -### Forgetting to Emit Outcomes - -**Don't** leave runs without outcomes: - -```python -# BAD - No outcome recorded -@botanu_workflow("process_order", event_id=order_id, customer_id=customer_id) -async def process_order(order_id, customer_id): - result = await process(order_id) - return result # Where's the outcome? -``` - -**Do** always emit an outcome: - -```python -# GOOD - Explicit outcome -@botanu_workflow("process_order", event_id=order_id, customer_id=customer_id) -async def process_order(order_id, customer_id): - try: - result = await process(order_id) - emit_outcome("success", value_type="orders_processed", value_amount=1) - return result - except Exception as e: - emit_outcome("failed", reason=type(e).__name__) - raise -``` - -### Multiple Outcomes Per Run - -**Don't** emit multiple outcomes: - -```python -# BAD - Multiple outcomes are confusing -@botanu_workflow("batch_processing", event_id=batch_id, customer_id=customer_id) -async def process_batch(items): - for item in items: - await process(item) - emit_outcome("success", value_type="item_processed") # Don't do this -``` - -**Do** emit one summary outcome: - -```python -# GOOD - One outcome at the end -@botanu_workflow("batch_processing", event_id=batch_id, customer_id=customer_id) -async def process_batch(items): - processed = 0 - for item in items: - await process(item) - processed += 1 - emit_outcome("success", value_type="items_processed", value_amount=processed) -``` - -### Missing Failure Reasons - -**Don't** emit failures without reasons: - -```python -# BAD - No context for debugging -except Exception: - emit_outcome("failed") # Why did it fail? - raise -``` - -**Do** include the failure reason: - -```python -# GOOD - Reason helps debugging -except ValidationError: - emit_outcome("failed", reason="validation_error") - raise -except RateLimitError: - emit_outcome("failed", reason="rate_limit_exceeded") - raise -except Exception as e: - emit_outcome("failed", reason=type(e).__name__) - raise -``` - -## LLM Tracking Anti-Patterns - -### Not Recording Tokens - -**Don't** skip token recording: +One event should cover one business transaction. Internal phases (retrieval, generation, validation) belong inside a single event, marked with `botanu.step(...)`. Otherwise each phase gets counted as a separate event and cost-per-outcome is computed per-phase instead of per-outcome. ```python -# BAD - No cost data -with track_llm_call(provider="openai", model="gpt-4"): - response = await client.chat.completions.create(...) - # Token usage not recorded -``` - -**Do** always record tokens: - -```python -# GOOD - Tokens enable cost calculation -with track_llm_call(provider="openai", model="gpt-4") as tracker: - response = await client.chat.completions.create(...) - tracker.set_tokens( - input_tokens=response.usage.prompt_tokens, - output_tokens=response.usage.completion_tokens, - ) -``` +# Each phase becomes its own event — event counts and cost-per-outcome +# get split across three unrelated rows in the dashboard. +@botanu.event(workflow="fetch", event_id=..., customer_id=...) +def fetch(req): ... -### Ignoring Cached Tokens +@botanu.event(workflow="process", event_id=..., customer_id=...) +def process(data): ... -**Don't** forget cache tokens (they have different pricing): - -```python -# BAD - Missing cache data -tracker.set_tokens( - input_tokens=response.usage.prompt_tokens, - output_tokens=response.usage.completion_tokens, -) +@botanu.event(workflow="send", event_id=..., customer_id=...) +def send(result): ... ``` -**Do** include cache breakdown: - ```python -# GOOD - Full token breakdown -tracker.set_tokens( - input_tokens=response.usage.prompt_tokens, - output_tokens=response.usage.completion_tokens, - cache_read_tokens=response.usage.cache_read_tokens, - cache_write_tokens=response.usage.cache_write_tokens, -) +# One event, three phases. Cost rolls up to one business transaction. +@botanu.event(workflow="Support", event_id=lambda r: r.id, customer_id=lambda r: r.user_id) +def handle(req): + with botanu.step("fetch"): + data = fetch(req) + with botanu.step("process"): + result = process(data) + with botanu.step("send"): + send(result) ``` -### Wrong Provider Names - -**Don't** use inconsistent provider names: - -```python -# BAD - Inconsistent naming -track_llm_call(provider="OpenAI", ...) # Mixed case -track_llm_call(provider="open-ai", ...) # Wrong format -track_llm_call(provider="gpt", ...) # Model as provider -``` +## Nesting `botanu.event(...)` inside another `botanu.event(...)` -**Do** use standard provider names (auto-normalized): +A nested event creates a second `run_id`, so what you meant as one transaction gets counted as two. Wrap at the outermost boundary only; use `botanu.step(...)` for finer granularity. ```python -# GOOD - Standard names (or let SDK normalize) -track_llm_call(provider="openai", ...) -track_llm_call(provider="anthropic", ...) -track_llm_call(provider="azure_openai", ...) +with botanu.event(event_id="ticket-1", customer_id="acme", workflow="Support"): + with botanu.event(event_id="ticket-1", customer_id="acme", workflow="Inner"): + ... ``` -## Configuration Anti-Patterns - -### Sampling for Cost Attribution - -### Hardcoding Configuration - -**Don't** hardcode production values: - ```python -# BAD - Hardcoded -enable( - service_name="my-service", - otlp_endpoint="http://prod-collector.internal:4318", -) +with botanu.event(event_id="ticket-1", customer_id="acme", workflow="Support"): + with botanu.step("classification"): + ... ``` -**Do** use environment variables: - -```python -# GOOD - Environment-based -enable(service_name=os.environ["OTEL_SERVICE_NAME"]) - -# Or use YAML with interpolation -# botanu.yaml -# otlp: -# endpoint: ${COLLECTOR_ENDPOINT} -``` +## Using a random UUID as `event_id` -### Disabling Auto-Instrumentation Unnecessarily +`event_id` is the join key. When a SoR webhook fires ("Zendesk ticket resolved") or an evaluator verdict lands, those systems need to find the matching event. Using a fresh random UUID each time means nothing ever matches, and your cost-per-outcome will stay at `pending` forever. -**Don't** disable auto-instrumentation without reason: +Use the identifier your downstream systems already know — the ticket ID, order ID, session ID. ```python -# BAD - Missing automatic tracing -enable( - service_name="my-service", - auto_instrumentation=False, # Why? -) +with botanu.event(event_id=uuid.uuid4().hex, customer_id=user.id, workflow="Support"): + agent.run(ticket) ``` -**Do** keep defaults or be selective: - ```python -# GOOD - Default instrumentation (auto_instrumentation=True by default) -enable(service_name="my-service") +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + agent.run(ticket) ``` -## Context Propagation Anti-Patterns +## High-cardinality workflow names -### Losing Context in Async Code - -**Don't** spawn tasks without context: +Workflow names drive filtering in the dashboard. If every request produces a new workflow name, the filter becomes unusable. Keep workflow names stable and descriptive. ```python -# BAD - Context lost -@botanu_workflow("parallel_work", event_id=event_id, customer_id=customer_id) -async def do_parallel_work(): - # These tasks don't inherit context - await asyncio.gather( - do_something(), - do_something_else(), - ) +workflow = f"request-{request.id}" +workflow = f"Support-{datetime.now().isoformat()}" ``` -**Do** ensure context propagates: - ```python -# GOOD - Context flows through asyncio -@botanu_workflow("parallel_work", event_id=event_id, customer_id=customer_id) -async def do_parallel_work(): - # asyncio with contextvars works correctly - await asyncio.gather( - do_something(), # Inherits context - do_something_else(), # Inherits context - ) +workflow = "Support" +workflow = "OrderFulfillment" ``` -### Not Extracting Context in Consumers +## Manually tracking LLM calls when auto-instrumentation covers them -**Don't** ignore incoming context: +Inside `botanu.event(...)`, the OpenAI, Anthropic, Vertex, and LangChain auto-instrumentors already produce GenAI-semconv spans for each call. Wrapping those calls in `track_llm_call` creates a duplicate span and duplicate token accounting. ```python -# BAD - Context not extracted -def process_message(message): - # run_id from producer is lost - do_work(message["payload"]) -``` - -**Do** extract and use context: - -```python -# GOOD - Context continues -from botanu.sdk import set_baggage - -def process_message(message): - baggage = message.get("baggage", {}) - for key, value in baggage.items(): - set_baggage(key, value) - do_work(message["payload"]) -``` - -## Data Tracking Anti-Patterns - -### Not Tracking Data Operations - -**Don't** ignore database/storage costs: - -```python -# BAD - Only LLM tracked -@botanu_workflow("analyze_data", event_id=event_id, customer_id=customer_id) -async def analyze_data(): - data = await snowflake.query(expensive_query) # Not tracked! - with track_llm_call(...) as tracker: - result = await llm.complete(data) +with botanu.event(event_id=..., customer_id=..., workflow="Support"): + with track_llm_call(provider="openai", model="gpt-4") as tracker: + resp = openai_client.chat.completions.create(...) tracker.set_tokens(...) ``` -**Do** track all cost-generating operations: - ```python -# GOOD - Complete cost picture -@botanu_workflow("analyze_data", event_id=event_id, customer_id=customer_id) -async def analyze_data(): - with track_db_operation(system="snowflake", operation="SELECT") as db: - data = await snowflake.query(expensive_query) - db.set_bytes_scanned(data.bytes_scanned) - - with track_llm_call(...) as tracker: - result = await llm.complete(data) - tracker.set_tokens(...) +with botanu.event(event_id=..., customer_id=..., workflow="Support"): + resp = openai_client.chat.completions.create(...) ``` -### Missing Bytes for Pay-Per-Scan +`track_llm_call` is still useful when you're calling a library OTel doesn't auto-instrument — a custom inference endpoint, a self-hosted model server, or a proprietary SDK. -**Don't** forget bytes for warehouses: +## Calling `emit_outcome` to report success or failure -```python -# BAD - Missing cost driver -with track_db_operation(system="bigquery", operation="SELECT") as db: - result = await bq.query(sql) - db.set_result(rows_returned=len(result)) # Rows don't determine cost! -``` - -**Do** include bytes scanned: +`emit_outcome` stamps diagnostic fields — `value_type`, `value_amount`, `reason`, `error_type`. The authoritative outcome (success, failed, partial, etc.) is resolved server-side from SoR connectors, HITL reviews, or eval verdicts. Calling `emit_outcome()` with no fields stamps nothing useful. ```python -# GOOD - Bytes scanned is the cost driver -with track_db_operation(system="bigquery", operation="SELECT") as db: - result = await bq.query(sql) - db.set_bytes_scanned(result.bytes_processed) - db.set_result(rows_returned=len(result)) +with botanu.event(event_id=..., customer_id=..., workflow="Support"): + agent.run(ticket) + botanu.emit_outcome() ``` -## Error Handling Anti-Patterns - -### Swallowing Errors - -**Don't** hide errors: - ```python -# BAD - Error hidden -with track_llm_call(...) as tracker: - try: - response = await llm.complete(...) - except Exception: - pass # Silently fails - no error recorded +with botanu.event(event_id=..., customer_id=..., workflow="Support"): + resolved = agent.run(ticket) + if resolved: + botanu.emit_outcome(value_type="tickets_resolved", value_amount=1) ``` -**Do** record and propagate errors: - -```python -# GOOD - Error tracked and raised -with track_llm_call(...) as tracker: - try: - response = await llm.complete(...) - except Exception as e: - tracker.set_error(e) - emit_outcome("failed", reason=type(e).__name__) - raise -``` +## Calling `enable()` repeatedly -### Ignoring Partial Successes - -**Don't** mark all-or-nothing: +`enable()` runs lazily on the first `botanu.event(...)` call. Calling it explicitly is only useful when you need to override config — a custom endpoint, an API key passed in code, a non-default content-capture rate. Calling `enable()` inside a request handler or library adds no value and can misconfigure the SDK. ```python -# BAD - All items fail if one fails -@botanu_workflow("batch_work", event_id=batch_id, customer_id=customer_id) -async def process_batch(items): - for item in items: - await process(item) # If one fails, no outcome - emit_outcome("success", value_amount=len(items)) +def handle_request(req): + botanu.enable() + with botanu.event(...): + ... ``` -**Do** track partial success: - ```python -# GOOD - Partial success recorded -@botanu_workflow("batch_work", event_id=batch_id, customer_id=customer_id) -async def process_batch(items): - processed = 0 - failed = 0 - for item in items: - try: - await process(item) - processed += 1 - except Exception: - failed += 1 - - if failed == 0: - emit_outcome("success", value_type="items_processed", value_amount=processed) - elif processed > 0: - emit_outcome("partial", value_type="items_processed", value_amount=processed, - reason=f"failed_{failed}_of_{len(items)}") - else: - emit_outcome("failed", reason="all_items_failed") +def handle_request(req): + with botanu.event(...): + ... ``` -## Testing Anti-Patterns - -### Testing with Real Exporters +## Putting secrets or PII in baggage -**Don't** send telemetry during tests: +Baggage travels in plaintext on every outbound HTTP request and through third-party HTTP middleware. User tokens, API keys, and PII should flow through your application's normal auth layer, not through `set_baggage`. ```python -# BAD - Tests hit real collector -def test_workflow(): - enable(service_name="test") # Sends to real endpoint! - await do_work() -``` - -**Do** use NoOp or in-memory exporters: - -```python -# GOOD - Tests are isolated -from opentelemetry.trace import NoOpTracerProvider - -def setup_test(): - trace.set_tracer_provider(NoOpTracerProvider()) - -def test_workflow(): - await do_work() # No external calls +botanu.set_baggage("api_token", user_token) ``` -## See Also +## See also -- [Best Practices](best-practices.md) - What to do -- [Quickstart](../getting-started/quickstart.md) - Getting started guide -- [Outcomes](../tracking/outcomes.md) - Outcome recording details +- [Best Practices](best-practices.md) +- [Context Propagation](../concepts/context-propagation.md) diff --git a/docs/patterns/best-practices.md b/docs/patterns/best-practices.md index f60367b..58d4089 100644 --- a/docs/patterns/best-practices.md +++ b/docs/patterns/best-practices.md @@ -1,417 +1,108 @@ # Best Practices -Patterns for effective cost attribution with Botanu SDK. +Patterns that produce clean cost-per-outcome attribution and dashboards that stay readable over time. -## Run Design +## One event per business outcome -### One Run Per Business Outcome - -A run should represent a complete business transaction: +An event is one business transaction — a support ticket resolved, an order fulfilled, a document summarized. Use `botanu.step(...)` for internal phases so everything rolls up to a single outcome. ```python -# GOOD - One run for one business outcome -@botanu_workflow("process_order", event_id=order_id, customer_id=customer_id) -async def process_order(order_id: str, customer_id: str): - data = await fetch_data(order_id) - result = await do_work(data) - emit_outcome("success", value_type="orders_processed", value_amount=1) -``` +import botanu -```python -# BAD - Multiple runs for one outcome -@botanu_workflow("fetch_data", event_id=event_id, customer_id=customer_id) -async def fetch_data(event_id: str, customer_id: str): - ... - -@botanu_workflow("do_work", event_id=event_id, customer_id=customer_id) # Don't do this -async def do_work(event_id: str, customer_id: str): - ... +with botanu.event(event_id=order.id, customer_id=order.customer_id, workflow="OrderFulfillment"): + with botanu.step("validate"): + validate(order) + with botanu.step("charge"): + charge_card(order) + with botanu.step("ship"): + create_shipment(order) ``` -### Use Descriptive Workflow Names +## Use real business IDs for `event_id` -Workflow names appear in dashboards and queries. Choose names carefully: +`event_id` is the join key. When a SoR webhook fires (Zendesk ticket reopened, Stripe refund issued) or an evaluator verdict lands, the server matches it against `event_id` to resolve the outcome. Pass the identifier your downstream systems already use — ticket ID, order ID, session ID. ```python -# GOOD - Clear, descriptive names -@botanu_workflow("support_resolution", event_id=event_id, customer_id=customer_id) -@botanu_workflow("document_analysis", event_id=event_id, customer_id=customer_id) -@botanu_workflow("lead_scoring", event_id=event_id, customer_id=customer_id) - -# BAD - Generic or technical names -@botanu_workflow("handle", event_id=event_id, customer_id=customer_id) -@botanu_workflow("process", event_id=event_id, customer_id=customer_id) -@botanu_workflow("main", event_id=event_id, customer_id=customer_id) +with botanu.event(event_id=ticket.id, customer_id=ticket.user_id, workflow="Support"): + agent.run(ticket) ``` -## Outcome Recording - -### Outcome is derived, not reported - -Event outcome is computed server-side from eval verdict rollup / HITL / SoR -connector — not from `emit_outcome(status=...)`. The `status` argument is -now a diagnostic helper only (it raises a `DeprecationWarning` on every -call). You do **not** need to call `emit_outcome` to record success or -failure; `@botanu_workflow` already creates the run and the platform will -resolve its outcome. - -`emit_outcome` is still useful when you want to annotate the run with -*diagnostic* context the dashboard can show alongside outcome: +If your internal ID differs from the SoR ID, add the correlation explicitly: ```python -@botanu_workflow("process_data", event_id=data_id, customer_id=customer_id) -async def process_data(data_id: str, customer_id: str): - try: - result = await process(data_id) - # Stamp value_type / value_amount for cost-per-value math. - emit_outcome("success", value_type="records_processed", value_amount=result.count) - return result - except ValidationError as exc: - # Stamp the reason/error_type so the dashboard can group failures. - emit_outcome("failed", reason="validation_error", error_type=type(exc).__name__) - raise - except TimeoutError: - emit_outcome("failed", reason="timeout", error_type="TimeoutError") - raise +with botanu.event(event_id=session.id, customer_id=user.id, workflow="Support"): + botanu.set_correlation(zendesk_ticket_id=session.zendesk_ticket_id) + agent.run(session) ``` -See [Outcomes](../tracking/outcomes.md) for the full list of diagnostic -fields that still stamp. +## Pick stable, low-cardinality workflow names -### Quantify Value When Possible - -Include value amounts for better ROI analysis: +Workflow names drive filtering and grouping in the dashboard. Keep them stable across deployments and descriptive of the business purpose. ```python -# GOOD - Quantified outcomes -emit_outcome("success", value_type="items_sent", value_amount=50) -emit_outcome("success", value_type="revenue_generated", value_amount=1299.99) -emit_outcome("success", value_type="documents_processed", value_amount=10) +@botanu.event(workflow="Support", event_id=lambda t: t.id, customer_id=lambda t: t.user_id) +def handle_ticket(ticket): ... -# LESS USEFUL - No quantity -emit_outcome("success") +@botanu.event(workflow="DocumentAnalysis", event_id=lambda d: d.id, customer_id=lambda d: d.tenant) +def analyze_doc(doc): ... ``` -### Use Consistent Value Types +## Let OTel auto-instrumentation do the heavy lifting -Standardize your value types across the organization: +Inside `botanu.event(...)`, the OTel auto-instrumentors already cover OpenAI, Anthropic, Vertex, LangChain, httpx, requests, SQLAlchemy, psycopg2, asyncpg, Redis, Celery, Kafka, and boto3. Each call automatically produces a span with the right semconv attributes and inherits the event's run context. -```python -# Define standard value types -class ValueTypes: - ITEMS_PROCESSED = "items_processed" - DOCUMENTS_ANALYZED = "documents_analyzed" - LEADS_SCORED = "leads_scored" - MESSAGES_SENT = "messages_sent" - REVENUE_GENERATED = "revenue_generated" - -# Use consistently -emit_outcome("success", value_type=ValueTypes.ITEMS_PROCESSED, value_amount=1) -``` +You only need `track_llm_call` or `track_db_operation` when: -### Include Reasons for Failures +- The library you're calling isn't auto-instrumented (custom inference server, niche vector DB, proprietary queue). +- You need to stamp semantic metrics the instrumentor doesn't capture, like `set_bytes_scanned` on a warehouse query. -Always explain why something failed: +## Annotate business value -```python -emit_outcome("failed", reason="rate_limit_exceeded") -emit_outcome("failed", reason="invalid_input") -emit_outcome("failed", reason="model_unavailable") -emit_outcome("failed", reason="context_too_long") -``` - -## LLM Tracking - -### Always Record Token Usage - -Tokens are the primary cost driver for LLMs: +Use `emit_outcome` to stamp diagnostic fields that the dashboard can render alongside cost-per-outcome. The authoritative outcome is resolved server-side; these fields add colour. ```python -with track_llm_call(provider="openai", model="gpt-4") as tracker: - response = await client.chat.completions.create(...) - # Always set tokens - tracker.set_tokens( - input_tokens=response.usage.prompt_tokens, - output_tokens=response.usage.completion_tokens, - ) +with botanu.event(event_id=ticket.id, customer_id=ticket.user_id, workflow="Support"): + resolved = agent.run(ticket) + if resolved: + botanu.emit_outcome(value_type="tickets_resolved", value_amount=1) ``` -### Record Provider Request IDs - -Request IDs enable reconciliation with provider invoices: +Quantified examples: ```python -tracker.set_request_id( - provider_request_id=response.id, # From provider - client_request_id=uuid.uuid4().hex, # Your internal ID -) +botanu.emit_outcome(value_type="revenue_generated", value_amount=1299.99) +botanu.emit_outcome(value_type="documents_processed", value_amount=10) +botanu.emit_outcome(reason="rate_limit_exceeded", error_type="RateLimitError") ``` -### Track Retries - -Record attempt numbers for accurate cost per success: +## Multi-tenant apps: always pass `tenant_id` ```python -for attempt in range(max_retries): - with track_llm_call(provider="openai", model="gpt-4") as tracker: - tracker.set_attempt(attempt + 1) - try: - response = await client.chat.completions.create(...) - break - except RateLimitError: - if attempt == max_retries - 1: - raise - await asyncio.sleep(backoff) -``` - -### Use Correct Operation Types - -Specify the operation type for accurate categorization: - -```python -from botanu.tracking.llm import track_llm_call, ModelOperation - -# Chat completion -with track_llm_call(provider="openai", model="gpt-4", operation=ModelOperation.CHAT): - ... - -# Embeddings -with track_llm_call(provider="openai", model="text-embedding-3-small", operation=ModelOperation.EMBEDDINGS): +with botanu.event( + event_id=request.id, + customer_id=request.user_id, + workflow="Support", + tenant_id=request.org_id, +): ... ``` -## Data Tracking - -### Track All Cost-Generating Operations - -Include databases, storage, and messaging: - -```python -@botanu_workflow("run_pipeline", event_id=pipeline_id, customer_id=customer_id) -async def run_pipeline(pipeline_id: str, customer_id: str): - # Track warehouse query (billed by bytes scanned) - with track_db_operation(system="snowflake", operation="SELECT") as db: - db.set_bytes_scanned(result.bytes_scanned) - db.set_query_id(result.query_id) - - # Track storage operations (billed by requests + data) - with track_storage_operation(system="s3", operation="PUT") as storage: - storage.set_result(bytes_written=len(data)) - - # Track messaging (billed by message count) - with track_messaging_operation(system="sqs", operation="publish", destination="queue") as msg: - msg.set_result(message_count=batch_size) -``` - -### Include Bytes for Pay-Per-Scan Services - -For data warehouses billed by data scanned: - -```python -with track_db_operation(system="bigquery", operation="SELECT") as db: - result = await bq_client.query(sql) - db.set_bytes_scanned(result.total_bytes_processed) - db.set_result(rows_returned=result.num_rows) -``` - -## Context Propagation - -### Use Middleware for Web Services - -Extract context from incoming requests: - -```python -from fastapi import FastAPI -from botanu.sdk.middleware import BotanuMiddleware - -app = FastAPI() -app.add_middleware(BotanuMiddleware) -``` - -### Propagate Context in Message Queues - -Inject and extract context manually for async messaging: - -```python -from botanu.sdk import set_baggage, get_baggage - -# Producer -def publish_message(payload): - message = { - "payload": payload, - "baggage": { - "botanu.workflow": get_baggage("botanu.workflow"), - "botanu.event_id": get_baggage("botanu.event_id"), - "botanu.customer_id": get_baggage("botanu.customer_id"), - } - } - queue.publish(message) - -# Consumer -def process_message(message): - baggage = message.get("baggage", {}) - for key, value in baggage.items(): - set_baggage(key, value) - do_work(message["payload"]) -``` - -### Use Lean Mode for High-Traffic Systems - -Default lean mode minimizes header overhead: - -```python -# Lean mode: ~100 bytes of baggage -# Propagates: run_id, botanu.workflow - -# Full mode: ~300 bytes of baggage -# Propagates: run_id, botanu.workflow, botanu.event_id, botanu.customer_id, -# environment, tenant_id, parent_run_id -``` - -## Configuration +`tenant_id` propagates via baggage and appears as a filter dimension in the dashboard. -### Use Environment Variables in Production +## Configuration belongs in the environment -Keep configuration out of code: +Keep secrets and endpoints out of code: ```bash -export OTEL_SERVICE_NAME=my-service -export OTEL_EXPORTER_OTLP_ENDPOINT=http://collector:4318 +export BOTANU_API_KEY= +export OTEL_SERVICE_NAME=support-service export BOTANU_ENVIRONMENT=production ``` -### Use YAML for Complex Configuration - -For multi-environment setups: - -```yaml -# config/production.yaml -service: - name: ${OTEL_SERVICE_NAME} - environment: production - -otlp: - endpoint: ${COLLECTOR_ENDPOINT} - -propagation: - mode: lean -``` - -## Multi-Tenant Systems - -### Always Include Tenant ID - -For accurate per-tenant cost attribution: - -```python -@botanu_workflow("handle_request", event_id=request_id, customer_id=cust_id, tenant_id=request.tenant_id) -async def handle_request(request): - ... -``` - -### Use Business Context - -Add additional attribution dimensions via baggage: - -```python -set_baggage("team", "engineering") -set_baggage("cost_center", "R&D") -set_baggage("region", "us-west-2") -``` - -## Error Handling - -### Record Errors Explicitly - -Don't lose error context: - -```python -with track_llm_call(provider="openai", model="gpt-4") as tracker: - try: - response = await client.chat.completions.create(...) - except openai.APIError as e: - tracker.set_error(e) # Records error type and message - raise -``` - -### Emit Outcomes for Errors - -Even failed runs should have outcomes: - -```python -@botanu_workflow("process_data", event_id=data_id, customer_id=customer_id) -async def process_data(data_id: str, customer_id: str): - try: - await do_work(data_id) - emit_outcome("success", value_type="items_processed", value_amount=1) - except ValidationError: - emit_outcome("failed", reason="validation_error") - raise - except Exception as e: - emit_outcome("failed", reason=type(e).__name__) - raise -``` - -## Performance - -### Use Async Tracking - -For async applications, ensure tracking is non-blocking: - -```python -# The SDK uses span events, not separate API calls -# This is already non-blocking -with track_llm_call(provider="openai", model="gpt-4") as tracker: - response = await do_something() - tracker.set_tokens(...) # Immediate, non-blocking -``` - -### Batch Database Tracking - -For batch operations, track at batch level: - -```python -# GOOD - Batch tracking -with track_db_operation(system="postgresql", operation="INSERT") as db: - await cursor.executemany(insert_sql, batch_of_1000_rows) - db.set_result(rows_affected=1000) - -# LESS EFFICIENT - Per-row tracking -for row in batch_of_1000_rows: - with track_db_operation(system="postgresql", operation="INSERT") as db: - await cursor.execute(insert_sql, row) - db.set_result(rows_affected=1) -``` - -## Testing - -### Mock Tracing in Tests - -Use the NoOp tracer for unit tests: - -```python -from opentelemetry import trace -from opentelemetry.trace import NoOpTracerProvider - -def setup_test_tracing(): - trace.set_tracer_provider(NoOpTracerProvider()) -``` - -### Test Outcome Recording - -Verify outcomes are emitted correctly: - -```python -from unittest.mock import patch - -def test_successful_outcome(): - with patch("botanu.sdk.span_helpers.emit_outcome") as mock_emit: - result = await do_work("123") - mock_emit.assert_called_with("success", value_type="items_processed", value_amount=1) -``` +For multi-environment setups, a YAML file works too — see [Configuration](../getting-started/configuration.md). -## See Also +## See also -- [Anti-Patterns](anti-patterns.md) - What to avoid -- [Architecture](../concepts/architecture.md) - SDK design principles -- [Configuration](../getting-started/configuration.md) - Configuration options +- [Anti-Patterns](anti-patterns.md) +- [Outcomes](../tracking/outcomes.md) +- [Context Propagation](../concepts/context-propagation.md) diff --git a/docs/tracking/content-capture.md b/docs/tracking/content-capture.md index 3762e69..a354063 100644 --- a/docs/tracking/content-capture.md +++ b/docs/tracking/content-capture.md @@ -44,15 +44,15 @@ each capture point — the SDK does not coordinate across processes. ### 1. Workflow-level (automatic, once per run) -`@botanu_workflow` will capture the decorated function's bound arguments as +`botanu.event` will capture the decorated function's bound arguments as input and its return value as output, **once per run**, when `content_capture_rate` fires. ```python -from botanu import botanu_workflow +import botanu -@botanu_workflow( - "summarize", +@botanu.event( + workflow="summarize", event_id=lambda req: req.id, customer_id=lambda req: req.tenant, ) @@ -103,8 +103,8 @@ expose optional content setters that respect the same rate. See | Attribute | Written by | Source | | --- | --- | --- | -| `botanu.eval.input_content` | `@botanu_workflow` | Bound function arguments (JSON) | -| `botanu.eval.output_content` | `@botanu_workflow` | Return value (JSON) | +| `botanu.eval.input_content` | `botanu.event` | Bound function arguments (JSON) | +| `botanu.eval.output_content` | `botanu.event` | Return value (JSON) | | `botanu.eval.input_content` | `LLMTracker.set_input_content()` | Explicit prompt text | | `botanu.eval.output_content` | `LLMTracker.set_output_content()` | Explicit response text | @@ -112,16 +112,77 @@ All values are truncated to 4096 characters before being stamped. ## PII handling -The SDK **does not scrub PII**. Scrubbing happens downstream: +The SDK scrubs PII **in-process** before a span attribute is written. This is +on by default — you do not need to configure anything to get it. Downstream +collector + evaluator passes remain as belt-and-suspenders. + +Pipeline for every captured string: + +```text +customer text + ↓ +content_capture_rate gate (skip capture entirely) + ↓ +regex scrub (default patterns) # src/botanu/sdk/pii.py + ↓ +optional Presidio NER # pip install botanu[pii-nlp] + ↓ +truncate to max_chars (4096) + ↓ +span.set_attribute("botanu.eval.*_content", ...) +``` + +### Built-in regex patterns + +Email, phone (E.164 + US), SSN, credit card (Luhn-validated), IPv4/IPv6, +JWT, bearer tokens, and common API-key prefixes (AWS `AKIA…`, GitHub +`ghp_…`, Stripe `sk_live_…`, Slack `xoxb-…`, OpenAI `sk-…`, +Anthropic `sk-ant-…`). + +Matches are replaced with `[REDACTED]` by default. + +### Configuration + +```yaml +eval: + content_capture_rate: 0.2 + pii: + enabled: true # default — opt-out is explicit + disable_patterns: [ipv4] # turn off specific built-ins + custom_patterns: + employee_id: 'EMP-\d{6}' + use_presidio: false # set true to add NER on top + replacement: "[REDACTED]" +``` + +Or via env: + +| Var | Default | Notes | +| --- | --- | --- | +| `BOTANU_PII_SCRUB_ENABLED` | `true` | Set to `false` to opt out | +| `BOTANU_PII_SCRUB_DISABLE_PATTERNS` | unset | Comma-separated names | +| `BOTANU_PII_SCRUB_USE_PRESIDIO` | `false` | Requires the `pii-nlp` extra | +| `BOTANU_PII_SCRUB_REPLACEMENT` | `[REDACTED]` | Any string | + +### Presidio NER (optional) + +For name/address/medical-term detection, install the optional extra: + +```bash +pip install botanu[pii-nlp] +``` + +…and set `pii_scrub_use_presidio=true`. Without the package installed, the +flag is a no-op and the regex pass continues to run (you get a warning log +on first use). Entities covered: `EMAIL_ADDRESS`, `PHONE_NUMBER`, +`CREDIT_CARD`, `US_SSN`, `PERSON`, `LOCATION`, `IP_ADDRESS`, +`US_BANK_NUMBER`, `MEDICAL_LICENSE`. -1. **Collector** — runs a regex redaction pass on `botanu.eval.*` attributes - (credit-card, email, phone, API-key patterns) before forwarding. -2. **Evaluator** — runs a Microsoft Presidio NER pass before storing captured - text against the eval record. +### If you need stricter privacy -If you have strict PII requirements, keep `content_capture_rate=0.0` and -drive eval off explicit tool/score annotations instead. The capture pipeline -is opt-in precisely so you can stay private by default. +Keep `content_capture_rate=0.0` and drive eval off explicit tool/score +annotations instead. The capture pipeline is opt-in precisely so you can +stay private by default. ## Verifying capture is on @@ -133,7 +194,7 @@ attributes. If they are absent, check in order: 1. `BotanuConfig.content_capture_rate` is actually > 0.0 in the running process (`BotanuConfig.from_yaml(...)` and env precedence can surprise you — print `get_config().content_capture_rate` to be sure). -2. You are inside a span (`@botanu_workflow` or `track_llm_call` scope). +2. You are inside a span (`botanu.event` or `track_llm_call` scope). 3. The random gate didn't miss — at `rate=0.1`, ~90% of calls will look empty. Set the rate to `1.0` temporarily to confirm plumbing. diff --git a/docs/tracking/data-tracking.md b/docs/tracking/data-tracking.md index bf7e06c..5429528 100644 --- a/docs/tracking/data-tracking.md +++ b/docs/tracking/data-tracking.md @@ -1,25 +1,17 @@ # Data Tracking -Track database, storage, and messaging operations for complete cost visibility. +> **Most customers don't need this.** Inside `botanu.event(...)`, OTel auto-instrumentors for SQLAlchemy, psycopg2, asyncpg, pymongo, redis, boto3, celery, and kafka already produce spans with `db.*` / `messaging.*` attributes and run-context stamping. Reach for `track_db_operation` / `track_storage_operation` / `track_messaging_operation` only when the library you're calling isn't auto-instrumented (custom query layer, proprietary queue, niche data store) or when you need to set result metrics (rows returned, bytes scanned) that the instrumentor doesn't capture. -## Overview - -Data operations often contribute significantly to AI workflow costs. Botanu provides tracking for: - -- **Databases** - SQL, NoSQL, data warehouses -- **Object Storage** - S3, GCS, Azure Blob -- **Messaging** - SQS, Kafka, Pub/Sub - -## Database Tracking - -### Basic Usage +## `track_db_operation` ```python +import asyncpg from botanu.tracking.data import track_db_operation -with track_db_operation(system="postgresql", operation="SELECT") as db: - result = await cursor.execute("SELECT * FROM users WHERE active = true") - db.set_result(rows_returned=len(result)) +async with asyncpg.connect(dsn) as conn: + with track_db_operation(system="postgresql", operation="SELECT") as db: + rows = await conn.fetch("SELECT id FROM users WHERE active = true") + db.set_result(rows_returned=len(rows)) ``` ### DBTracker Methods @@ -300,67 +292,83 @@ set_warehouse_metrics( ) ``` -## Example: Complete Data Pipeline +## Example: complete data pipeline + +Full working sketch: a batch ETL that scans Snowflake, runs an LLM per row, writes to S3, inserts into Postgres, and publishes to SQS. ```python -from botanu import botanu_workflow, emit_outcome +import json + +import asyncpg +import boto3 +import snowflake.connector +from openai import AsyncOpenAI + +import botanu from botanu.tracking.data import ( + DBOperation, track_db_operation, - track_storage_operation, track_messaging_operation, - DBOperation, + track_storage_operation, ) from botanu.tracking.llm import track_llm_call -@botanu_workflow("etl-pipeline", event_id=batch_id, customer_id=customer_id) -async def process_batch(batch_id: str): - """Complete ETL pipeline with cost tracking.""" +snow = snowflake.connector.connect(...) +s3 = boto3.client("s3") +sqs = boto3.client("sqs") +openai = AsyncOpenAI() +SQS_QUEUE_URL = "https://sqs.us-east-1.amazonaws.com/123/batch-complete" - # 1. Read from data warehouse + +@botanu.event( + workflow="etl-pipeline", + event_id=lambda batch_id, customer_id: batch_id, + customer_id=lambda batch_id, customer_id: customer_id, +) +async def process_batch(batch_id: str, customer_id: str): with track_db_operation(system="snowflake", operation=DBOperation.SELECT) as db: db.set_query_id(batch_id) - rows = await snowflake_client.execute( - "SELECT * FROM raw_data WHERE batch_id = %s", - batch_id - ) + cur = snow.cursor() + cur.execute("SELECT id, payload FROM raw_data WHERE batch_id = %s", (batch_id,)) + rows = cur.fetchall() db.set_result(rows_returned=len(rows)) - db.set_bytes_scanned(rows.bytes_scanned) - # 2. Process with LLM processed = [] - for row in rows: + for row_id, payload in rows: with track_llm_call(provider="openai", model="gpt-4") as llm: - result = await analyze_row(row) - llm.set_tokens(input_tokens=result.input_tokens, output_tokens=result.output_tokens) - processed.append(result) - - # 3. Write to storage + resp = await openai.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": payload}], + ) + llm.set_tokens( + input_tokens=resp.usage.prompt_tokens, + output_tokens=resp.usage.completion_tokens, + ) + processed.append({"id": row_id, "result": resp.choices[0].message.content}) + + body = json.dumps(processed) with track_storage_operation(system="s3", operation="PUT") as storage: storage.set_bucket("processed-data") - await s3_client.put_object( - Bucket="processed-data", - Key=f"batch/{batch_id}.json", - Body=json.dumps(processed) - ) - storage.set_result(bytes_written=len(json.dumps(processed))) - - # 4. Write to database - with track_db_operation(system="postgresql", operation=DBOperation.INSERT) as db: - await pg_client.executemany( - "INSERT INTO processed_data VALUES (%s, %s, %s)", - [(r.id, r.result, r.score) for r in processed] - ) - db.set_result(rows_affected=len(processed)) + s3.put_object(Bucket="processed-data", Key=f"batch/{batch_id}.json", Body=body) + storage.set_result(bytes_written=len(body)) + + async with asyncpg.create_pool(dsn="postgresql://localhost/db") as pool: + async with pool.acquire() as conn: + with track_db_operation(system="postgresql", operation=DBOperation.INSERT) as db: + await conn.executemany( + "INSERT INTO processed_data(id, result) VALUES ($1, $2)", + [(r["id"], r["result"]) for r in processed], + ) + db.set_result(rows_affected=len(processed)) - # 5. Publish completion event with track_messaging_operation(system="sqs", operation="publish", destination="batch-complete") as msg: - await sqs_client.send_message( - QueueUrl=queue_url, - MessageBody=json.dumps({"batch_id": batch_id, "count": len(processed)}) + sqs.send_message( + QueueUrl=SQS_QUEUE_URL, + MessageBody=json.dumps({"batch_id": batch_id, "count": len(processed)}), ) msg.set_result(message_count=1) - emit_outcome("success", value_type="batches_processed", value_amount=1) + botanu.emit_outcome(value_type="batches_processed", value_amount=1) return processed ``` diff --git a/docs/tracking/llm-tracking.md b/docs/tracking/llm-tracking.md index f414fcf..be6d36f 100644 --- a/docs/tracking/llm-tracking.md +++ b/docs/tracking/llm-tracking.md @@ -1,14 +1,8 @@ # LLM Tracking -Track AI model usage for accurate cost attribution across providers. +> **Most customers don't need this.** Inside `botanu.event(...)`, the OTel auto-instrumentors for OpenAI, Anthropic, Vertex AI, and LangChain already produce [GenAI semantic-convention](https://opentelemetry.io/docs/specs/semconv/gen-ai/) spans with `gen_ai.*` attributes and run-context stamping. Reach for `track_llm_call` only when the library you're calling isn't auto-instrumented (custom inference endpoint, self-hosted model server, proprietary SDK) or when you need to set content for eval manually. -## Overview - -Botanu provides LLM tracking that aligns with [OpenTelemetry GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/). This ensures compatibility with standard observability tooling while enabling detailed cost analysis. - -## Basic Usage - -### Context Manager (Recommended) +## `track_llm_call` ```python from botanu.tracking.llm import track_llm_call @@ -96,10 +90,13 @@ When capture fires, the SDK writes: | `botanu.eval.input_content` | `set_input_content(text)` | | `botanu.eval.output_content` | `set_output_content(text)` | -**PII is not scrubbed in the SDK.** The collector runs a regex redaction pass -and the evaluator runs Presidio NER before any captured text is stored. -See [Content Capture](content-capture.md) for the full pipeline and for the -workflow-level auto-capture path that `@botanu_workflow` provides. +**PII is scrubbed in-process by default** before the attribute is written — +regex patterns for email, phone, SSN, credit card, IPs, JWTs, and common API +keys. Optional Presidio NER adds name/address/medical-term detection (install +with `pip install botanu[pii-nlp]`). Collector regex + evaluator Presidio +remain downstream as belt-and-suspenders. See +[Content Capture](content-capture.md) for the full pipeline, opt-out knobs, +and the event-level auto-capture path that `botanu.event(...)` provides. ### set_request_params() @@ -314,38 +311,53 @@ The SDK automatically records these metrics: | `gen_ai.client.operation.duration` | Histogram | Operation duration in seconds | | `botanu.gen_ai.attempts` | Counter | Request attempts (including retries) | -## Example: Multi-Provider Workflow +## Example: multi-provider fallback ```python -from botanu import botanu_workflow, emit_outcome +from anthropic import AsyncAnthropic, RateLimitError +from openai import AsyncOpenAI + +import botanu from botanu.tracking.llm import track_llm_call -@botanu_workflow("process-with-fallback", event_id=event_id, customer_id=customer_id) -async def process_with_fallback(data: str): - """Try one provider first, fall back to another.""" +anthropic = AsyncAnthropic() +openai = AsyncOpenAI() + +@botanu.event( + workflow="process-with-fallback", + event_id=lambda data: data["id"], + customer_id=lambda data: data["customer_id"], +) +async def process_with_fallback(data): try: with track_llm_call(provider="anthropic", model="claude-3-opus") as tracker: tracker.set_attempt(1) - response = await do_work(data, provider="anthropic") + response = await anthropic.messages.create( + model="claude-3-opus-20240229", + max_tokens=1024, + messages=[{"role": "user", "content": data["prompt"]}], + ) tracker.set_tokens( input_tokens=response.usage.input_tokens, output_tokens=response.usage.output_tokens, ) - emit_outcome("success", value_type="items_processed", value_amount=1) - return response.content + botanu.emit_outcome(value_type="items_processed", value_amount=1) + return response.content[0].text except RateLimitError: - # Fallback to second provider with track_llm_call(provider="openai", model="gpt-4") as tracker: tracker.set_attempt(2) - response = await do_work(data, provider="openai") + response = await openai.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": data["prompt"]}], + ) tracker.set_tokens( input_tokens=response.usage.prompt_tokens, output_tokens=response.usage.completion_tokens, ) - emit_outcome("success", value_type="items_processed", value_amount=1) - return response.content + botanu.emit_outcome(value_type="items_processed", value_amount=1) + return response.choices[0].message.content ``` ## See Also diff --git a/docs/tracking/outcomes.md b/docs/tracking/outcomes.md index 1a24a0b..6c4aa0b 100644 --- a/docs/tracking/outcomes.md +++ b/docs/tracking/outcomes.md @@ -1,180 +1,86 @@ # Outcomes -> **⚠️ DEPRECATED (2026-04-16):** The `status` argument on `emit_outcome()` no -> longer stamps `botanu.outcome.status` on the span. Customer-reported outcome -> was removed because it was trivially fakeable — a misconfigured or -> adversarial SDK could claim every event succeeded and skew cost-per-outcome. -> -> **What determines event outcome now:** botanu derives the outcome server-side -> from, in priority order: -> -> 1. A System-of-Record (SoR) connector (Zendesk, Stripe, your own webhook). -> 2. A human reviewer verdict from the HITL queue. -> 3. The evaluator's LLM-as-judge verdict rollup for the event's runs. -> 4. `pending` if nothing above fires yet. -> -> **What still works:** the other `emit_outcome(...)` fields (`reason`, -> `error_type`, `value_type`, `value_amount`, `confidence`, `metadata`) still -> stamp as *diagnostic* span attributes. They are useful for debugging and -> drill-down in the dashboard; they are not the authoritative outcome. -> -> Every call to `emit_outcome(status=...)` emits a `DeprecationWarning`. - -## Overview - -An **event** is one business transaction (a support ticket, an order, a report -generation). An event has one outcome that botanu determines from the signals -above. What you can do from the SDK is enrich the event with diagnostic -context — a reason string, an error classification, a value figure for -cost-per-value math, or arbitrary metadata. - -**Hierarchy refresher:** - -- **Event** — one business unit of work (has an `event_id`). Outcome lives here. -- **Run** — one execution attempt for an event. Retries replace the previous - attempt; see [Run Context](../concepts/run-context.md). -- **Span** — one LLM/DB/tool call within a run. - -## The diagnostic helpers - -`emit_outcome` is now a thin helper that writes a fixed set of diagnostic -attributes onto the current span. Everything except `status` still stamps. +Event outcome is resolved **server-side** from, in priority order: -```python -from botanu import botanu_workflow, emit_outcome - -@botanu_workflow("fulfill-order", event_id=order.id, customer_id=customer.id) -async def process_order(order): - result = await do_work(order) - - # Diagnostic fields — useful for dashboard drill-down and cost-per-value - # math, but not the authoritative outcome. - emit_outcome( - "success", # accepted for back-compat, DeprecationWarning - value_type="orders_fulfilled", - value_amount=1, - metadata={"sku_count": len(order.items)}, - ) - return result -``` +1. A System-of-Record (SoR) connector (Zendesk, Stripe, custom webhook) +2. A human-in-the-loop reviewer verdict +3. The evaluator's LLM-as-judge verdict rollup for the event's runs +4. `pending` if none of the above has fired yet -If you do not need any of the diagnostic fields, you can drop `emit_outcome` -entirely. `@botanu_workflow` already creates the run and span — outcome will -be filled in server-side from the signals above. +The SDK does not set the authoritative outcome. It stamps diagnostic fields that appear alongside cost-per-outcome in the dashboard. -## emit_outcome() reference +## `emit_outcome()` ```python -emit_outcome( - status: str, # Required for validation; does NOT stamp the outcome. - *, - value_type: str | None = None, # Free-form business-value label. - value_amount: float | None = None, - confidence: float | None = None, # 0.0–1.0 - reason: str | None = None, # Free-form; especially for failures. - error_type: str | None = None, # Exception/classification name. - metadata: dict | None = None, # Arbitrary diagnostic kv. +import botanu + +botanu.emit_outcome( + value_type="tickets_resolved", + value_amount=1, + confidence=0.92, ) ``` -### status (required but diagnostic-only) - -`status` is still validated against the set below. It is accepted for -backward compatibility and its value is not written to the span. +All parameters are optional and diagnostic-only: -| Value | Intended meaning | +| Parameter | Description | | --- | --- | -| `success` | Event produced the intended result | -| `partial` | Event produced some of the intended result | -| `failed` | Event did not produce a result | -| `timeout` | Event did not finish in its deadline | -| `canceled` | Event was canceled by user or system | -| `abandoned` | Event was abandoned without completion | - -A value outside this set raises `ValueError`. `"failure"` is not valid — use -`"failed"`. - -### Other fields - -Everything below stamps as a `botanu.outcome.*` diagnostic attribute: - -```python -emit_outcome("success", value_type="tickets_resolved", value_amount=1) -emit_outcome("success", value_type="revenue_generated", value_amount=1299.99) -emit_outcome("success", value_type="classifications_completed", - value_amount=1, confidence=0.92) -emit_outcome("failed", reason="upstream_unavailable", error_type="ServiceUnavailable") -emit_outcome("timeout", reason="model_took_too_long", error_type="DeadlineExceeded") -emit_outcome("partial", reason="processed_3_of_5", value_amount=3) -emit_outcome("success", value_type="items_processed", value_amount=10, - metadata={"batch_id": "abc-123", "retry_count": 2}) -``` +| `value_type` | Free-form business-value label (e.g. `"tickets_resolved"`, `"revenue_generated"`) | +| `value_amount` | Quantified value amount | +| `confidence` | Confidence score, `0.0`–`1.0` | +| `reason` | Free-form diagnostic string | +| `error_type` | Error classification (e.g. `"TimeoutError"`) | +| `metadata` | Arbitrary key-value dict | -## Span attributes that are still emitted +### Span attributes stamped -| Attribute | Description | +| Attribute | Source | | --- | --- | -| `botanu.outcome.value_type` | What was achieved (free-form label) | -| `botanu.outcome.value_amount` | Quantified value | -| `botanu.outcome.confidence` | Confidence score (0.0–1.0) | -| `botanu.outcome.reason` | Reason string (especially for failures) | -| `botanu.outcome.error_type` | Error classification | -| `botanu.outcome.metadata.*` | Flattened metadata dict | - -> `botanu.outcome.status` is **not** emitted. Dashboards that read from -> `runs.outcome_status` are reading a legacy physical column kept only for -> backward compatibility; the authoritative field is `events.final_outcome`, -> which is written by the platform, not the SDK. - -## Automatic outcome (convenience) +| `botanu.outcome.value_type` | `value_type` | +| `botanu.outcome.value_amount` | `value_amount` | +| `botanu.outcome.confidence` | `confidence` | +| `botanu.outcome.reason` | `reason` | +| `botanu.outcome.error_type` | `error_type` | +| `botanu.outcome.metadata.` | each `metadata` key | -`@botanu_workflow(..., auto_outcome_on_success=True)` (default) automatically -calls `emit_outcome("success")` at the end of a successful call, and -`emit_outcome("failed", reason=type(exc).__name__)` on exception. Since -`status` no longer stamps the outcome, this is pure convenience — it still -writes `reason` and `error_type` for failures, which is useful diagnostic -context. +There is no `botanu.outcome.status` attribute. Authoritative outcome lives in `events.final_outcome` server-side. -Disable if you prefer explicit calls: +## Examples ```python -@botanu_workflow("my-workflow", event_id=event_id, customer_id=customer_id, - auto_outcome_on_success=False) -async def my_function(): - result = await do_work() - emit_outcome("success", value_type="items", value_amount=1) - return result +botanu.emit_outcome(value_type="tickets_resolved", value_amount=1) +botanu.emit_outcome(value_type="revenue_generated", value_amount=1299.99) +botanu.emit_outcome(reason="upstream_unavailable", error_type="ServiceUnavailable") +botanu.emit_outcome( + value_type="items_processed", + value_amount=10, + metadata={"batch_id": "abc-123"}, +) ``` -## Context manager form - -When you can't use the decorator: +## Usage inside an event ```python -from botanu import run_botanu, emit_outcome - -async def my_function(event_id: str, customer_id: str): - async with run_botanu("my-workflow", event_id=event_id, customer_id=customer_id): - result = await do_work() - emit_outcome("success", value_type="items_processed", - value_amount=result.count) - return result +import botanu + +with botanu.event(event_id=order.id, customer_id=order.customer_id, workflow="Fulfillment"): + result = process_order(order) + botanu.emit_outcome(value_type="orders_fulfilled", value_amount=1) ``` -## Cost-per-outcome math +If you don't need diagnostic fields, skip `emit_outcome` entirely. The platform resolves the outcome from the signals above. + +## Cost-per-outcome -Cost-per-outcome is computed by the platform from: +Cost-per-outcome is computed server-side from: -- the `runs.cost_total_usd` column populated by the cost engine, and -- the `events.final_outcome` column populated by the outcome resolver. +- `runs.cost_total_usd` — populated by the cost engine +- `events.final_outcome` — populated by the outcome resolver -You don't query these yourself — open the dashboard. What you *can* do from -the SDK is annotate with `value_type` / `value_amount` so a business-value -column appears alongside cost-per-outcome in the dashboard. +Annotate with `value_type` and `value_amount` so a business-value column appears alongside cost-per-outcome in the dashboard. ## See also -- [Run Context](../concepts/run-context.md) — the event/run/span hierarchy -- [LLM Tracking](llm-tracking.md) — per-call attribution -- [Content Capture](content-capture.md) — capturing prompts/responses for eval -- [Best Practices](../patterns/best-practices.md) +- [Run Context](../concepts/run-context.md) +- [LLM Tracking](llm-tracking.md) +- [Content Capture](content-capture.md) diff --git a/pyproject.toml b/pyproject.toml index 0f2eec6..ef8979b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,6 +139,14 @@ cloud = [ "opentelemetry-resource-detector-azure >= 0.1b0", "opentelemetry-resource-detector-container >= 0.1b0", ] + +# NER-based PII scrubbing on top of the built-in regex pass. Regex covers +# structured tokens (emails, card numbers, API keys); Presidio adds names, +# addresses, and medical terms. Heavy (~200MB with spaCy model) — opt-in only. +pii-nlp = [ + "presidio-analyzer >= 2.2", + "presidio-anonymizer >= 2.2", +] dev = [ "pytest >= 7.4.0", "pytest-asyncio >= 0.21.0", diff --git a/src/botanu/__init__.py b/src/botanu/__init__.py index 12bccb6..cd28599 100644 --- a/src/botanu/__init__.py +++ b/src/botanu/__init__.py @@ -5,15 +5,22 @@ Quick Start:: - from botanu import enable, botanu_workflow, emit_outcome + import botanu - enable() # reads config from OTEL_SERVICE_NAME, OTEL_EXPORTER_OTLP_ENDPOINT env vars + botanu.enable() # reads OTEL_SERVICE_NAME, OTEL_EXPORTER_OTLP_ENDPOINT env vars - @botanu_workflow(name="Customer Support") - async def handle_request(data): - result = await process(data) - emit_outcome("success", value_type="tickets_resolved", value_amount=1) - return result + # One wrap around the agent entrypoint captures every LLM/HTTP/DB call. + with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + agent.run(ticket) + + # Or as a decorator, with lambda extractors from the function args: + @botanu.event( + workflow="Support", + event_id=lambda t: t.id, + customer_id=lambda t: t.user_id, + ) + def handle_ticket(ticket): + ... """ from __future__ import annotations @@ -45,11 +52,11 @@ async def handle_request(data): set_baggage, ) -# Decorators (primary integration point) -from botanu.sdk.decorators import botanu_workflow, run_botanu, workflow +# Primary integration API +from botanu.sdk.decorators import event, step # Span helpers -from botanu.sdk.span_helpers import emit_outcome, set_business_context +from botanu.sdk.span_helpers import emit_outcome, set_business_context, set_correlation __all__ = [ "__version__", @@ -59,13 +66,13 @@ async def handle_request(data): "is_enabled", # Configuration "BotanuConfig", - # Decorators / context managers - "botanu_workflow", - "run_botanu", - "workflow", + # Primary API + "event", + "step", # Span helpers "emit_outcome", "set_business_context", + "set_correlation", "get_current_span", # Context "get_run_id", diff --git a/src/botanu/models/run_context.py b/src/botanu/models/run_context.py index 7161061..e5283ec 100644 --- a/src/botanu/models/run_context.py +++ b/src/botanu/models/run_context.py @@ -212,22 +212,30 @@ def duration_ms(self) -> Optional[float]: # Serialisation # ------------------------------------------------------------------ - def to_baggage_dict(self, lean_mode: Optional[bool] = None) -> Dict[str, str]: - """Convert to dict for W3C Baggage propagation.""" - if lean_mode is None: - env_mode = os.getenv("BOTANU_PROPAGATION_MODE", "lean") - lean_mode = env_mode != "full" - + def to_baggage_dict(self) -> Dict[str, str]: + """Convert to dict for W3C Baggage propagation. + + Always present: ``botanu.run_id``, ``botanu.workflow``, + ``botanu.event_id``, ``botanu.customer_id``, ``botanu.environment``. + + Included when set on the context: ``botanu.tenant_id``, + ``botanu.parent_run_id``, ``botanu.root_run_id`` (if non-root), + ``botanu.attempt`` (if > 1), ``botanu.retry_of_run_id``, + ``botanu.deadline``, ``botanu.cancelled``. + + The :class:`RunContextEnricher` stamps only the first seven + (run_id, workflow, event_id, customer_id, environment, tenant_id, + parent_run_id) on downstream spans. The remaining keys are for + :meth:`from_baggage` to reconstruct retry/deadline state on the + receiving side of cross-process propagation (e.g. message queues). + """ baggage: Dict[str, str] = { "botanu.run_id": self.run_id, "botanu.workflow": self.workflow, "botanu.event_id": self.event_id, "botanu.customer_id": self.customer_id, + "botanu.environment": self.environment, } - if lean_mode: - return baggage - - baggage["botanu.environment"] = self.environment if self.tenant_id: baggage["botanu.tenant_id"] = self.tenant_id if self.parent_run_id: diff --git a/src/botanu/processors/enricher.py b/src/botanu/processors/enricher.py index ee77377..173f1fb 100644 --- a/src/botanu/processors/enricher.py +++ b/src/botanu/processors/enricher.py @@ -8,10 +8,13 @@ - Only the SDK can read baggage and write it to span attributes. - The collector only sees spans after they're exported. -All heavy processing should happen in the OTel Collector: -- PII redaction → ``redactionprocessor`` +Heavy non-content processing happens in the OTel Collector: - Cardinality limits → ``attributesprocessor`` - Vendor detection → ``transformprocessor`` +- Belt-and-suspenders PII regex → ``redactionprocessor`` + +In-process PII scrubbing of content-capture attributes is handled by +:mod:`botanu.sdk.pii` at the tracker methods, not by a span processor. """ from __future__ import annotations @@ -29,17 +32,18 @@ class RunContextEnricher(SpanProcessor): """Enriches ALL spans with run context from baggage. - This ensures that every span (including auto-instrumented ones) - gets ``botanu.run_id``, ``botanu.workflow``, etc. attributes. - - Without this processor, only the root ``botanu.run`` span would - have these attributes. + This ensures that every span (including auto-instrumented ones) gets + ``botanu.run_id``, ``botanu.workflow``, ``botanu.event_id``, + ``botanu.customer_id``, ``botanu.environment``, ``botanu.tenant_id``, + and ``botanu.parent_run_id`` attributes when those baggage keys are + present on the active OTel context. - In ``lean_mode`` (default), only ``run_id`` and ``workflow`` are - propagated to minimise per-span overhead. + Without this processor, only the root ``botanu.run`` span would carry + these attributes; downstream auto-instrumented spans (LLM, HTTP, DB) + would not. """ - BAGGAGE_KEYS_FULL: ClassVar[List[str]] = [ + BAGGAGE_KEYS: ClassVar[List[str]] = [ "botanu.run_id", "botanu.workflow", "botanu.event_id", @@ -49,17 +53,6 @@ class RunContextEnricher(SpanProcessor): "botanu.parent_run_id", ] - BAGGAGE_KEYS_LEAN: ClassVar[List[str]] = [ - "botanu.run_id", - "botanu.workflow", - "botanu.event_id", - "botanu.customer_id", - ] - - def __init__(self, lean_mode: bool = True) -> None: - self._lean_mode = lean_mode - self._baggage_keys = self.BAGGAGE_KEYS_LEAN if lean_mode else self.BAGGAGE_KEYS_FULL - def on_start( self, span: Span, @@ -68,7 +61,7 @@ def on_start( """Called when a span starts — enrich with run context from baggage.""" ctx = parent_context or context.get_current() - for key in self._baggage_keys: + for key in self.BAGGAGE_KEYS: value = baggage.get_baggage(key, ctx) if value: if not span.attributes or key not in span.attributes: diff --git a/src/botanu/register.py b/src/botanu/register.py index 26ebb4d..600696e 100644 --- a/src/botanu/register.py +++ b/src/botanu/register.py @@ -23,7 +23,7 @@ def on_starting(server): uvicorn app:app --env-file .env # Or in Dockerfile - ENV BOTANU_API_KEY=btnu_live_... + ENV BOTANU_API_KEY= ENV BOTANU_SERVICE_NAME=my-service CMD ["python", "-c", "import botanu.register; import uvicorn; uvicorn.run('app:app')"] diff --git a/src/botanu/sampling/content_sampler.py b/src/botanu/sampling/content_sampler.py index 1978660..d94f566 100644 --- a/src/botanu/sampling/content_sampler.py +++ b/src/botanu/sampling/content_sampler.py @@ -36,4 +36,4 @@ def should_capture_content(rate: float, event_id: Optional[str] = None) -> bool: return False if rate >= 1.0: return True - return random.random() < rate + return random.random() < rate # noqa: S311 diff --git a/src/botanu/sdk/__init__.py b/src/botanu/sdk/__init__.py index daf3764..edbd038 100644 --- a/src/botanu/sdk/__init__.py +++ b/src/botanu/sdk/__init__.py @@ -14,7 +14,7 @@ get_workflow, set_baggage, ) -from botanu.sdk.decorators import botanu_outcome, botanu_workflow, run_botanu, workflow +from botanu.sdk.decorators import event, step from botanu.sdk.span_helpers import ( emit_outcome, set_business_context, @@ -23,20 +23,18 @@ __all__ = [ "BotanuConfig", - "botanu_outcome", - "botanu_workflow", "disable", "emit_outcome", "enable", + "event", "get_baggage", "get_config", "get_current_span", "get_run_id", "get_workflow", "is_enabled", - "run_botanu", "set_baggage", "set_business_context", "set_correlation", - "workflow", + "step", ] diff --git a/src/botanu/sdk/bootstrap.py b/src/botanu/sdk/bootstrap.py index c9658b5..e508817 100644 --- a/src/botanu/sdk/bootstrap.py +++ b/src/botanu/sdk/bootstrap.py @@ -216,9 +216,8 @@ def enable( resource = Resource.create(resource_attrs) from opentelemetry.trace import ProxyTracerProvider - from botanu.processors import ResourceEnricher, SampledSpanProcessor - lean_mode = cfg.propagation_mode == "lean" + from botanu.processors import ResourceEnricher, SampledSpanProcessor botanu_exporter = OTLPSpanExporter( endpoint=traces_endpoint, @@ -287,7 +286,7 @@ def enable( else: provider.add_span_processor(proc) - provider.add_span_processor(RunContextEnricher(lean_mode=lean_mode)) + provider.add_span_processor(RunContextEnricher()) if cfg.auto_instrument_resources: provider.add_span_processor(ResourceEnricher()) provider.add_span_processor(botanu_batch) @@ -312,7 +311,7 @@ def enable( elif isinstance(existing, ProxyTracerProvider): # GREENFIELD: no real provider — create fresh provider = TracerProvider(resource=resource, sampler=ALWAYS_ON) - provider.add_span_processor(RunContextEnricher(lean_mode=lean_mode)) + provider.add_span_processor(RunContextEnricher()) if cfg.auto_instrument_resources: provider.add_span_processor(ResourceEnricher()) provider.add_span_processor(botanu_batch) @@ -330,7 +329,7 @@ def enable( type(existing).__name__, ) provider = TracerProvider(resource=resource, sampler=ALWAYS_ON) - provider.add_span_processor(RunContextEnricher(lean_mode=lean_mode)) + provider.add_span_processor(RunContextEnricher()) if cfg.auto_instrument_resources: provider.add_span_processor(ResourceEnricher()) provider.add_span_processor(botanu_batch) @@ -350,9 +349,9 @@ def enable( # Set up LoggerProvider for outcome event emission try: from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.sdk._logs import LoggerProvider as _LoggerProvider from opentelemetry.sdk._logs.export import BatchLogRecordProcessor - from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter logs_endpoint = cfg.otlp_endpoint if logs_endpoint and not logs_endpoint.endswith("/v1/logs"): diff --git a/src/botanu/sdk/config.py b/src/botanu/sdk/config.py index 07378f7..4c0e537 100644 --- a/src/botanu/sdk/config.py +++ b/src/botanu/sdk/config.py @@ -3,11 +3,13 @@ """Configuration for Botanu SDK. -The SDK is intentionally minimal on the hot path. Heavy processing happens in -the OpenTelemetry Collector, not in the application: +The SDK is intentionally minimal on the hot path. Heavy non-content +processing happens in the OpenTelemetry Collector: -- **SDK responsibility**: Generate run_id, propagate minimal context (run_id, workflow) -- **Collector responsibility**: PII redaction, vendor detection, attribute enrichment +- **SDK responsibility**: generate run_id, propagate context, in-process PII + scrub on captured content (see :mod:`botanu.sdk.pii`) +- **Collector responsibility**: vendor detection, attribute enrichment, and + belt-and-suspenders PII regex on everything else Configuration precedence (highest to lowest): 1. Code arguments (explicit values passed to BotanuConfig) @@ -30,7 +32,7 @@ _BOTANU_HOST_SUFFIXES = (".botanu.ai",) -_BOTANU_DEV_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "0.0.0.0"}) +_BOTANU_DEV_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "0.0.0.0"}) # noqa: S104 _SENSITIVE_HEADER_NAMES = frozenset({"authorization", "x-api-key", "botanu-api-key"}) @@ -90,8 +92,9 @@ def _redact_headers(headers: Optional[Dict[str, str]]) -> Optional[Dict[str, str class BotanuConfig: """Configuration for Botanu SDK and OpenTelemetry. - The SDK is a thin wrapper on OpenTelemetry. PII redaction, cardinality - limits, and vendor enrichment are handled by the OTel Collector — not here. + The SDK is a thin wrapper on OpenTelemetry. In-process PII scrubbing + runs on captured content via :mod:`botanu.sdk.pii`; cardinality limits + and vendor enrichment are handled by the OTel Collector. Typically configured via environment variables (no hardcoded values):: @@ -123,16 +126,25 @@ class BotanuConfig: schedule_delay_millis: int = 5000 export_timeout_millis: int = 30000 - # Propagation mode: "lean" (run_id + workflow only) or "full" (all context) - propagation_mode: str = "lean" - # Content capture for eval — 0.0 disables entirely (default, privacy-safe). - # Set to 1.0 for sandbox/shadow, 0.10–0.20 for production. Customers must also + # Set to 1.0 for sandbox/shadow, 0.10-0.20 for production. Customers must also # call set_input_content() / set_output_content() on their trackers; this rate - # gates whether those calls actually write to span attributes. PII scrubbing - # happens downstream (collector regex + evaluator Presidio NER), not here. + # gates whether those calls actually write to span attributes. In-process PII + # scrubbing runs on the captured text before it hits the span (see + # pii_scrub_* fields below); collector regex + evaluator Presidio NER + # remain belt-and-suspenders. content_capture_rate: float = 0.0 + # In-process PII scrubbing — runs on text passed to set_input_content / + # set_output_content / set_retrieval_content before the span attribute is + # written. Default ON: safer for customers who enable content_capture_rate + # without reading the docs. + pii_scrub_enabled: bool = True + pii_scrub_disable_patterns: Optional[List[str]] = None + pii_scrub_custom_patterns: Optional[Dict[str, str]] = None + pii_scrub_use_presidio: bool = False + pii_scrub_replacement: str = "[REDACTED]" + # Resource-cost inference — default ON. When True, enable() attaches the # ResourceEnricher SpanProcessor which reads OTel semconv attributes # (db.system, http.*.body.size, aws.service, …) and writes the botanu- @@ -257,10 +269,6 @@ def __post_init__(self) -> None: urlparse(self.otlp_endpoint).hostname or "unknown", ) - env_propagation_mode = os.getenv("BOTANU_PROPAGATION_MODE") - if env_propagation_mode and env_propagation_mode in ("lean", "full"): - self.propagation_mode = env_propagation_mode - # Export tuning via env vars env_queue_size = os.getenv("BOTANU_MAX_QUEUE_SIZE") if env_queue_size: @@ -290,6 +298,24 @@ def __post_init__(self) -> None: except ValueError: pass + env_pii_enabled = os.getenv("BOTANU_PII_SCRUB_ENABLED") + if env_pii_enabled is not None: + self.pii_scrub_enabled = env_pii_enabled.lower() in ("true", "1", "yes") + + env_pii_disable = os.getenv("BOTANU_PII_SCRUB_DISABLE_PATTERNS") + if env_pii_disable is not None: + self.pii_scrub_disable_patterns = [ + name.strip() for name in env_pii_disable.split(",") if name.strip() + ] + + env_pii_presidio = os.getenv("BOTANU_PII_SCRUB_USE_PRESIDIO") + if env_pii_presidio is not None: + self.pii_scrub_use_presidio = env_pii_presidio.lower() in ("true", "1", "yes") + + env_pii_replacement = os.getenv("BOTANU_PII_SCRUB_REPLACEMENT") + if env_pii_replacement is not None: + self.pii_scrub_replacement = env_pii_replacement + # ------------------------------------------------------------------ # YAML loading # ------------------------------------------------------------------ @@ -381,9 +407,9 @@ def _from_dict( service = data.get("service", {}) otlp = data.get("otlp", {}) export = data.get("export", {}) - propagation = data.get("propagation", {}) resource = data.get("resource", {}) eval_cfg = data.get("eval", {}) + pii_cfg = eval_cfg.get("pii", {}) if isinstance(eval_cfg, dict) else {} auto_packages = data.get("auto_instrument_packages") return cls( @@ -398,8 +424,12 @@ def _from_dict( max_queue_size=export.get("queue_size", 65536), schedule_delay_millis=export.get("delay_ms", 5000), export_timeout_millis=export.get("export_timeout_ms", 30000), - propagation_mode=propagation.get("mode", "lean"), content_capture_rate=max(0.0, min(1.0, float(eval_cfg.get("content_capture_rate", 0.0)))), + pii_scrub_enabled=bool(pii_cfg.get("enabled", True)), + pii_scrub_disable_patterns=pii_cfg.get("disable_patterns"), + pii_scrub_custom_patterns=pii_cfg.get("custom_patterns"), + pii_scrub_use_presidio=bool(pii_cfg.get("use_presidio", False)), + pii_scrub_replacement=str(pii_cfg.get("replacement", "[REDACTED]")), auto_instrument_packages=(auto_packages if auto_packages else BotanuConfig().auto_instrument_packages), _config_file=config_file, ) @@ -426,11 +456,18 @@ def to_dict(self) -> Dict[str, Any]: "delay_ms": self.schedule_delay_millis, "export_timeout_ms": self.export_timeout_millis, }, - "propagation": { - "mode": self.propagation_mode, - }, "eval": { "content_capture_rate": self.content_capture_rate, + "pii": { + "enabled": self.pii_scrub_enabled, + "disable_patterns": self.pii_scrub_disable_patterns, + # Don't serialize custom regex bodies — they may be proprietary + # business rules the customer doesn't want round-tripped through + # logs. Expose only the count. + "custom_pattern_count": len(self.pii_scrub_custom_patterns or {}), + "use_presidio": self.pii_scrub_use_presidio, + "replacement": self.pii_scrub_replacement, + }, }, "auto_instrument_packages": self.auto_instrument_packages, } @@ -446,8 +483,9 @@ def __repr__(self) -> str: f"deployment_environment={self.deployment_environment!r}, " f"otlp_endpoint={redacted_endpoint!r}, " f"otlp_headers={redacted_headers!r}, " - f"propagation_mode={self.propagation_mode!r}, " - f"content_capture_rate={self.content_capture_rate!r})" + f"content_capture_rate={self.content_capture_rate!r}, " + f"pii_scrub_enabled={self.pii_scrub_enabled!r}, " + f"pii_scrub_use_presidio={self.pii_scrub_use_presidio!r})" ) diff --git a/src/botanu/sdk/decorators.py b/src/botanu/sdk/decorators.py index 424c57c..ee14d16 100644 --- a/src/botanu/sdk/decorators.py +++ b/src/botanu/sdk/decorators.py @@ -1,25 +1,25 @@ # SPDX-FileCopyrightText: 2026 The Botanu Authors # SPDX-License-Identifier: Apache-2.0 -"""Decorators for automatic run span creation and context propagation. +"""Primary integration API: :func:`event` and :func:`step`. + +``botanu.event(...)`` is the single integration point — works as a context +manager, an async context manager, or a decorator. It creates a run span +that: -The ``@botanu_workflow`` decorator is the primary integration point. -It creates a "run span" that: - Generates a UUIDv7 run_id - Emits ``run.started`` and ``run.completed`` events - Propagates run context via W3C Baggage -- Records outcome at completion +- Stamps business attributes on every downstream span via the enricher """ from __future__ import annotations -import contextlib import functools import hashlib import inspect import json -from collections.abc import Mapping -from contextlib import asynccontextmanager, contextmanager +from contextlib import contextmanager from datetime import datetime, timezone from typing import Any, Callable, Dict, Generator, Optional, TypeVar, Union @@ -49,21 +49,22 @@ def _get_parent_run_id() -> Optional[str]: return get_baggage("botanu.run_id") -# ── Content capture (workflow-level) ────────────────────────────────────── +# ── Content capture ─────────────────────────────────────────────────────── # # Gated by the same `content_capture_rate` config as LLMTracker so a single -# toggle controls both workflow-level and span-level capture. PII scrubbing -# is downstream (collector + evaluator) — see botanu/tracking/llm.py:332-333. +# toggle controls both workflow-level and span-level capture. In-process PII +# scrubbing runs on the serialized payload before the attribute is written — +# see botanu/sdk/pii.py. _CAPTURE_MAX_CHARS = 4096 def _should_capture_content() -> bool: - """Single decision per workflow invocation — applied to both input + output + """Single decision per event invocation — applied to both input + output so we never land a half-captured pair.""" try: - from botanu.sdk.bootstrap import get_config from botanu.sampling.content_sampler import should_capture_content + from botanu.sdk.bootstrap import get_config cfg = get_config() rate = cfg.content_capture_rate if cfg else 0.0 @@ -73,7 +74,11 @@ def _should_capture_content() -> bool: def _serialize_for_capture(obj: Any) -> str: - """Best-effort stringification. JSON first, repr fallback, truncated.""" + """Best-effort stringification. JSON first, repr fallback, truncated. + + PII scrub runs after serialization and before truncation so the regex + sees the joined string (catches e.g. an email spanning dict values). + """ try: text = json.dumps(obj, default=repr, ensure_ascii=False) except Exception: @@ -81,6 +86,15 @@ def _serialize_for_capture(obj: Any) -> str: text = repr(obj) except Exception: text = "" + try: + from botanu.sdk.bootstrap import get_config + from botanu.sdk.pii import apply_scrub + + cfg = get_config() + if cfg is not None: + text = apply_scrub(text, cfg) + except Exception: + pass return text[:_CAPTURE_MAX_CHARS] @@ -106,224 +120,6 @@ def _capture_output(span: trace.Span, result: Any) -> None: span.set_attribute("botanu.eval.output_content", _serialize_for_capture(result)) -def botanu_workflow( - name: str, - *, - event_id: Union[str, Callable[..., str]], - customer_id: Union[str, Callable[..., str]], - step: Optional[str] = None, - environment: Optional[str] = None, - tenant_id: Optional[str] = None, - auto_outcome_on_success: bool = True, - span_kind: SpanKind = SpanKind.SERVER, -) -> Callable[[Callable[..., T]], Callable[..., T]]: - """Decorator to create a run span with automatic context propagation. - - This is the primary integration point. It: - - 1. Creates a UUIDv7 ``run_id`` (sortable, globally unique) - 2. Creates a ``botanu.run`` span as the root of the run - 3. Emits ``run.started`` event - 4. Propagates run context via W3C Baggage - 5. On completion: emits ``run.completed`` event with outcome - - Args: - name: Workflow name (low cardinality, e.g. ``"Customer Support"``). - event_id: Business unit of work (e.g. ticket ID). Required. - Can be a static string or a callable that receives the same - ``(*args, **kwargs)`` as the decorated function and returns a string. - customer_id: End-customer being served (e.g. org ID). Required. - Can be a static string or a callable (same signature as *event_id*). - step: Step name within a multi-step workflow (e.g. ``"classify"``). - Optional — defaults to *name* for single-step workflows. - For downstream agents, workflow name and event_id are inherited - from W3C Baggage; only *step* needs to be set. - environment: Deployment environment. - tenant_id: Tenant identifier for multi-tenant apps. - auto_outcome_on_success: Emit ``"success"`` if no exception. - span_kind: OpenTelemetry span kind (default: ``SERVER``). - - Examples:: - - # Single-step workflow (step defaults to name): - @botanu_workflow("Support", event_id="ticket-123", customer_id="acme-corp") - async def handle_ticket(): ... - - # Multi-step workflow (explicit step name): - @botanu_workflow( - "Support", - step="classify", - event_id=lambda request: request.workflow_id, - customer_id=lambda request: request.customer_id, - ) - async def classify_ticket(request: TicketRequest): ... - - # Downstream step (inherits workflow from baggage): - @botanu_workflow("Support", step="research", event_id=lambda r: r.event_id, customer_id=lambda r: r.cid) - async def research(request): ... - """ - if isinstance(event_id, str) and not event_id: - raise ValueError("event_id is required and must be a non-empty string") - if isinstance(customer_id, str) and not customer_id: - raise ValueError("customer_id is required and must be a non-empty string") - if not callable(event_id) and not isinstance(event_id, str): - raise ValueError("event_id must be a non-empty string or a callable") - if not callable(customer_id) and not isinstance(customer_id, str): - raise ValueError("customer_id must be a non-empty string or a callable") - - def decorator(func: Callable[..., T]) -> Callable[..., T]: - workflow_version = _compute_workflow_version(func) - is_async = inspect.iscoroutinefunction(func) - - @functools.wraps(func) - async def async_wrapper(*args: Any, **kwargs: Any) -> T: - resolved_event_id = event_id(*args, **kwargs) if callable(event_id) else event_id - resolved_customer_id = customer_id(*args, **kwargs) if callable(customer_id) else customer_id - parent_run_id = _get_parent_run_id() - run_ctx = RunContext.create( - workflow=name, - event_id=resolved_event_id, - customer_id=resolved_customer_id, - workflow_version=workflow_version, - environment=environment, - tenant_id=tenant_id, - parent_run_id=parent_run_id, - ) - - with tracer.start_as_current_span( - name=f"botanu.run/{name}", - kind=span_kind, - ) as span: - for key, value in run_ctx.to_span_attributes().items(): - span.set_attribute(key, value) - - span.add_event( - "botanu.run.started", - attributes={ - "run_id": run_ctx.run_id, - "workflow": run_ctx.workflow, - }, - ) - - ctx = get_current() - for key, value in run_ctx.to_baggage_dict().items(): - ctx = otel_baggage.set_baggage(key, value, context=ctx) - baggage_token = attach(ctx) - - capture_content = _should_capture_content() - if capture_content: - _capture_input(span, func, args, kwargs) - - try: - result = await func(*args, **kwargs) - - if capture_content: - _capture_output(span, result) - - span_attrs = getattr(span, "attributes", None) - existing_outcome = ( - span_attrs.get("botanu.outcome.status") if isinstance(span_attrs, Mapping) else None - ) - - if existing_outcome is None and auto_outcome_on_success: - run_ctx.complete(RunStatus.SUCCESS) - - span.set_status(Status(StatusCode.OK)) - _emit_run_completed(span, run_ctx, RunStatus.SUCCESS) - return result - - except Exception as exc: - span.set_status(Status(StatusCode.ERROR, str(exc))) - span.record_exception(exc) - run_ctx.complete(RunStatus.FAILURE, error_class=exc.__class__.__name__) - _emit_run_completed( - span, - run_ctx, - RunStatus.FAILURE, - error_class=exc.__class__.__name__, - ) - raise - finally: - detach(baggage_token) - - @functools.wraps(func) - def sync_wrapper(*args: Any, **kwargs: Any) -> T: - resolved_event_id = event_id(*args, **kwargs) if callable(event_id) else event_id - resolved_customer_id = customer_id(*args, **kwargs) if callable(customer_id) else customer_id - parent_run_id = _get_parent_run_id() - run_ctx = RunContext.create( - workflow=name, - event_id=resolved_event_id, - customer_id=resolved_customer_id, - workflow_version=workflow_version, - environment=environment, - tenant_id=tenant_id, - parent_run_id=parent_run_id, - ) - - with tracer.start_as_current_span( - name=f"botanu.run/{name}", - kind=span_kind, - ) as span: - for key, value in run_ctx.to_span_attributes().items(): - span.set_attribute(key, value) - - span.add_event( - "botanu.run.started", - attributes={ - "run_id": run_ctx.run_id, - "workflow": run_ctx.workflow, - }, - ) - - ctx = get_current() - for key, value in run_ctx.to_baggage_dict().items(): - ctx = otel_baggage.set_baggage(key, value, context=ctx) - baggage_token = attach(ctx) - - capture_content = _should_capture_content() - if capture_content: - _capture_input(span, func, args, kwargs) - - try: - result = func(*args, **kwargs) - - if capture_content: - _capture_output(span, result) - - span_attrs = getattr(span, "attributes", None) - existing_outcome = ( - span_attrs.get("botanu.outcome.status") if isinstance(span_attrs, Mapping) else None - ) - - if existing_outcome is None and auto_outcome_on_success: - run_ctx.complete(RunStatus.SUCCESS) - - span.set_status(Status(StatusCode.OK)) - _emit_run_completed(span, run_ctx, RunStatus.SUCCESS) - return result - - except Exception as exc: - span.set_status(Status(StatusCode.ERROR, str(exc))) - span.record_exception(exc) - run_ctx.complete(RunStatus.FAILURE, error_class=exc.__class__.__name__) - _emit_run_completed( - span, - run_ctx, - RunStatus.FAILURE, - error_class=exc.__class__.__name__, - ) - raise - finally: - detach(baggage_token) - - if is_async: - return async_wrapper # type: ignore[return-value] - return sync_wrapper # type: ignore[return-value] - - return decorator - - def _emit_run_completed( span: trace.Span, run_ctx: RunContext, @@ -346,146 +142,318 @@ def _emit_run_completed( event_attrs["value_amount"] = run_ctx.outcome.value_amount span.add_event("botanu.run.completed", attributes=event_attrs) - - # `botanu.outcome.status` no longer emitted (removed 2026-04-16): - # customer-reported outcome is trivially fakeable. Event outcome derives - # from eval verdict rollup / HITL / SoR. `duration_ms` stays for perf. span.set_attribute("botanu.run.duration_ms", duration_ms) -workflow = botanu_workflow +# ── Unified primary API: botanu.event + botanu.step ────────────────────── +# +# One concept, three shapes (all equivalent): +# +# with botanu.event(event_id=..., customer_id=..., workflow="..."): ... +# async with botanu.event(event_id=..., customer_id=..., workflow="..."): ... +# @botanu.event(event_id=lambda t: t.id, customer_id=..., workflow="...") ... +# +# Capture of downstream LLM/HTTP/DB spans is done by enable()'s global OTel +# auto-instrumentation + the RunContextEnricher reading baggage — not by +# this class. -def botanu_outcome( - success: Optional[str] = None, - partial: Optional[str] = None, - failed: Optional[str] = None, -) -> Callable[[Callable[..., T]], Callable[..., T]]: - """Decorator to automatically emit outcomes based on function result. +class _Event: + """Dual-use: context manager (sync + async) and decorator. - This is a convenience decorator for sub-functions within a workflow. - It does NOT create a new run — use ``@botanu_workflow`` for that. + Customers construct via :func:`event`; they don't instantiate this + directly. Re-entrancy is not supported on a single instance — each + ``with`` / ``async with`` / ``@`` usage should get its own ``event(...)`` + call (which is the natural pattern). """ - from botanu.sdk.span_helpers import emit_outcome - def decorator(func: Callable[..., T]) -> Callable[..., T]: + def __init__( + self, + *, + event_id: Union[str, Callable[..., str]], + customer_id: Union[str, Callable[..., str]], + workflow: str, + environment: Optional[str] = None, + tenant_id: Optional[str] = None, + auto_outcome_on_success: bool = True, + capture_input: Optional[bool] = None, + span_kind: SpanKind = SpanKind.SERVER, + ) -> None: + if not workflow or not isinstance(workflow, str): + raise ValueError("workflow is required and must be a non-empty string") + if not callable(event_id): + if not isinstance(event_id, str) or not event_id: + raise ValueError("event_id must be a non-empty string or a callable") + if not callable(customer_id): + if not isinstance(customer_id, str) or not customer_id: + raise ValueError("customer_id must be a non-empty string or a callable") + + self.event_id = event_id + self.customer_id = customer_id + self.workflow = workflow + self.environment = environment + self.tenant_id = tenant_id + self.auto_outcome_on_success = auto_outcome_on_success + self.capture_input = capture_input + self.span_kind = span_kind + + self._span_cm: Any = None + self._span: Optional[trace.Span] = None + self._baggage_token: Any = None + self._run_ctx: Optional[RunContext] = None + + def _begin( + self, + resolved_event_id: str, + resolved_customer_id: str, + workflow_version: Optional[str] = None, + ): + parent_run_id = _get_parent_run_id() + run_ctx = RunContext.create( + workflow=self.workflow, + event_id=resolved_event_id, + customer_id=resolved_customer_id, + workflow_version=workflow_version, + environment=self.environment, + tenant_id=self.tenant_id, + parent_run_id=parent_run_id, + ) + span_cm = tracer.start_as_current_span( + name=f"botanu.run/{self.workflow}", + kind=self.span_kind, + ) + span = span_cm.__enter__() + for key, value in run_ctx.to_span_attributes().items(): + span.set_attribute(key, value) + span.add_event( + "botanu.run.started", + attributes={"run_id": run_ctx.run_id, "workflow": run_ctx.workflow}, + ) + ctx = get_current() + for key, value in run_ctx.to_baggage_dict().items(): + ctx = otel_baggage.set_baggage(key, value, context=ctx) + baggage_token = attach(ctx) + return span_cm, span, baggage_token, run_ctx + + def _end_success(self, span_cm, span, baggage_token, run_ctx) -> None: + if self.auto_outcome_on_success: + run_ctx.complete(RunStatus.SUCCESS) + span.set_status(Status(StatusCode.OK)) + _emit_run_completed(span, run_ctx, RunStatus.SUCCESS) + detach(baggage_token) + span_cm.__exit__(None, None, None) + + def _end_failure(self, span_cm, span, baggage_token, run_ctx, exc) -> None: + span.set_status(Status(StatusCode.ERROR, str(exc))) + span.record_exception(exc) + run_ctx.complete(RunStatus.FAILURE, error_class=exc.__class__.__name__) + _emit_run_completed( + span, run_ctx, RunStatus.FAILURE, error_class=exc.__class__.__name__, + ) + detach(baggage_token) + span_cm.__exit__(type(exc), exc, exc.__traceback__) + + def _resolve_capture(self) -> bool: + if self.capture_input is True: + return True + if self.capture_input is False: + return False + return _should_capture_content() + + # ── Sync context manager ── + def __enter__(self) -> RunContext: + if callable(self.event_id) or callable(self.customer_id): + raise TypeError( + "botanu.event(...) with a callable event_id/customer_id must be " + "used as a decorator, not as a context manager. " + "Either pass resolved string values, or use @botanu.event(...)." + ) + span_cm, span, token, run_ctx = self._begin(self.event_id, self.customer_id) + self._span_cm = span_cm + self._span = span + self._baggage_token = token + self._run_ctx = run_ctx + return run_ctx + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + span_cm = self._span_cm + span = self._span + token = self._baggage_token + run_ctx = self._run_ctx + self._span_cm = self._span = self._baggage_token = self._run_ctx = None + if exc_type is None: + self._end_success(span_cm, span, token, run_ctx) + else: + self._end_failure(span_cm, span, token, run_ctx, exc_val) + return False # propagate exceptions + + # ── Async context manager ── + async def __aenter__(self) -> RunContext: + return self.__enter__() + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> bool: + return self.__exit__(exc_type, exc_val, exc_tb) + + # ── Decorator ── + def __call__(self, func: Callable[..., T]) -> Callable[..., T]: + workflow_version = _compute_workflow_version(func) is_async = inspect.iscoroutinefunction(func) + parent = self @functools.wraps(func) async def async_wrapper(*args: Any, **kwargs: Any) -> T: + eid = parent.event_id(*args, **kwargs) if callable(parent.event_id) else parent.event_id + cid = parent.customer_id(*args, **kwargs) if callable(parent.customer_id) else parent.customer_id + span_cm, span, token, run_ctx = parent._begin(eid, cid, workflow_version) + capture = parent._resolve_capture() + if capture: + _capture_input(span, func, args, kwargs) try: result = await func(*args, **kwargs) - span = trace.get_current_span() - if not span.attributes or "botanu.outcome.status" not in span.attributes: - emit_outcome("success") + if capture: + _capture_output(span, result) + parent._end_success(span_cm, span, token, run_ctx) return result except Exception as exc: - emit_outcome("failed", reason=exc.__class__.__name__) + parent._end_failure(span_cm, span, token, run_ctx, exc) raise @functools.wraps(func) def sync_wrapper(*args: Any, **kwargs: Any) -> T: + eid = parent.event_id(*args, **kwargs) if callable(parent.event_id) else parent.event_id + cid = parent.customer_id(*args, **kwargs) if callable(parent.customer_id) else parent.customer_id + span_cm, span, token, run_ctx = parent._begin(eid, cid, workflow_version) + capture = parent._resolve_capture() + if capture: + _capture_input(span, func, args, kwargs) try: result = func(*args, **kwargs) - span = trace.get_current_span() - if not span.attributes or "botanu.outcome.status" not in span.attributes: - emit_outcome("success") + if capture: + _capture_output(span, result) + parent._end_success(span_cm, span, token, run_ctx) return result except Exception as exc: - emit_outcome("failed", reason=exc.__class__.__name__) + parent._end_failure(span_cm, span, token, run_ctx, exc) raise - if is_async: - return async_wrapper # type: ignore[return-value] - return sync_wrapper # type: ignore[return-value] + return async_wrapper if is_async else sync_wrapper # type: ignore[return-value] - return decorator +def _ensure_enabled() -> None: + """Lazy-init the SDK on first ``event()`` call so customers don't have to + remember a separate ``botanu.enable()``. Explicit ``enable(...)`` is still + honoured — this is a no-op if already initialised.""" + from botanu.sdk.bootstrap import enable, is_enabled + if not is_enabled(): + enable() -@contextmanager -def run_botanu( - name: str, + +def event( *, - event_id: str, - customer_id: str, + event_id: Union[str, Callable[..., str]], + customer_id: Union[str, Callable[..., str]], + workflow: str, environment: Optional[str] = None, tenant_id: Optional[str] = None, auto_outcome_on_success: bool = True, + capture_input: Optional[bool] = None, span_kind: SpanKind = SpanKind.SERVER, -) -> Generator[RunContext, None, None]: - """Context manager to create a run span — non-decorator alternative to ``@botanu_workflow``. +) -> _Event: + """Mark a scope as a botanu business event — the primary integration point. - Use this when you can't decorate a function (dynamic workflows, simple scripts, - or when the workflow name is determined at runtime). + Works as a context manager, an async context manager, or a decorator. All + three forms capture the same LLM/HTTP/DB calls inside the scope and stamp + ``event_id``, ``customer_id``, and ``workflow`` onto every captured span + via W3C baggage. Args: - name: Workflow name (low cardinality, e.g. ``"Customer Support"``). - event_id: Business unit of work (e.g. ticket ID). - customer_id: End-customer being served (e.g. org ID). + event_id: Business event identifier — the primary join key for outcome + correlation (SoR webhooks, HITL reviews, etc.). In decorator form + can be a callable that receives the decorated function's args. + In context manager form must be a resolved string. + customer_id: End-customer being served. Same callable/string rules as + *event_id*. + workflow: Workflow name (low cardinality, e.g. ``"Customer Support"``). environment: Deployment environment. - tenant_id: Tenant identifier for multi-tenant apps. - auto_outcome_on_success: Emit ``"success"`` if no exception. + tenant_id: Tenant identifier. + auto_outcome_on_success: Mark run complete with SUCCESS on clean exit. + capture_input: Force content capture on/off. ``None`` (default) uses + the sampled ``content_capture_rate`` from bootstrap config. span_kind: OpenTelemetry span kind (default: ``SERVER``). - Yields: - RunContext with the generated run_id and metadata. + Examples:: - Example:: + # Context manager — works anywhere (scripts, notebooks, wrapping agents) + with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + agent.run(ticket) + + # Async context manager + async with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + await agent.arun(ticket) - with run_botanu("Support", event_id="ticket-42", customer_id="acme") as run: - result = call_llm(...) - emit_outcome("success", value_type="tickets_resolved", value_amount=1) + # Decorator — sugar for production handlers (supports lambda extractors) + @botanu.event( + workflow="Support", + event_id=lambda t: t.id, + customer_id=lambda t: t.user_id, + ) + def handle_ticket(ticket): + ... + + .. note:: + ``enable()`` runs implicitly on the first ``event()`` call if the SDK + hasn't been initialised yet. Call ``botanu.enable(...)`` explicitly + only if you need to override config (custom endpoint, API key, etc.). """ - parent_run_id = _get_parent_run_id() - run_ctx = RunContext.create( - workflow=name, + _ensure_enabled() + return _Event( event_id=event_id, customer_id=customer_id, + workflow=workflow, environment=environment, tenant_id=tenant_id, - parent_run_id=parent_run_id, + auto_outcome_on_success=auto_outcome_on_success, + capture_input=capture_input, + span_kind=span_kind, ) - with tracer.start_as_current_span( - name=f"botanu.run/{name}", - kind=span_kind, - ) as span: - for key, value in run_ctx.to_span_attributes().items(): - span.set_attribute(key, value) - - span.add_event( - "botanu.run.started", - attributes={"run_id": run_ctx.run_id, "workflow": run_ctx.workflow}, - ) - ctx = get_current() - for key, value in run_ctx.to_baggage_dict().items(): - ctx = otel_baggage.set_baggage(key, value, context=ctx) - baggage_token = attach(ctx) +@contextmanager +def step(name: str) -> Generator[trace.Span, None, None]: + """Mark a phase within a :func:`event` scope. - try: - yield run_ctx + Use nested inside ``with botanu.event(...)`` to break a multi-step workflow + into phases (e.g., ``"retrieval"``, ``"generation"``, ``"validation"``). + Each step emits its own span and propagates ``botanu.step=`` via + baggage so downstream spans inherit the step label. - span_attrs = getattr(span, "attributes", None) - existing_outcome = ( - span_attrs.get("botanu.outcome.status") - if isinstance(span_attrs, Mapping) - else None - ) + Args: + name: Step name (low cardinality, stable across invocations). - if existing_outcome is None and auto_outcome_on_success: - run_ctx.complete(RunStatus.SUCCESS) + Yields: + The step span. - span.set_status(Status(StatusCode.OK)) - _emit_run_completed(span, run_ctx, RunStatus.SUCCESS) + Example:: + with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + with botanu.step("retrieval"): + docs = vector_db.query(...) + with botanu.step("generation"): + response = llm.complete(docs) + """ + with tracer.start_as_current_span( + name=f"botanu.step/{name}", + kind=SpanKind.INTERNAL, + ) as span: + span.set_attribute("botanu.step", name) + ctx = get_current() + ctx = otel_baggage.set_baggage("botanu.step", name, context=ctx) + token = attach(ctx) + try: + yield span except Exception as exc: span.set_status(Status(StatusCode.ERROR, str(exc))) span.record_exception(exc) - run_ctx.complete(RunStatus.FAILURE, error_class=exc.__class__.__name__) - _emit_run_completed( - span, run_ctx, RunStatus.FAILURE, error_class=exc.__class__.__name__, - ) raise finally: - detach(baggage_token) + detach(token) diff --git a/src/botanu/sdk/middleware.py b/src/botanu/sdk/middleware.py index 1f01175..603db96 100644 --- a/src/botanu/sdk/middleware.py +++ b/src/botanu/sdk/middleware.py @@ -10,7 +10,6 @@ from __future__ import annotations import uuid -from typing import Optional from opentelemetry import baggage as otel_baggage from opentelemetry import trace diff --git a/src/botanu/sdk/pii.py b/src/botanu/sdk/pii.py new file mode 100644 index 0000000..24a3a2a --- /dev/null +++ b/src/botanu/sdk/pii.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: 2026 The Botanu Authors +# SPDX-License-Identifier: Apache-2.0 + +"""In-process PII scrubber for LLM content capture. + +Runs inside the customer process, before content is written to span +attributes — so prompts and responses never leave the application with +emails, keys, card numbers, etc. intact. Defense-in-depth alongside the +collector regex pass and the evaluator Presidio NER. + +Scope is intentionally narrow: this module only scrubs text passed to +``LLMTracker.set_input_content`` / ``LLMTracker.set_output_content`` / +``DBTracker.set_retrieval_content``. Auto-instrumented attributes +(``http.request.body``, ``db.statement``, …) are handled by the +collector denylist. +""" + +from __future__ import annotations + +import logging +import re +import threading +from typing import Dict, List, Optional, Tuple + +logger = logging.getLogger(__name__) + + +_BUILTIN_PATTERNS: Dict[str, str] = { + # Order-sensitive — specific API-key prefixes must run before the generic + # openai_key / anthropic_key patterns, and credit_card runs before phone + # to avoid a long card number being partially captured as a phone. + "bearer_token": r"Bearer\s+[A-Za-z0-9._\-]+", + "jwt": r"\beyJ[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+\b", + "aws_access_key": r"\bAKIA[0-9A-Z]{16}\b", + "github_token": r"\bgh[pousr]_[A-Za-z0-9]{36,}\b", + "stripe_key": r"\b(?:sk|rk)_live_[A-Za-z0-9]{24,}\b", + "slack_token": r"\bxox[baprs]-[A-Za-z0-9-]+\b", + "anthropic_key": r"\bsk-ant-[A-Za-z0-9\-_]{20,}\b", + "openai_key": r"\bsk-(?:proj-)?[A-Za-z0-9]{20,}\b", + "credit_card": r"\b(?:\d[ -]?){12,18}\d\b", + "ssn_us": r"\b\d{3}-\d{2}-\d{4}\b", + "email": r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b", + "phone_e164": r"\+[1-9]\d{6,14}\b", + "phone_us": r"(?:\(\d{3}\)\s?\d{3}[-.]\d{4}|\b\d{3}[-.]\d{3}[-.]\d{4}\b)", + "ipv6": r"\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\b", + "ipv4": r"\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b", +} + + +def _luhn_valid(digits: str) -> bool: + """Return True if *digits* passes the Luhn checksum. Strips non-digits.""" + d = [int(c) for c in digits if c.isdigit()] + if len(d) < 13 or len(d) > 19: + return False + checksum = 0 + parity = len(d) % 2 + for i, n in enumerate(d): + if i % 2 == parity: + n *= 2 + if n > 9: + n -= 9 + checksum += n + return checksum % 10 == 0 + + +class PIIScrubber: + """Stateless regex-based PII scrubber. + + Compile once at construction; call :meth:`scrub` on every content string. + Credit-card hits are Luhn-validated to reject the 16-digit order IDs and + nonces that would otherwise trip the pattern. + """ + + def __init__( + self, + enabled_patterns: Optional[List[str]] = None, + disabled_patterns: Optional[List[str]] = None, + custom_patterns: Optional[Dict[str, str]] = None, + replacement: str = "[REDACTED]", + ) -> None: + self.replacement = replacement + selected: List[Tuple[str, str]] = [] + disabled = set(disabled_patterns or []) + for name, pattern in _BUILTIN_PATTERNS.items(): + if enabled_patterns is not None and name not in enabled_patterns: + continue + if name in disabled: + continue + selected.append((name, pattern)) + for name, pattern in (custom_patterns or {}).items(): + selected.append((name, pattern)) + self._compiled: List[Tuple[str, re.Pattern[str]]] = [] + for name, pattern in selected: + try: + self._compiled.append((name, re.compile(pattern))) + except re.error as exc: + logger.warning("Botanu PII scrubber: invalid regex for %s: %s", name, exc) + + def scrub(self, text: str) -> str: + if not text: + return text + out = text + for name, compiled in self._compiled: + if name == "credit_card": + out = compiled.sub( + lambda m: self.replacement if _luhn_valid(m.group(0)) else m.group(0), + out, + ) + else: + out = compiled.sub(self.replacement, out) + return out + + +_cached_scrubber: Optional[PIIScrubber] = None +_cached_scrubber_key: Optional[Tuple] = None # type: ignore[type-arg] +_cache_lock = threading.RLock() + + +def _scrubber_from_config(cfg: object) -> Optional[PIIScrubber]: + """Return a cached :class:`PIIScrubber` built from *cfg*, or None if disabled. + + Cache key is the subset of config fields the scrubber depends on, so a + config reload with different PII settings rebuilds automatically without + recompiling on every call. Lock-guarded for the FastAPI / Celery threaded + worker case — lock contention is only paid on cache miss (rare). + """ + global _cached_scrubber, _cached_scrubber_key + enabled = bool(getattr(cfg, "pii_scrub_enabled", False)) + if not enabled: + return None + disabled = tuple(getattr(cfg, "pii_scrub_disable_patterns", None) or ()) + custom_items = getattr(cfg, "pii_scrub_custom_patterns", None) or {} + custom = tuple(sorted(custom_items.items())) + replacement = str(getattr(cfg, "pii_scrub_replacement", "[REDACTED]")) + use_presidio = bool(getattr(cfg, "pii_scrub_use_presidio", False)) + key = (enabled, disabled, custom, replacement, use_presidio) + # Fast path: read without holding the lock — regex compilation is idempotent + # so even a torn read at worst returns a scrubber one generation stale. + if key == _cached_scrubber_key and _cached_scrubber is not None: + return _cached_scrubber + with _cache_lock: + if key != _cached_scrubber_key: + _cached_scrubber = PIIScrubber( + disabled_patterns=list(disabled), + custom_patterns=dict(custom), + replacement=replacement, + ) + _cached_scrubber_key = key + return _cached_scrubber + + +def apply_scrub(text: str, cfg: object) -> str: + """Scrub *text* according to *cfg*. Returns *text* unchanged if disabled. + + Called from the three content-capture tracker methods after the content + sampling gate has already decided to capture. Safe to call on empty / + None text — returns the input verbatim. + """ + if not text: + return text + scrubber = _scrubber_from_config(cfg) + if scrubber is None: + return text + scrubbed = scrubber.scrub(text) + if bool(getattr(cfg, "pii_scrub_use_presidio", False)): + try: + from botanu.sdk.pii_presidio import presidio_scrub + except ImportError: + return scrubbed + scrubbed = presidio_scrub(scrubbed, replacement=str(getattr(cfg, "pii_scrub_replacement", "[REDACTED]"))) + return scrubbed + + +def _reset_cache_for_tests() -> None: + """Test-only hook: drop the cached scrubber so a new config takes effect.""" + global _cached_scrubber, _cached_scrubber_key + with _cache_lock: + _cached_scrubber = None + _cached_scrubber_key = None diff --git a/src/botanu/sdk/pii_presidio.py b/src/botanu/sdk/pii_presidio.py new file mode 100644 index 0000000..48fffb1 --- /dev/null +++ b/src/botanu/sdk/pii_presidio.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: 2026 The Botanu Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Optional Presidio NER layer on top of the regex scrubber. + +Opt-in via ``BotanuConfig(pii_scrub_use_presidio=True)`` or +``BOTANU_PII_SCRUB_USE_PRESIDIO=true``. Requires the ``pii-nlp`` extra:: + + pip install botanu[pii-nlp] + +If Presidio is not installed, :func:`presidio_scrub` logs a warning once +and returns the input unchanged — callers should rely on the regex pass +for their floor guarantee. + +Mirrors the evaluator's :mod:`app.pii.presidio_scrubber` entity list so +SDK and evaluator redact the same categories (double coverage by design). +""" + +from __future__ import annotations + +import logging +from typing import List, Optional + +logger = logging.getLogger(__name__) + +_DEFAULT_ENTITIES: List[str] = [ + "EMAIL_ADDRESS", + "PHONE_NUMBER", + "CREDIT_CARD", + "US_SSN", + "PERSON", + "LOCATION", + "IP_ADDRESS", + "US_BANK_NUMBER", + "MEDICAL_LICENSE", +] + +_analyzer = None +_anonymizer = None +_operators = None +_initialized = False +_available = False +_warned = False + + +def _ensure_initialized(replacement: str) -> bool: + """Lazy-load Presidio engines. Returns True if available. + + Cold start is ~1s (spaCy model load); cached for process lifetime. + """ + global _analyzer, _anonymizer, _operators, _initialized, _available, _warned + if _initialized: + return _available + _initialized = True + try: + from presidio_analyzer import AnalyzerEngine + from presidio_anonymizer import AnonymizerEngine + from presidio_anonymizer.entities import OperatorConfig + + _analyzer = AnalyzerEngine() + _anonymizer = AnonymizerEngine() + _operators = {"DEFAULT": OperatorConfig("replace", {"new_value": replacement})} + _available = True + return True + except ImportError: + if not _warned: + logger.warning( + "Botanu PII scrubber: pii_scrub_use_presidio=True but presidio is not installed. " + "Install with: pip install botanu[pii-nlp]. Falling back to regex-only scrubbing." + ) + _warned = True + return False + + +def presidio_scrub(text: str, replacement: str = "[REDACTED]", entities: Optional[List[str]] = None) -> str: + """Return *text* with Presidio-detected PII replaced by *replacement*. + + No-op when the package is not installed — regex-only scrubbing still + runs upstream in :func:`botanu.sdk.pii.apply_scrub`. + """ + if not text: + return text + if not _ensure_initialized(replacement): + return text + results = _analyzer.analyze(text=text, entities=entities or _DEFAULT_ENTITIES, language="en") # type: ignore[union-attr] + anonymized = _anonymizer.anonymize(text=text, analyzer_results=results, operators=_operators) # type: ignore[union-attr] + return anonymized.text + + +def _reset_for_tests() -> None: + """Test-only hook to re-trigger the lazy init after monkey-patching imports.""" + global _analyzer, _anonymizer, _operators, _initialized, _available, _warned + _analyzer = None + _anonymizer = None + _operators = None + _initialized = False + _available = False + _warned = False diff --git a/src/botanu/sdk/span_helpers.py b/src/botanu/sdk/span_helpers.py index 042e8a6..fd39827 100644 --- a/src/botanu/sdk/span_helpers.py +++ b/src/botanu/sdk/span_helpers.py @@ -9,28 +9,14 @@ from __future__ import annotations import logging -import warnings from typing import Optional from opentelemetry import trace logger = logging.getLogger(__name__) -VALID_OUTCOME_STATUSES = { - "success", "partial", "failed", "timeout", "canceled", "abandoned", -} - -_DEPRECATION_MSG = ( - "emit_outcome(status=...) no longer stamps `botanu.outcome.status` on the " - "span — customer-reported outcome has been removed (it was trivially " - "fakeable). Event outcome is now derived from eval verdict rollup / HITL / " - "SoR. You can remove this call, or keep it for the diagnostic fields " - "(reason, error_type, value_*, confidence, metadata) which still stamp." -) - def emit_outcome( - status: str, *, value_type: Optional[str] = None, value_amount: Optional[float] = None, @@ -39,63 +25,37 @@ def emit_outcome( error_type: Optional[str] = None, metadata: Optional[dict[str, str]] = None, ) -> None: - """Emit diagnostic outcome fields on the current span. (DEPRECATED for status.) + """Emit diagnostic outcome fields on the current span. - The ``status`` argument no longer stamps ``botanu.outcome.status`` — - customer-reported outcome was removed on 2026-04-16 (trivially fakeable). - Event outcome is now derived from eval verdict rollup / HITL / SoR. - - All other fields (``value_type``, ``value_amount``, ``confidence``, - ``reason``, ``error_type``, ``metadata``) still stamp as diagnostic - attributes — useful for debugging and dashboards, not for authoritative - outcome determination. + These are **diagnostic only** — the authoritative event outcome is + resolved server-side from SoR connectors / HITL reviews / eval verdict + rollup. Use these fields to enrich dashboards with business value + signals (tickets resolved, dollars saved, etc.) and error diagnostics. Args: - status: Accepted for backward compatibility. A ``DeprecationWarning`` - is emitted. Must still be one of the valid statuses for validation. value_type: Type of business value (e.g., ``"tickets_resolved"``). value_amount: Quantified value amount. confidence: Confidence score (0.0–1.0). reason: Optional diagnostic reason. error_type: Error classification (e.g., ``"ValidationError"``). metadata: Additional diagnostic key-value metadata. - - Raises: - ValueError: If *status* is not a recognised outcome status. """ - if status not in VALID_OUTCOME_STATUSES: - raise ValueError( - f"Invalid outcome status '{status}'. " - f"Must be one of: {', '.join(sorted(VALID_OUTCOME_STATUSES))}" - ) - - warnings.warn(_DEPRECATION_MSG, DeprecationWarning, stacklevel=2) - span = trace.get_current_span() - # `botanu.outcome.status` is NOT emitted — see deprecation notice. - if value_type: span.set_attribute("botanu.outcome.value_type", value_type) - if value_amount is not None: span.set_attribute("botanu.outcome.value_amount", value_amount) - if confidence is not None: span.set_attribute("botanu.outcome.confidence", confidence) - if reason: span.set_attribute("botanu.outcome.reason", reason) - if error_type: span.set_attribute("botanu.outcome.error_type", error_type) - if metadata: for key, value in metadata.items(): span.set_attribute(f"botanu.outcome.metadata.{key}", value) - # Keep the span event for diagnostic visibility (event, not authoritative), - # minus the `status` attribute to stay consistent with the removal. event_attrs: dict[str, object] = {} if value_type: event_attrs["value_type"] = value_type @@ -103,14 +63,8 @@ def emit_outcome( event_attrs["value_amount"] = value_amount if error_type: event_attrs["error_type"] = error_type - span.add_event("botanu.outcome_emitted", event_attrs) - # OTel log emission for collector flush trigger has been removed: - # the collector's outcome-log flush trigger is being retired as part of - # the customer-push outcome deprecation. Events flush via idle timeout - # and max-lifetime triggers instead. - def set_business_context( *, @@ -178,9 +132,9 @@ def set_business_context( def set_correlation(**correlations: Optional[str]) -> None: """Stamp one or more `botanu.correlation.*` span attributes. - Called inside a ``@botanu_workflow`` to link the current event to one or - more external SoR records. The sor-connector uses these attributes to - correlate inbound webhooks (ticket reopen, refund, etc.) back to this + Called inside a :func:`botanu.event` scope to link the current event to + one or more external SoR records. The sor-connector uses these attributes + to correlate inbound webhooks (ticket reopen, refund, etc.) back to this event via Tier-1 correlation. Each keyword becomes a span attribute. A ``None`` or empty-string value @@ -192,8 +146,7 @@ def set_correlation(**correlations: Optional[str]) -> None: Example:: - @botanu_workflow("Support", event_id="evt-42", customer_id="acme") - def handle(ticket): + with botanu.event(event_id="evt-42", customer_id="acme", workflow="Support"): set_correlation(zendesk_ticket_id=ticket.id) ... """ diff --git a/src/botanu/tracking/data.py b/src/botanu/tracking/data.py index 0c73195..5852ca0 100644 --- a/src/botanu/tracking/data.py +++ b/src/botanu/tracking/data.py @@ -194,22 +194,25 @@ def set_retrieval_content(self, text: str, max_chars: int = 4096) -> DBTracker: Writes the ``botanu.eval.retrieval_content`` span attribute only if the active config's ``content_capture_rate`` > 0.0 allows this call. - Truncates to ``max_chars`` (default 4096) before stamping. + Pipeline: sampling gate → PII scrub (on by default, see + :mod:`botanu.sdk.pii`) → truncate to ``max_chars`` (default 4096) → + ``set_attribute``. - PII scrubbing is handled downstream (collector + evaluator). No-op when ``span`` is unset, ``text`` is empty/None, or the rate excludes this call. """ if not self.span or not text: return self - from botanu.sdk.bootstrap import get_config from botanu.sampling.content_sampler import should_capture_content + from botanu.sdk.bootstrap import get_config + from botanu.sdk.pii import apply_scrub cfg = get_config() rate = cfg.content_capture_rate if cfg else 0.0 if not should_capture_content(rate): return self - self.span.set_attribute("botanu.eval.retrieval_content", text[:max_chars]) + scrubbed = apply_scrub(text, cfg) if cfg else text + self.span.set_attribute("botanu.eval.retrieval_content", scrubbed[:max_chars]) return self def set_error(self, error: Exception) -> DBTracker: diff --git a/src/botanu/tracking/llm.py b/src/botanu/tracking/llm.py index 188be22..f75e655 100644 --- a/src/botanu/tracking/llm.py +++ b/src/botanu/tracking/llm.py @@ -326,43 +326,48 @@ def set_input_content(self, text: str, max_chars: int = 4096) -> LLMTracker: Writes the ``botanu.eval.input_content`` span attribute only if the active :class:`~botanu.sdk.config.BotanuConfig` has a ``content_capture_rate`` > 0.0 that allows this call (simple - ``random.random() < rate`` gate). Truncates to ``max_chars`` - (default 4096) before stamping. + ``random.random() < rate`` gate). - PII scrubbing is handled downstream by the collector (regex pass) - and the evaluator (Presidio NER), not here. + Pipeline: sampling gate → PII scrub (on by default, regex + optional + Presidio — see :mod:`botanu.sdk.pii`) → truncate to ``max_chars`` + (default 4096) → ``set_attribute``. No-op when ``span`` is unset, ``text`` is empty/None, or the config rate excludes this call. """ if not self.span or not text: return self - from botanu.sdk.bootstrap import get_config from botanu.sampling.content_sampler import should_capture_content + from botanu.sdk.bootstrap import get_config + from botanu.sdk.pii import apply_scrub cfg = get_config() rate = cfg.content_capture_rate if cfg else 0.0 if not should_capture_content(rate): return self - self.span.set_attribute("botanu.eval.input_content", text[:max_chars]) + scrubbed = apply_scrub(text, cfg) if cfg else text + self.span.set_attribute("botanu.eval.input_content", scrubbed[:max_chars]) return self def set_output_content(self, text: str, max_chars: int = 4096) -> LLMTracker: """Capture the response/output text for eval. - See :meth:`set_input_content` for sampling and truncation semantics. - Writes the ``botanu.eval.output_content`` span attribute. + See :meth:`set_input_content` for sampling, PII scrubbing, and + truncation semantics. Writes the ``botanu.eval.output_content`` + span attribute. """ if not self.span or not text: return self - from botanu.sdk.bootstrap import get_config from botanu.sampling.content_sampler import should_capture_content + from botanu.sdk.bootstrap import get_config + from botanu.sdk.pii import apply_scrub cfg = get_config() rate = cfg.content_capture_rate if cfg else 0.0 if not should_capture_content(rate): return self - self.span.set_attribute("botanu.eval.output_content", text[:max_chars]) + scrubbed = apply_scrub(text, cfg) if cfg else text + self.span.set_attribute("botanu.eval.output_content", scrubbed[:max_chars]) return self def set_request_params( diff --git a/tests/conftest.py b/tests/conftest.py index 4cd5ad8..e7472e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from opentelemetry import trace from opentelemetry._logs import set_logger_provider from opentelemetry.sdk._logs import LoggerProvider -from opentelemetry.sdk._logs.export import SimpleLogRecordProcessor, InMemoryLogExporter +from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter diff --git a/tests/unit/test_bootstrap.py b/tests/unit/test_bootstrap.py index 22ee4aa..e26ae7b 100644 --- a/tests/unit/test_bootstrap.py +++ b/tests/unit/test_bootstrap.py @@ -117,30 +117,6 @@ def test_explicit_args_override_all_env(self): # --------------------------------------------------------------------------- -class TestConfigPropagationMode: - """Tests for propagation mode configuration.""" - - def test_default_lean(self): - with mock.patch.dict(os.environ, {}, clear=True): - cfg = BotanuConfig() - assert cfg.propagation_mode == "lean" - - def test_env_var_full(self): - with mock.patch.dict(os.environ, {"BOTANU_PROPAGATION_MODE": "full"}): - cfg = BotanuConfig() - assert cfg.propagation_mode == "full" - - def test_env_var_lean(self): - with mock.patch.dict(os.environ, {"BOTANU_PROPAGATION_MODE": "lean"}): - cfg = BotanuConfig() - assert cfg.propagation_mode == "lean" - - def test_invalid_propagation_mode_ignored(self): - with mock.patch.dict(os.environ, {"BOTANU_PROPAGATION_MODE": "invalid"}): - cfg = BotanuConfig() - assert cfg.propagation_mode == "lean" - - # --------------------------------------------------------------------------- # Config: auto-detect resources # --------------------------------------------------------------------------- @@ -693,12 +669,11 @@ def _restore_bootstrap(self): def test_existing_sdk_provider_always_on(self): """When an AlwaysOn SDKTracerProvider exists, botanu migrates its processors.""" - from opentelemetry import trace + from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.sdk.trace.sampling import ALWAYS_ON - from opentelemetry.sdk.resources import Resource self._reset_bootstrap() try: @@ -728,12 +703,11 @@ def test_existing_sdk_provider_always_on(self): def test_existing_sdk_provider_ratio_sampling(self): """When a ratio-sampling provider exists, botanu wraps processors in SampledSpanProcessor.""" - from opentelemetry import trace + from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.sdk.trace.sampling import TraceIdRatioBased - from opentelemetry.sdk.resources import Resource self._reset_bootstrap() try: @@ -819,6 +793,7 @@ def test_enable_called_twice_returns_false(self): def test_sampled_span_processor_deterministic(self): """SampledSpanProcessor produces deterministic results for the same trace_id.""" from unittest.mock import MagicMock + from botanu.processors.sampled import SampledSpanProcessor inner = MagicMock() @@ -841,6 +816,7 @@ def test_sampled_span_processor_deterministic(self): def test_sampled_span_processor_ratio_bounds(self): """SampledSpanProcessor respects ratio=0.0 (drop all) and ratio=1.0 (keep all).""" from unittest.mock import MagicMock + from botanu.processors.sampled import SampledSpanProcessor inner_zero = MagicMock() @@ -862,6 +838,7 @@ def test_extract_sampler_ratio_always_on(self): """_extract_sampler_ratio returns 1.0 for AlwaysOn sampler.""" from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.sampling import ALWAYS_ON + from botanu.sdk.bootstrap import _extract_sampler_ratio provider = TracerProvider(sampler=ALWAYS_ON) @@ -871,6 +848,7 @@ def test_extract_sampler_ratio_trace_id_ratio(self): """_extract_sampler_ratio returns correct ratio for TraceIdRatioBased.""" from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.sampling import TraceIdRatioBased + from botanu.sdk.bootstrap import _extract_sampler_ratio provider = TracerProvider(sampler=TraceIdRatioBased(0.25)) @@ -881,6 +859,7 @@ def test_extract_sampler_ratio_parent_based(self): """_extract_sampler_ratio extracts ratio from ParentBased wrapping ratio sampler.""" from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased + from botanu.sdk.bootstrap import _extract_sampler_ratio provider = TracerProvider(sampler=ParentBased(TraceIdRatioBased(0.05))) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 4f3a955..23c8d50 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -52,7 +52,6 @@ def test_default_values(self): assert config.service_name == "unknown_service" assert config.deployment_environment == "production" - assert config.propagation_mode == "lean" assert config.auto_detect_resources is True def test_env_var_service_name(self): @@ -84,12 +83,6 @@ def test_explicit_values_override_env(self): config = BotanuConfig(service_name="explicit-service") assert config.service_name == "explicit-service" - def test_env_var_propagation_mode(self): - with mock.patch.dict(os.environ, {"BOTANU_PROPAGATION_MODE": "full"}): - config = BotanuConfig() - assert config.propagation_mode == "full" - - class TestBotanuConfigFromYaml: """Tests for loading config from YAML.""" @@ -329,11 +322,6 @@ def test_botanu_environment_over_otel(self): config = BotanuConfig() assert config.deployment_environment == "staging" - def test_propagation_mode_rejects_invalid(self): - with mock.patch.dict(os.environ, {"BOTANU_PROPAGATION_MODE": "invalid"}): - config = BotanuConfig() - assert config.propagation_mode == "lean" - def test_auto_detect_resources_env_false(self): with mock.patch.dict(os.environ, {"BOTANU_AUTO_DETECT_RESOURCES": "false"}): config = BotanuConfig() @@ -504,3 +492,76 @@ def test_to_dict_roundtrip(self): config = BotanuConfig(content_capture_rate=0.1) d = config.to_dict() assert d["eval"]["content_capture_rate"] == 0.1 + + +class TestPIIScrubConfig: + """Tests for pii_scrub_* config fields and env-var parsing.""" + + def test_default_enabled(self): + with mock.patch.dict(os.environ, {}, clear=True): + config = BotanuConfig() + assert config.pii_scrub_enabled is True + assert config.pii_scrub_use_presidio is False + assert config.pii_scrub_replacement == "[REDACTED]" + + def test_env_var_disables(self): + with mock.patch.dict(os.environ, {"BOTANU_PII_SCRUB_ENABLED": "false"}): + config = BotanuConfig() + assert config.pii_scrub_enabled is False + + def test_env_var_enabled_variants(self): + for value in ("true", "1", "yes"): + with mock.patch.dict(os.environ, {"BOTANU_PII_SCRUB_ENABLED": value}): + assert BotanuConfig().pii_scrub_enabled is True + + def test_env_var_unset_keeps_default_on(self): + with mock.patch.dict(os.environ, {}, clear=True): + os.environ.pop("BOTANU_PII_SCRUB_ENABLED", None) + assert BotanuConfig().pii_scrub_enabled is True + + def test_env_disable_patterns_list(self): + with mock.patch.dict(os.environ, {"BOTANU_PII_SCRUB_DISABLE_PATTERNS": "ipv4, ipv6 , email"}): + config = BotanuConfig() + assert config.pii_scrub_disable_patterns == ["ipv4", "ipv6", "email"] + + def test_env_use_presidio(self): + with mock.patch.dict(os.environ, {"BOTANU_PII_SCRUB_USE_PRESIDIO": "true"}): + assert BotanuConfig().pii_scrub_use_presidio is True + + def test_env_replacement(self): + with mock.patch.dict(os.environ, {"BOTANU_PII_SCRUB_REPLACEMENT": ""}): + assert BotanuConfig().pii_scrub_replacement == "" + + def test_yaml_loads_pii_section(self, tmp_path): + cfg_path = tmp_path / "botanu.yaml" + cfg_path.write_text( + "eval:\n" + " content_capture_rate: 0.2\n" + " pii:\n" + " enabled: false\n" + " disable_patterns: [ipv4]\n" + " custom_patterns:\n" + " order: 'ORD-\\d+'\n" + " use_presidio: true\n" + " replacement: ''\n" + ) + config = BotanuConfig.from_yaml(str(cfg_path)) + assert config.pii_scrub_enabled is False + assert config.pii_scrub_disable_patterns == ["ipv4"] + assert config.pii_scrub_custom_patterns == {"order": "ORD-\\d+"} + assert config.pii_scrub_use_presidio is True + assert config.pii_scrub_replacement == "" + + def test_to_dict_exposes_pii_settings(self): + config = BotanuConfig(pii_scrub_custom_patterns={"emp": r"EMP-\d+"}) + d = config.to_dict() + assert d["eval"]["pii"]["enabled"] is True + assert d["eval"]["pii"]["custom_pattern_count"] == 1 + # Custom regex body must not leak — only the count is exposed. + assert "EMP" not in str(d) + + def test_repr_does_not_leak_custom_patterns(self): + config = BotanuConfig(pii_scrub_custom_patterns={"emp": r"EMP-\d+"}) + text = repr(config) + assert "EMP" not in text + assert "pii_scrub_enabled=True" in text diff --git a/tests/unit/test_data_tracking.py b/tests/unit/test_data_tracking.py index 7fd2cef..b24a335 100644 --- a/tests/unit/test_data_tracking.py +++ b/tests/unit/test_data_tracking.py @@ -552,3 +552,53 @@ def test_returns_self_for_chaining(self, memory_exporter): ) as tracker: result = tracker.set_retrieval_content("doc").set_table("docs") assert result is tracker + + +class TestRetrievalContentPIIScrubbing: + """Retrieval content runs through the same scrubber pipeline.""" + + def _with_config(self, **cfg_kwargs): + from contextlib import contextmanager + + from botanu.sdk import bootstrap + from botanu.sdk.config import BotanuConfig + from botanu.sdk.pii import _reset_cache_for_tests + + @contextmanager + def _cm(): + prev = bootstrap._current_config + cfg_kwargs.setdefault("content_capture_rate", 1.0) + bootstrap._current_config = BotanuConfig(**cfg_kwargs) + _reset_cache_for_tests() + try: + yield + finally: + bootstrap._current_config = prev + _reset_cache_for_tests() + + return _cm() + + def test_retrieval_content_scrubbed_by_default(self, memory_exporter): + with self._with_config(): + with track_db_operation( + system="postgresql", + operation=DBOperation.SELECT, + database="kb", + ) as tracker: + tracker.set_retrieval_content("Snippet about alice@example.com") + + attrs = dict(memory_exporter.get_finished_spans()[0].attributes) + assert "alice@example.com" not in attrs["botanu.eval.retrieval_content"] + + def test_opt_out_preserves_retrieval_text(self, memory_exporter): + text = "Snippet about alice@example.com" + with self._with_config(pii_scrub_enabled=False): + with track_db_operation( + system="postgresql", + operation=DBOperation.SELECT, + database="kb", + ) as tracker: + tracker.set_retrieval_content(text) + + attrs = dict(memory_exporter.get_finished_spans()[0].attributes) + assert attrs["botanu.eval.retrieval_content"] == text diff --git a/tests/unit/test_decorators.py b/tests/unit/test_decorators.py deleted file mode 100644 index a3069fa..0000000 --- a/tests/unit/test_decorators.py +++ /dev/null @@ -1,460 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The Botanu Authors -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for SDK decorators.""" - -from __future__ import annotations - -import pytest -from opentelemetry import baggage, trace -from opentelemetry import context as otel_context -from opentelemetry.context import get_current - -from botanu.sdk.decorators import botanu_outcome, botanu_workflow - - -@pytest.fixture(autouse=True) -def _clean_otel_context(): - """Reset OTel context before each test to avoid baggage leaking between tests.""" - token = otel_context.attach(otel_context.Context()) - yield - otel_context.detach(token) - - -class TestBotanuWorkflowDecorator: - """Tests for @botanu_workflow decorator.""" - - def test_sync_function_creates_span(self, memory_exporter): - @botanu_workflow("Test Workflow", event_id="evt-1", customer_id="cust-1") - def my_function(): - return "result" - - result = my_function() - - assert result == "result" - spans = memory_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "botanu.run/Test Workflow" - - def test_span_has_run_attributes(self, memory_exporter): - @botanu_workflow("Customer Support", event_id="ticket-42", customer_id="bigretail") - def my_function(): - return "done" - - my_function() - - spans = memory_exporter.get_finished_spans() - attrs = dict(spans[0].attributes) - - assert "botanu.run_id" in attrs - assert attrs["botanu.workflow"] == "Customer Support" - assert attrs["botanu.event_id"] == "ticket-42" - assert attrs["botanu.customer_id"] == "bigretail" - - def test_emits_started_event(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def my_function(): - pass - - my_function() - - spans = memory_exporter.get_finished_spans() - events = spans[0].events - - started_events = [e for e in events if e.name == "botanu.run.started"] - assert len(started_events) == 1 - - def test_emits_completed_event(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def my_function(): - return "done" - - my_function() - - spans = memory_exporter.get_finished_spans() - events = spans[0].events - - completed_events = [e for e in events if e.name == "botanu.run.completed"] - assert len(completed_events) == 1 - assert completed_events[0].attributes["status"] == "success" - - def test_records_exception_on_failure(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def failing_function(): - raise ValueError("test error") - - with pytest.raises(ValueError): - failing_function() - - spans = memory_exporter.get_finished_spans() - assert len(spans) == 1 - - events = spans[0].events - completed_events = [e for e in events if e.name == "botanu.run.completed"] - assert len(completed_events) == 1 - assert completed_events[0].attributes["status"] == "failure" - assert completed_events[0].attributes["error_class"] == "ValueError" - - @pytest.mark.asyncio - async def test_async_function_creates_span(self, memory_exporter): - @botanu_workflow("Async Test", event_id="evt-1", customer_id="cust-1") - async def async_function(): - return "async result" - - result = await async_function() - - assert result == "async result" - spans = memory_exporter.get_finished_spans() - assert len(spans) == 1 - assert spans[0].name == "botanu.run/Async Test" - - @pytest.mark.asyncio - async def test_async_exception_handling(self, memory_exporter): - @botanu_workflow("Async Test", event_id="evt-1", customer_id="cust-1") - async def failing_async(): - raise RuntimeError("async error") - - with pytest.raises(RuntimeError): - await failing_async() - - spans = memory_exporter.get_finished_spans() - events = spans[0].events - completed_events = [e for e in events if e.name == "botanu.run.completed"] - assert completed_events[0].attributes["status"] == "failure" - - def test_workflow_version_computed(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def versioned_function(): - return "versioned" - - versioned_function() - - spans = memory_exporter.get_finished_spans() - attrs = dict(spans[0].attributes) - - assert "botanu.workflow.version" in attrs - assert attrs["botanu.workflow.version"].startswith("v:") - - def test_return_value_preserved(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def returns_dict(): - return {"key": "value", "count": 42} - - result = returns_dict() - assert result == {"key": "value", "count": 42} - - @pytest.mark.asyncio - async def test_async_return_value_preserved(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - async def returns_data(): - return [1, 2, 3] - - result = await returns_data() - assert result == [1, 2, 3] - - def test_exception_re_raised(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def raises(): - raise TypeError("bad type") - - with pytest.raises(TypeError, match="bad type"): - raises() - - def test_outcome_status_not_emitted_on_success(self, memory_exporter): - """`botanu.outcome.status` is no longer emitted (removed 2026-04-16) — - customer-reported outcome is trivially fakeable. Event outcome is - derived from eval verdict rollup / HITL / SoR instead.""" - - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def my_fn(): - return "ok" - - my_fn() - spans = memory_exporter.get_finished_spans() - attrs = dict(spans[0].attributes) - assert "botanu.outcome.status" not in attrs - - def test_outcome_status_not_emitted_on_failure(self, memory_exporter): - """Same removal applies on the failure path.""" - - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def failing(): - raise RuntimeError("boom") - - with pytest.raises(RuntimeError): - failing() - - spans = memory_exporter.get_finished_spans() - attrs = dict(spans[0].attributes) - assert "botanu.outcome.status" not in attrs - - def test_duration_ms_recorded(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def quick_fn(): - return "done" - - quick_fn() - spans = memory_exporter.get_finished_spans() - attrs = dict(spans[0].attributes) - assert "botanu.run.duration_ms" in attrs - assert attrs["botanu.run.duration_ms"] >= 0 - - def test_custom_span_kind(self, memory_exporter): - from opentelemetry.trace import SpanKind - - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1", span_kind=SpanKind.CLIENT) - def client_fn(): - return "ok" - - client_fn() - spans = memory_exporter.get_finished_spans() - assert spans[0].kind == SpanKind.CLIENT - - def test_root_run_id_equals_run_id_for_root(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1") - def root_fn(): - return "root" - - root_fn() - spans = memory_exporter.get_finished_spans() - attrs = dict(spans[0].attributes) - # For a root run, root_run_id should equal run_id - assert attrs["botanu.root_run_id"] == attrs["botanu.run_id"] - - def test_tenant_id_propagated(self, memory_exporter): - @botanu_workflow("Test", event_id="evt-1", customer_id="cust-1", tenant_id="tenant-abc") - def tenant_fn(): - return "ok" - - tenant_fn() - spans = memory_exporter.get_finished_spans() - attrs = dict(spans[0].attributes) - assert attrs["botanu.tenant_id"] == "tenant-abc" - - def test_baggage_cleaned_up_after_sync(self, memory_exporter): - """Verify baggage does NOT leak after the decorated function completes.""" - - @botanu_workflow("Leak Test", event_id="evt-1", customer_id="cust-1") - def my_fn(): - # Inside the function, baggage should be set - assert baggage.get_baggage("botanu.run_id", get_current()) is not None - return "ok" - - # Before: no baggage - assert baggage.get_baggage("botanu.run_id", get_current()) is None - - my_fn() - - # After: baggage must be cleaned up (detached) - assert baggage.get_baggage("botanu.run_id", get_current()) is None - - @pytest.mark.asyncio - async def test_baggage_cleaned_up_after_async(self, memory_exporter): - """Verify baggage does NOT leak after an async decorated function.""" - - @botanu_workflow("Async Leak Test", event_id="evt-1", customer_id="cust-1") - async def my_fn(): - assert baggage.get_baggage("botanu.run_id", get_current()) is not None - return "ok" - - assert baggage.get_baggage("botanu.run_id", get_current()) is None - - await my_fn() - - assert baggage.get_baggage("botanu.run_id", get_current()) is None - - def test_baggage_cleaned_up_after_exception(self, memory_exporter): - """Verify baggage is cleaned up even when the function raises.""" - - @botanu_workflow("Exception Leak Test", event_id="evt-1", customer_id="cust-1") - def failing_fn(): - raise RuntimeError("boom") - - assert baggage.get_baggage("botanu.run_id", get_current()) is None - - with pytest.raises(RuntimeError): - failing_fn() - - # Must be cleaned up despite the exception - assert baggage.get_baggage("botanu.run_id", get_current()) is None - - def test_event_id_required(self): - """Should raise ValueError if event_id is missing.""" - with pytest.raises(ValueError, match="event_id is required"): - @botanu_workflow("Test", event_id="", customer_id="cust-1") - def my_fn(): - pass - - def test_customer_id_required(self): - """Should raise ValueError if customer_id is missing.""" - with pytest.raises(ValueError, match="customer_id is required"): - @botanu_workflow("Test", event_id="evt-1", customer_id="") - def my_fn(): - pass - - def test_event_id_and_customer_id_in_baggage(self, memory_exporter): - """Verify event_id and customer_id are propagated via baggage.""" - - @botanu_workflow("Baggage Test", event_id="ticket-99", customer_id="acme-corp") - def my_fn(): - assert baggage.get_baggage("botanu.event_id", get_current()) == "ticket-99" - assert baggage.get_baggage("botanu.customer_id", get_current()) == "acme-corp" - return "ok" - - my_fn() - - -class TestBotanuWorkflowContentCapture: - """Tests for @botanu_workflow content capture into botanu.eval.* attrs.""" - - def test_no_capture_when_rate_is_zero(self, memory_exporter, monkeypatch): - # Default rate=0.0 → nothing captured. - @botanu_workflow("Triage", event_id="ticket-1", customer_id="acme") - def handle(ticket_id: str, priority: int = 1) -> dict: - return {"status": "resolved", "ticket_id": ticket_id} - - handle("ticket-1", priority=3) - - attrs = dict(memory_exporter.get_finished_spans()[0].attributes) - assert "botanu.eval.input_content" not in attrs - assert "botanu.eval.output_content" not in attrs - - def test_captures_input_and_output_when_enabled(self, memory_exporter, monkeypatch): - # Force the capture gate on — bypass the random sampler for determinism. - monkeypatch.setattr( - "botanu.sdk.decorators._should_capture_content", lambda: True - ) - - @botanu_workflow("Triage", event_id="ticket-2", customer_id="acme") - def handle(ticket_id: str, priority: int = 1) -> dict: - return {"status": "resolved", "ticket_id": ticket_id} - - handle("ticket-2", priority=5) - - attrs = dict(memory_exporter.get_finished_spans()[0].attributes) - assert "botanu.eval.input_content" in attrs - assert "botanu.eval.output_content" in attrs - - # Input payload keys are the function's parameter names, not "args"/"kwargs" - assert "ticket_id" in attrs["botanu.eval.input_content"] - assert "priority" in attrs["botanu.eval.input_content"] - assert "5" in attrs["botanu.eval.input_content"] - - assert "resolved" in attrs["botanu.eval.output_content"] - assert "ticket-2" in attrs["botanu.eval.output_content"] - - def test_capture_truncates_large_output(self, memory_exporter, monkeypatch): - monkeypatch.setattr( - "botanu.sdk.decorators._should_capture_content", lambda: True - ) - - @botanu_workflow("Bulk", event_id="evt-3", customer_id="acme") - def handle() -> str: - return "x" * 10_000 - - handle() - - attrs = dict(memory_exporter.get_finished_spans()[0].attributes) - captured = attrs["botanu.eval.output_content"] - assert len(captured) <= 4096 + len('""') # 4096 content chars + JSON quotes - - def test_capture_survives_unserializable_args(self, memory_exporter, monkeypatch): - monkeypatch.setattr( - "botanu.sdk.decorators._should_capture_content", lambda: True - ) - - class Opaque: - def __repr__(self) -> str: - return "" - - @botanu_workflow("Weird", event_id="evt-4", customer_id="acme") - def handle(obj) -> str: - return "ok" - - handle(Opaque()) - - attrs = dict(memory_exporter.get_finished_spans()[0].attributes) - # Should not raise; should contain the repr of Opaque - assert "botanu.eval.input_content" in attrs - assert "Opaque" in attrs["botanu.eval.input_content"] - - def test_input_captured_even_if_function_raises(self, memory_exporter, monkeypatch): - """Input is captured BEFORE the call; output is not captured on exception.""" - monkeypatch.setattr( - "botanu.sdk.decorators._should_capture_content", lambda: True - ) - - @botanu_workflow("Fails", event_id="evt-5", customer_id="acme") - def handle(x: int) -> int: - raise RuntimeError("boom") - - with pytest.raises(RuntimeError): - handle(42) - - attrs = dict(memory_exporter.get_finished_spans()[0].attributes) - assert "botanu.eval.input_content" in attrs - assert "42" in attrs["botanu.eval.input_content"] - assert "botanu.eval.output_content" not in attrs - - -class TestBotanuOutcomeDecorator: - """Tests for @botanu_outcome decorator.""" - - def test_sync_success_emits_outcome(self, memory_exporter): - tracer_instance = trace.get_tracer("test") - - @botanu_outcome() - def my_fn(): - return "ok" - - with tracer_instance.start_as_current_span("parent"): - result = my_fn() - - assert result == "ok" - - def test_sync_failure_emits_failed(self, memory_exporter): - tracer_instance = trace.get_tracer("test") - - @botanu_outcome() - def failing_fn(): - raise ValueError("broken") - - with tracer_instance.start_as_current_span("parent"): - with pytest.raises(ValueError, match="broken"): - failing_fn() - - @pytest.mark.asyncio - async def test_async_success_emits_outcome(self, memory_exporter): - tracer_instance = trace.get_tracer("test") - - @botanu_outcome() - async def async_fn(): - return "async ok" - - with tracer_instance.start_as_current_span("parent"): - result = await async_fn() - - assert result == "async ok" - - @pytest.mark.asyncio - async def test_async_failure_emits_failed(self, memory_exporter): - tracer_instance = trace.get_tracer("test") - - @botanu_outcome() - async def async_fail(): - raise RuntimeError("async boom") - - with tracer_instance.start_as_current_span("parent"): - with pytest.raises(RuntimeError, match="async boom"): - await async_fail() - - def test_exception_re_raised(self, memory_exporter): - tracer_instance = trace.get_tracer("test") - - @botanu_outcome() - def raises(): - raise TypeError("type err") - - with tracer_instance.start_as_current_span("parent"): - with pytest.raises(TypeError, match="type err"): - raises() diff --git a/tests/unit/test_enricher.py b/tests/unit/test_enricher.py index ed4dde1..40047a4 100644 --- a/tests/unit/test_enricher.py +++ b/tests/unit/test_enricher.py @@ -16,52 +16,10 @@ class TestRunContextEnricher: """Tests for RunContextEnricher processor.""" - def test_init_lean_mode_default(self): - """Default should be lean mode.""" - enricher = RunContextEnricher() - assert enricher._lean_mode is True - assert enricher._baggage_keys == RunContextEnricher.BAGGAGE_KEYS_LEAN - - def test_init_lean_mode_false(self): - """Can enable full mode.""" - enricher = RunContextEnricher(lean_mode=False) - assert enricher._lean_mode is False - assert enricher._baggage_keys == RunContextEnricher.BAGGAGE_KEYS_FULL - def test_on_start_reads_baggage(self, memory_exporter): - """on_start should read baggage and set span attributes.""" - enricher = RunContextEnricher(lean_mode=True) - - # Set up baggage context - start from a clean context - ctx = context.Context() - ctx = baggage.set_baggage("botanu.run_id", "test-run-123", context=ctx) - ctx = baggage.set_baggage("botanu.workflow", "Test Case", context=ctx) - ctx = baggage.set_baggage("botanu.event_id", "evt-42", context=ctx) - ctx = baggage.set_baggage("botanu.customer_id", "cust-abc", context=ctx) - - # Create a span with the baggage context - tracer = trace.get_tracer("test") - token = context.attach(ctx) - try: - with tracer.start_as_current_span("test-span") as span: - # Manually call on_start to simulate processor behavior - enricher.on_start(span, ctx) - finally: - context.detach(token) - - spans = memory_exporter.get_finished_spans() - assert len(spans) == 1 - attrs = dict(spans[0].attributes) - assert attrs.get("botanu.run_id") == "test-run-123" - assert attrs.get("botanu.workflow") == "Test Case" - assert attrs.get("botanu.event_id") == "evt-42" - assert attrs.get("botanu.customer_id") == "cust-abc" - - def test_on_start_full_mode(self, memory_exporter): - """Full mode should read all baggage keys.""" - enricher = RunContextEnricher(lean_mode=False) + """on_start reads all baggage keys and stamps them as span attributes.""" + enricher = RunContextEnricher() - # Set up baggage context with all keys - start from a clean context ctx = context.Context() ctx = baggage.set_baggage("botanu.run_id", "run-456", context=ctx) ctx = baggage.set_baggage("botanu.workflow", "Full Test", context=ctx) @@ -161,15 +119,11 @@ def test_force_flush_returns_true(self): def test_baggage_keys_constants(self): """Verify baggage key constants.""" - assert "botanu.run_id" in RunContextEnricher.BAGGAGE_KEYS_LEAN - assert "botanu.workflow" in RunContextEnricher.BAGGAGE_KEYS_LEAN - assert "botanu.event_id" in RunContextEnricher.BAGGAGE_KEYS_LEAN - assert "botanu.customer_id" in RunContextEnricher.BAGGAGE_KEYS_LEAN - assert len(RunContextEnricher.BAGGAGE_KEYS_LEAN) == 4 - - assert "botanu.run_id" in RunContextEnricher.BAGGAGE_KEYS_FULL - assert "botanu.event_id" in RunContextEnricher.BAGGAGE_KEYS_FULL - assert "botanu.customer_id" in RunContextEnricher.BAGGAGE_KEYS_FULL - assert "botanu.workflow" in RunContextEnricher.BAGGAGE_KEYS_FULL - assert "botanu.environment" in RunContextEnricher.BAGGAGE_KEYS_FULL - assert len(RunContextEnricher.BAGGAGE_KEYS_FULL) == 7 + assert "botanu.run_id" in RunContextEnricher.BAGGAGE_KEYS + assert "botanu.event_id" in RunContextEnricher.BAGGAGE_KEYS + assert "botanu.customer_id" in RunContextEnricher.BAGGAGE_KEYS + assert "botanu.workflow" in RunContextEnricher.BAGGAGE_KEYS + assert "botanu.environment" in RunContextEnricher.BAGGAGE_KEYS + assert "botanu.tenant_id" in RunContextEnricher.BAGGAGE_KEYS + assert "botanu.parent_run_id" in RunContextEnricher.BAGGAGE_KEYS + assert len(RunContextEnricher.BAGGAGE_KEYS) == 7 diff --git a/tests/unit/test_event.py b/tests/unit/test_event.py new file mode 100644 index 0000000..995ca06 --- /dev/null +++ b/tests/unit/test_event.py @@ -0,0 +1,393 @@ +# SPDX-FileCopyrightText: 2026 The Botanu Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the unified primary API — :func:`botanu.event` and :func:`botanu.step`. + +Covers all three shapes (sync CM, async CM, decorator) and verifies +capture-equivalence with the legacy ``@botanu_workflow`` decorator: same +span attributes, same baggage, same run.started/run.completed events. +""" + +from __future__ import annotations + +import pytest +from opentelemetry import baggage as otel_baggage +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.context import get_current + +import botanu +from botanu.processors.enricher import RunContextEnricher +from botanu.sdk.decorators import _Event + + +@pytest.fixture(autouse=True) +def _clean_otel_context(): + token = otel_context.attach(otel_context.Context()) + yield + otel_context.detach(token) + + +def _attrs(spans): + return dict(spans[0].attributes) + + +# ── Context manager form ───────────────────────────────────────────────── + +class TestEventContextManager: + def test_sync_cm_creates_span(self, memory_exporter): + with botanu.event(event_id="evt-1", customer_id="cust-1", workflow="Support"): + pass + spans = memory_exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == "botanu.run/Support" + + def test_sync_cm_stamps_business_attributes(self, memory_exporter): + with botanu.event(event_id="ticket-42", customer_id="acme", workflow="Support"): + pass + attrs = _attrs(memory_exporter.get_finished_spans()) + assert attrs["botanu.event_id"] == "ticket-42" + assert attrs["botanu.customer_id"] == "acme" + assert attrs["botanu.workflow"] == "Support" + assert "botanu.run_id" in attrs + + def test_sync_cm_emits_started_and_completed_events(self, memory_exporter): + with botanu.event(event_id="e", customer_id="c", workflow="W"): + pass + events = memory_exporter.get_finished_spans()[0].events + assert any(e.name == "botanu.run.started" for e in events) + completed = [e for e in events if e.name == "botanu.run.completed"] + assert len(completed) == 1 + assert completed[0].attributes["status"] == "success" + + def test_sync_cm_records_exception_as_failure(self, memory_exporter): + with pytest.raises(ValueError, match="boom"): + with botanu.event(event_id="e", customer_id="c", workflow="W"): + raise ValueError("boom") + events = memory_exporter.get_finished_spans()[0].events + completed = next(e for e in events if e.name == "botanu.run.completed") + assert completed.attributes["status"] == "failure" + assert completed.attributes["error_class"] == "ValueError" + + def test_sync_cm_sets_baggage_for_downstream_spans(self, memory_exporter): + with botanu.event(event_id="e", customer_id="c", workflow="W"): + ctx = get_current() + assert otel_baggage.get_baggage("botanu.event_id", context=ctx) == "e" + assert otel_baggage.get_baggage("botanu.customer_id", context=ctx) == "c" + assert otel_baggage.get_baggage("botanu.workflow", context=ctx) == "W" + + def test_cm_rejects_callable_event_id(self, memory_exporter): + ev = botanu.event( + event_id=lambda *a, **k: "dynamic", + customer_id="c", + workflow="W", + ) + with pytest.raises(TypeError, match="decorator"): + with ev: + pass + + def test_missing_workflow_raises(self): + with pytest.raises(ValueError, match="workflow"): + botanu.event(event_id="e", customer_id="c", workflow="") + + def test_empty_event_id_raises(self): + with pytest.raises(ValueError, match="event_id"): + botanu.event(event_id="", customer_id="c", workflow="W") + + def test_empty_customer_id_raises(self): + with pytest.raises(ValueError, match="customer_id"): + botanu.event(event_id="e", customer_id="", workflow="W") + + +# ── Async context manager form ─────────────────────────────────────────── + +class TestEventAsyncContextManager: + @pytest.mark.asyncio + async def test_async_cm_creates_span(self, memory_exporter): + async with botanu.event(event_id="e", customer_id="c", workflow="Async"): + pass + spans = memory_exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == "botanu.run/Async" + + @pytest.mark.asyncio + async def test_async_cm_records_exception(self, memory_exporter): + with pytest.raises(RuntimeError): + async with botanu.event(event_id="e", customer_id="c", workflow="W"): + raise RuntimeError("async-boom") + events = memory_exporter.get_finished_spans()[0].events + completed = next(e for e in events if e.name == "botanu.run.completed") + assert completed.attributes["status"] == "failure" + + +# ── Decorator form ─────────────────────────────────────────────────────── + +class TestEventDecorator: + def test_sync_decorator_creates_span(self, memory_exporter): + @botanu.event(event_id="e", customer_id="c", workflow="Dec") + def fn(): + return 42 + + assert fn() == 42 + spans = memory_exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == "botanu.run/Dec" + + def test_decorator_with_lambda_event_id_resolves_from_args(self, memory_exporter): + class Ticket: + def __init__(self, tid, uid): + self.id = tid + self.user_id = uid + + @botanu.event( + workflow="Support", + event_id=lambda t: t.id, + customer_id=lambda t: t.user_id, + ) + def handle(ticket): + return f"handled {ticket.id}" + + result = handle(Ticket("T-99", "user-7")) + assert result == "handled T-99" + attrs = _attrs(memory_exporter.get_finished_spans()) + assert attrs["botanu.event_id"] == "T-99" + assert attrs["botanu.customer_id"] == "user-7" + + @pytest.mark.asyncio + async def test_async_decorator_creates_span(self, memory_exporter): + @botanu.event(event_id="e", customer_id="c", workflow="AsyncDec") + async def fn(): + return "async" + + assert await fn() == "async" + spans = memory_exporter.get_finished_spans() + assert spans[0].name == "botanu.run/AsyncDec" + + def test_decorator_records_exception(self, memory_exporter): + @botanu.event(event_id="e", customer_id="c", workflow="W") + def bad(): + raise KeyError("nope") + + with pytest.raises(KeyError): + bad() + events = memory_exporter.get_finished_spans()[0].events + completed = next(e for e in events if e.name == "botanu.run.completed") + assert completed.attributes["status"] == "failure" + assert completed.attributes["error_class"] == "KeyError" + + def test_decorator_preserves_function_metadata(self, memory_exporter): + @botanu.event(event_id="e", customer_id="c", workflow="W") + def my_function(): + """docstring""" + return None + + assert my_function.__name__ == "my_function" + assert my_function.__doc__ == "docstring" + + +# ── Capture parity: CM form should match decorator form ────────────────── + +class TestEventCaptureParity: + def test_cm_and_decorator_produce_same_attrs(self, memory_exporter): + """Same inputs → same span attributes in either shape.""" + with botanu.event(event_id="e", customer_id="c", workflow="W"): + pass + cm_attrs = _attrs(memory_exporter.get_finished_spans()) + + memory_exporter.clear() + + @botanu.event(event_id="e", customer_id="c", workflow="W") + def fn(): + pass + fn() + dec_attrs = _attrs(memory_exporter.get_finished_spans()) + + for k in ("botanu.event_id", "botanu.customer_id", "botanu.workflow"): + assert cm_attrs[k] == dec_attrs[k] + + +# ── botanu.step() ──────────────────────────────────────────────────────── + +class TestStep: + def test_step_creates_nested_span(self, memory_exporter): + with botanu.event(event_id="e", customer_id="c", workflow="W"): + with botanu.step("retrieval"): + pass + spans = memory_exporter.get_finished_spans() + names = [s.name for s in spans] + assert "botanu.step/retrieval" in names + assert "botanu.run/W" in names + + def test_step_stamps_attribute(self, memory_exporter): + with botanu.event(event_id="e", customer_id="c", workflow="W"): + with botanu.step("generation"): + pass + step_span = next( + s for s in memory_exporter.get_finished_spans() + if s.name == "botanu.step/generation" + ) + assert dict(step_span.attributes)["botanu.step"] == "generation" + + def test_multiple_steps_within_one_event(self, memory_exporter): + with botanu.event(event_id="e", customer_id="c", workflow="W"): + with botanu.step("retrieval"): + pass + with botanu.step("generation"): + pass + step_names = [ + s.name for s in memory_exporter.get_finished_spans() + if s.name.startswith("botanu.step/") + ] + assert step_names == ["botanu.step/retrieval", "botanu.step/generation"] + + def test_step_records_exception(self, memory_exporter): + with pytest.raises(RuntimeError): + with botanu.event(event_id="e", customer_id="c", workflow="W"): + with botanu.step("failing"): + raise RuntimeError("step-boom") + step_span = next( + s for s in memory_exporter.get_finished_spans() + if s.name == "botanu.step/failing" + ) + from opentelemetry.trace import StatusCode + assert step_span.status.status_code == StatusCode.ERROR + + +# ── _Event internal ────────────────────────────────────────────────────── + +class TestEventInternalContract: + def test_event_factory_returns_event_instance(self): + ev = botanu.event(event_id="e", customer_id="c", workflow="W") + assert isinstance(ev, _Event) + + +class TestLazyAutoEnable: + """event() implicitly runs enable() on first call if the SDK isn't already + initialised. Customers don't need to remember a separate setup step.""" + + def test_event_calls_enable_when_not_initialised(self, monkeypatch): + from botanu.sdk import bootstrap as _bs + + calls = [] + monkeypatch.setattr(_bs, "_initialized", False) + + def spy_enable(*args, **kwargs): + calls.append((args, kwargs)) + # Mark initialised without doing real OTel setup (harness already did). + _bs._initialized = True + return True + + monkeypatch.setattr(_bs, "enable", spy_enable) + botanu.event(event_id="e", customer_id="c", workflow="W") + assert len(calls) == 1, "enable() should be auto-called on first event()" + + def test_event_skips_enable_when_already_initialised(self, monkeypatch): + from botanu.sdk import bootstrap as _bs + + calls = [] + monkeypatch.setattr(_bs, "_initialized", True) + monkeypatch.setattr(_bs, "enable", lambda *a, **k: calls.append((a, k)) or True) + botanu.event(event_id="e", customer_id="c", workflow="W") + assert calls == [], "enable() should NOT run when already initialised" + + +# ── End-to-end trace completeness ──────────────────────────────────────── +# +# The core product claim: "one wrap around the agent captures everything +# that happens inside." Verify this by installing a RunContextEnricher on +# the tracer provider, then creating a simulated downstream span (as if it +# were an auto-instrumented OpenAI call) INSIDE the event scope, and +# confirming the business attributes were stamped on it via the enricher +# reading baggage. This is the same path an openai/anthropic/httpx +# auto-instrumentor span takes in production. + +_enricher_installed = False + + +@pytest.fixture(autouse=True) +def _install_enricher(tracer_provider): + global _enricher_installed + if not _enricher_installed: + tracer_provider.add_span_processor(RunContextEnricher()) + # The test harness installs its own tracer provider + enricher already, + # so suppress the lazy auto-enable inside event() — otherwise it would + # try to install real OTel auto-instrumentation on top of the fixture. + from botanu.sdk import bootstrap as _bs + _bs._initialized = True + _enricher_installed = True + yield + + +class TestTraceCompleteness: + def test_downstream_span_inside_cm_gets_business_attrs(self, memory_exporter): + tracer = trace.get_tracer("simulated-auto-instrument") + with botanu.event(event_id="evt-xyz", customer_id="cust-xyz", workflow="Support"): + with tracer.start_as_current_span("simulated.llm.openai.chat") as child: + child.set_attribute("llm.request.model", "gpt-5.2") + + spans_by_name = {s.name: dict(s.attributes) for s in memory_exporter.get_finished_spans()} + assert "simulated.llm.openai.chat" in spans_by_name + llm_attrs = spans_by_name["simulated.llm.openai.chat"] + + # The four core business keys propagate via baggage in every mode. + assert llm_attrs.get("botanu.event_id") == "evt-xyz" + assert llm_attrs.get("botanu.customer_id") == "cust-xyz" + assert llm_attrs.get("botanu.workflow") == "Support" + assert "botanu.run_id" in llm_attrs + # The LLM's own attribute untouched + assert llm_attrs.get("llm.request.model") == "gpt-5.2" + + def test_downstream_span_inside_decorator_gets_business_attrs(self, memory_exporter): + tracer = trace.get_tracer("simulated-auto-instrument") + + @botanu.event(event_id="evt-dec", customer_id="cust-dec", workflow="W") + def handle(): + with tracer.start_as_current_span("simulated.db.query") as child: + child.set_attribute("db.system", "postgresql") + + handle() + attrs = next( + dict(s.attributes) + for s in memory_exporter.get_finished_spans() + if s.name == "simulated.db.query" + ) + assert attrs.get("botanu.event_id") == "evt-dec" + assert attrs.get("botanu.workflow") == "W" + + def test_downstream_spans_across_nested_steps_all_stamped(self, memory_exporter): + """Simulates a multi-phase agent: event → step → LLM span. Every + span in the trace (event root, step, LLM) should carry the event_id.""" + tracer = trace.get_tracer("simulated-auto-instrument") + + with botanu.event(event_id="evt-n", customer_id="cust-n", workflow="Support"): + with botanu.step("retrieval"): + with tracer.start_as_current_span("vector.search"): + pass + with botanu.step("generation"): + with tracer.start_as_current_span("openai.chat.completion"): + pass + + for s in memory_exporter.get_finished_spans(): + attrs = dict(s.attributes) + assert attrs.get("botanu.event_id") == "evt-n", f"{s.name} missing event_id" + assert attrs.get("botanu.workflow") == "Support", f"{s.name} missing workflow" + + def test_business_attrs_removed_outside_event_scope(self, memory_exporter): + """After leaving the event CM, baggage is detached — downstream spans + should NOT carry stale business attributes.""" + tracer = trace.get_tracer("simulated-auto-instrument") + + with botanu.event(event_id="evt-in", customer_id="cust-in", workflow="W"): + pass + + # Outside the CM — no baggage + with tracer.start_as_current_span("span.outside.event"): + pass + + outside = next( + dict(x.attributes) + for x in memory_exporter.get_finished_spans() + if x.name == "span.outside.event" + ) + assert "botanu.event_id" not in outside + assert "botanu.customer_id" not in outside diff --git a/tests/unit/test_llm_tracking.py b/tests/unit/test_llm_tracking.py index bbcb97c..65f3117 100644 --- a/tests/unit/test_llm_tracking.py +++ b/tests/unit/test_llm_tracking.py @@ -634,3 +634,64 @@ def test_returns_self_for_chaining(self, memory_exporter): with track_llm_call(model="gpt-4", vendor="openai") as tracker: result = tracker.set_input_content("hi").set_output_content("hello") assert result is tracker + + +class TestContentCapturePIIScrubbing: + """PII scrubbing runs between the sampling gate and set_attribute.""" + + def _with_config(self, **cfg_kwargs): + from contextlib import contextmanager + + from botanu.sdk import bootstrap + from botanu.sdk.config import BotanuConfig + from botanu.sdk.pii import _reset_cache_for_tests + + @contextmanager + def _cm(): + prev = bootstrap._current_config + cfg_kwargs.setdefault("content_capture_rate", 1.0) + bootstrap._current_config = BotanuConfig(**cfg_kwargs) + _reset_cache_for_tests() + try: + yield + finally: + bootstrap._current_config = prev + _reset_cache_for_tests() + + return _cm() + + def test_input_content_scrubbed_by_default(self, memory_exporter): + with self._with_config(): + with track_llm_call(model="gpt-4", vendor="openai") as tracker: + tracker.set_input_content("email me at alice@example.com thanks") + + attrs = dict(memory_exporter.get_finished_spans()[0].attributes) + captured = attrs["botanu.eval.input_content"] + assert "alice@example.com" not in captured + assert "[REDACTED]" in captured + + def test_output_content_scrubbed_by_default(self, memory_exporter): + with self._with_config(): + with track_llm_call(model="gpt-4", vendor="openai") as tracker: + tracker.set_output_content("your SSN 123-45-6789 is on file") + + attrs = dict(memory_exporter.get_finished_spans()[0].attributes) + assert "123-45-6789" not in attrs["botanu.eval.output_content"] + + def test_opt_out_preserves_text(self, memory_exporter): + with self._with_config(pii_scrub_enabled=False): + with track_llm_call(model="gpt-4", vendor="openai") as tracker: + tracker.set_input_content("email alice@example.com") + + attrs = dict(memory_exporter.get_finished_spans()[0].attributes) + assert attrs["botanu.eval.input_content"] == "email alice@example.com" + + def test_scrub_happens_before_truncation(self, memory_exporter): + """Scrub runs on full text so a long prefix doesn't bury PII past max_chars.""" + prefix = "x" * 4000 + with self._with_config(): + with track_llm_call(model="gpt-4", vendor="openai") as tracker: + tracker.set_input_content(prefix + " mail alice@example.com tail") + + attrs = dict(memory_exporter.get_finished_spans()[0].attributes) + assert "alice@example.com" not in attrs["botanu.eval.input_content"] diff --git a/tests/unit/test_pii.py b/tests/unit/test_pii.py new file mode 100644 index 0000000..b879338 --- /dev/null +++ b/tests/unit/test_pii.py @@ -0,0 +1,223 @@ +# SPDX-FileCopyrightText: 2026 The Botanu Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for botanu.sdk.pii — regex scrubber + Presidio opt-in path.""" + +from __future__ import annotations + +import importlib.util + +import pytest + +from botanu.sdk.config import BotanuConfig +from botanu.sdk.pii import PIIScrubber, _luhn_valid, _reset_cache_for_tests, apply_scrub + + +@pytest.fixture(autouse=True) +def _reset_cache(): + _reset_cache_for_tests() + yield + _reset_cache_for_tests() + + +class TestBuiltinPatterns: + def test_scrubs_email(self): + out = PIIScrubber().scrub("contact me at alice@example.com please") + assert "alice@example.com" not in out + assert "[REDACTED]" in out + + def test_scrubs_phone_e164(self): + out = PIIScrubber().scrub("call +14155551234 now") + assert "+14155551234" not in out + assert "[REDACTED]" in out + + def test_scrubs_phone_us(self): + out = PIIScrubber().scrub("call (415) 555-1234 or 415-555-1234") + assert "(415) 555-1234" not in out + assert "415-555-1234" not in out + + def test_scrubs_ssn(self): + out = PIIScrubber().scrub("SSN 123-45-6789 on file") + assert "123-45-6789" not in out + + def test_scrubs_credit_card_luhn_valid(self): + # 4111 1111 1111 1111 is the standard Visa test card — Luhn-valid. + out = PIIScrubber().scrub("card 4111 1111 1111 1111 expires soon") + assert "4111" not in out + assert "[REDACTED]" in out + + def test_luhn_rejects_non_card(self): + # 16-digit order ID that fails Luhn — must pass through untouched. + assert _luhn_valid("4111111111111112") is False + out = PIIScrubber().scrub("order id 4111111111111112 placed") + assert "4111111111111112" in out + + def test_scrubs_ipv4(self): + out = PIIScrubber().scrub("request from 192.168.1.100 failed") + assert "192.168.1.100" not in out + + def test_scrubs_ipv6(self): + out = PIIScrubber().scrub("peer 2001:0db8:85a3:0000:0000:8a2e:0370:7334 down") + assert "2001:0db8" not in out + + def test_scrubs_bearer_token(self): + out = PIIScrubber().scrub("Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.foo.bar") + assert "Bearer " not in out or "[REDACTED]" in out + + def test_scrubs_jwt(self): + out = PIIScrubber().scrub("token=eyJabc.eyJdef.XYZ123 next") + assert "eyJabc.eyJdef.XYZ123" not in out + + # Test secrets are assembled at runtime to avoid tripping source-side + # secret scanners (GitHub push-protection, gitleaks, etc.) that match + # on literal prefixes like ``sk_live_`` / ``ghp_`` / ``xoxb-``. + + def test_scrubs_aws_access_key(self): + fake = "AKIA" + "IOSFODNN7EXAMPLE" + out = PIIScrubber().scrub(f"AWS key {fake} detected") + assert fake not in out + + def test_scrubs_github_token(self): + fake = "ghp" + "_" + ("a" * 36) + out = PIIScrubber().scrub(f"export GH={fake} done") + assert "ghp" + "_" not in out + + def test_scrubs_stripe_key(self): + fake = "sk" + "_live_" + "abcdefghijklmnopqrstuvwxyz" + out = PIIScrubber().scrub(f"STRIPE={fake} used") + assert "sk" + "_live_" not in out + + def test_scrubs_slack_token(self): + fake = "xoxb" + "-12345-67890-abcdef" + out = PIIScrubber().scrub(f"token {fake} caught") + assert "xoxb" + "-" not in out + + def test_scrubs_openai_key(self): + fake = "sk" + "-abcdefghijklmnopqrstuv" + out = PIIScrubber().scrub(f"OPENAI={fake} leaked") + assert fake not in out + + def test_scrubs_anthropic_key(self): + fake = "sk" + "-ant-abcdefghijklmnopqrst123" + out = PIIScrubber().scrub(f"ANTHROPIC={fake} leaked") + assert "sk" + "-ant-" not in out + + +class TestConstructorOptions: + def test_empty_text_noop(self): + assert PIIScrubber().scrub("") == "" + + def test_none_like_noop(self): + assert PIIScrubber().scrub("") == "" + + def test_clean_text_unchanged(self): + text = "The quick brown fox jumps over the lazy dog" + assert PIIScrubber().scrub(text) == text + + def test_disable_specific_pattern(self): + scrubber = PIIScrubber(disabled_patterns=["ipv4"]) + out = scrubber.scrub("server 10.0.0.1 and email me@x.com") + assert "10.0.0.1" in out # ipv4 disabled + assert "me@x.com" not in out # email still scrubbed + + def test_enabled_patterns_allowlist(self): + scrubber = PIIScrubber(enabled_patterns=["email"]) + out = scrubber.scrub("email a@b.com and ip 10.0.0.1") + assert "a@b.com" not in out + assert "10.0.0.1" in out + + def test_custom_pattern(self): + scrubber = PIIScrubber(custom_patterns={"employee_id": r"EMP-\d{6}"}) + out = scrubber.scrub("assigned to EMP-123456 today") + assert "EMP-123456" not in out + + def test_custom_replacement_token(self): + scrubber = PIIScrubber(replacement="") + out = scrubber.scrub("mail me at a@b.com") + assert "" in out + assert "a@b.com" not in out + + def test_invalid_custom_regex_logged_not_raised(self, caplog): + # Malformed regex should not crash construction. + scrubber = PIIScrubber(custom_patterns={"bad": "(unclosed"}) + out = scrubber.scrub("a@b.com here") + assert "a@b.com" not in out # built-ins still work + + +class TestApplyScrubWithConfig: + def test_default_config_scrubs(self): + cfg = BotanuConfig() # default pii_scrub_enabled=True + out = apply_scrub("send to alice@acme.com", cfg) + assert "alice@acme.com" not in out + + def test_disabled_scrubber_noop(self): + cfg = BotanuConfig(pii_scrub_enabled=False) + text = "send to alice@acme.com" + assert apply_scrub(text, cfg) == text + + def test_empty_text_short_circuits(self): + cfg = BotanuConfig() + assert apply_scrub("", cfg) == "" + + def test_custom_replacement_roundtrips(self): + cfg = BotanuConfig(pii_scrub_replacement="") + out = apply_scrub("alice@acme.com here", cfg) + assert "" in out + + def test_custom_patterns_from_config(self): + cfg = BotanuConfig(pii_scrub_custom_patterns={"order": r"ORD-\d{4}"}) + out = apply_scrub("order ORD-1234 shipped to bob@x.com", cfg) + assert "ORD-1234" not in out + assert "bob@x.com" not in out + + def test_disable_patterns_from_config(self): + cfg = BotanuConfig(pii_scrub_disable_patterns=["email"]) + out = apply_scrub("ping alice@acme.com from 1.2.3.4", cfg) + assert "alice@acme.com" in out + assert "1.2.3.4" not in out + + def test_cache_rebuilds_on_config_change(self): + cfg1 = BotanuConfig(pii_scrub_enabled=True) + apply_scrub("alice@acme.com", cfg1) + cfg2 = BotanuConfig(pii_scrub_enabled=False) + assert apply_scrub("alice@acme.com", cfg2) == "alice@acme.com" + + +_presidio_available = importlib.util.find_spec("presidio_analyzer") is not None + + +class TestPresidioOptIn: + def test_opt_in_without_install_falls_back(self, caplog): + """pii_scrub_use_presidio=True without Presidio installed should not raise + and should still apply the regex pass.""" + if _presidio_available: + pytest.skip("presidio installed — this test exercises the missing-package path") + cfg = BotanuConfig(pii_scrub_use_presidio=True) + out = apply_scrub("ping alice@acme.com", cfg) + # regex pass still scrubs the email even though presidio is absent + assert "alice@acme.com" not in out + + @pytest.mark.skipif(not _presidio_available, reason="presidio not installed") + def test_scrubs_person_name_with_presidio(self): + from botanu.sdk.pii_presidio import _reset_for_tests, presidio_scrub + + _reset_for_tests() + out = presidio_scrub("meeting with John Smith tomorrow") + # "John Smith" is a PERSON entity — Presidio should redact at least one token. + assert "[REDACTED]" in out + + +class TestLuhnHelper: + @pytest.mark.parametrize( + "number, expected", + [ + ("4111111111111111", True), + ("4111 1111 1111 1111", True), + ("5500 0000 0000 0004", True), + ("4111111111111112", False), + ("1234", False), + ("12345678901234567890", False), + ], + ) + def test_luhn_valid(self, number, expected): + assert _luhn_valid(number) is expected diff --git a/tests/unit/test_run_context.py b/tests/unit/test_run_context.py index f20a68d..cd48a2b 100644 --- a/tests/unit/test_run_context.py +++ b/tests/unit/test_run_context.py @@ -150,36 +150,22 @@ def test_complete_sets_outcome(self): class TestRunContextSerialization: """Tests for baggage and span attribute serialization.""" - def test_to_baggage_dict_lean_mode(self): - with mock.patch.dict(os.environ, {"BOTANU_PROPAGATION_MODE": "lean"}): - ctx = RunContext.create( - workflow="Customer Support", - event_id="ticket-42", - customer_id="bigretail", - tenant_id="tenant-123", - ) - baggage = ctx.to_baggage_dict() - - # Lean mode includes run_id, workflow, event_id, customer_id - assert "botanu.run_id" in baggage - assert "botanu.workflow" in baggage - assert baggage["botanu.event_id"] == "ticket-42" - assert baggage["botanu.customer_id"] == "bigretail" - assert "botanu.tenant_id" not in baggage - - def test_to_baggage_dict_full_mode(self): - with mock.patch.dict(os.environ, {"BOTANU_PROPAGATION_MODE": "full"}): - ctx = RunContext.create( - workflow="Customer Support", - event_id="ticket-42", - customer_id="bigretail", - tenant_id="tenant-123", - ) - baggage = ctx.to_baggage_dict() - - assert baggage["botanu.event_id"] == "ticket-42" - assert baggage["botanu.customer_id"] == "bigretail" - assert baggage["botanu.tenant_id"] == "tenant-123" + def test_to_baggage_dict_includes_all_keys(self): + """to_baggage_dict always returns all context keys — there is no lean mode.""" + ctx = RunContext.create( + workflow="Customer Support", + event_id="ticket-42", + customer_id="bigretail", + tenant_id="tenant-123", + ) + baggage = ctx.to_baggage_dict() + + assert "botanu.run_id" in baggage + assert baggage["botanu.workflow"] == "Customer Support" + assert baggage["botanu.event_id"] == "ticket-42" + assert baggage["botanu.customer_id"] == "bigretail" + assert baggage["botanu.tenant_id"] == "tenant-123" + assert "botanu.environment" in baggage def test_to_span_attributes(self): ctx = RunContext.create( @@ -218,7 +204,7 @@ def test_from_baggage_roundtrip(self): customer_id="bigretail", tenant_id="tenant-abc", ) - baggage = original.to_baggage_dict(lean_mode=False) + baggage = original.to_baggage_dict() restored = RunContext.from_baggage(baggage) assert restored is not None diff --git a/tests/unit/test_span_helpers.py b/tests/unit/test_span_helpers.py index 85d48b1..a40169c 100644 --- a/tests/unit/test_span_helpers.py +++ b/tests/unit/test_span_helpers.py @@ -5,42 +5,36 @@ from __future__ import annotations -import pytest from opentelemetry import trace from botanu.sdk.span_helpers import emit_outcome, set_business_context, set_correlation class TestEmitOutcome: - """emit_outcome is deprecated as an outcome-status signal but retained for - diagnostic fields. Status is validated but no longer stamped on the span. + """emit_outcome stamps diagnostic fields only. Authoritative event outcome + is resolved server-side from SoR connectors / HITL / eval verdict rollup. """ def test_emit_outcome_does_not_stamp_status(self, memory_exporter): - """The status argument is validated but NOT emitted as - `botanu.outcome.status` — removed 2026-04-16.""" + """There is no `botanu.outcome.status` attribute at all.""" tracer = trace.get_tracer("test") - with pytest.warns(DeprecationWarning, match="trivially fakeable"): - with tracer.start_as_current_span("test-span"): - emit_outcome("success") + with tracer.start_as_current_span("test-span"): + emit_outcome(value_type="tickets_resolved", value_amount=1) spans = memory_exporter.get_finished_spans() attrs = dict(spans[0].attributes) assert "botanu.outcome.status" not in attrs def test_emit_outcome_emits_diagnostic_fields(self, memory_exporter): - """Diagnostic fields still stamp (reason, error_type, value_*, confidence).""" tracer = trace.get_tracer("test") - with pytest.warns(DeprecationWarning): - with tracer.start_as_current_span("test-span"): - emit_outcome( - "failed", - reason="timeout", - error_type="TimeoutError", - value_type="tickets_resolved", - value_amount=5.0, - confidence=0.95, - ) + with tracer.start_as_current_span("test-span"): + emit_outcome( + reason="timeout", + error_type="TimeoutError", + value_type="tickets_resolved", + value_amount=5.0, + confidence=0.95, + ) spans = memory_exporter.get_finished_spans() attrs = dict(spans[0].attributes) @@ -49,48 +43,19 @@ def test_emit_outcome_emits_diagnostic_fields(self, memory_exporter): assert attrs.get("botanu.outcome.value_type") == "tickets_resolved" assert attrs.get("botanu.outcome.value_amount") == 5.0 assert attrs.get("botanu.outcome.confidence") == 0.95 - # Still NOT stamping status - assert "botanu.outcome.status" not in attrs - - def test_emit_outcome_raises_on_invalid_status(self, memory_exporter): - """Status validation retained for backward compatibility.""" - tracer = trace.get_tracer("test") - with tracer.start_as_current_span("test-span"): - with pytest.raises(ValueError, match="Invalid outcome status"): - emit_outcome("not_a_real_status") def test_emit_outcome_event_no_status_attr(self, memory_exporter): - """The `botanu.outcome_emitted` span event still fires for diagnostics - but does NOT carry `status` in its attributes.""" + """The `botanu.outcome_emitted` span event fires for diagnostics and + does NOT carry `status` in its attributes.""" tracer = trace.get_tracer("test") - with pytest.warns(DeprecationWarning): - with tracer.start_as_current_span("test-span"): - emit_outcome("success", value_type="orders", value_amount=1) + with tracer.start_as_current_span("test-span"): + emit_outcome(value_type="orders", value_amount=1) spans = memory_exporter.get_finished_spans() events = [e for e in spans[0].events if e.name == "botanu.outcome_emitted"] assert len(events) == 1 assert "status" not in dict(events[0].attributes) - def test_emit_outcome_no_log_record(self, memory_exporter, log_exporter): - """The OTel log record path has been removed — no collector flush - trigger from emit_outcome any more (customer-push outcome deprecated).""" - from opentelemetry import baggage, context - - tracer = trace.get_tracer("test") - ctx = baggage.set_baggage("botanu.event_id", "ticket-42", context=context.Context()) - token = context.attach(ctx) - try: - with pytest.warns(DeprecationWarning): - with tracer.start_as_current_span("test-span"): - emit_outcome("success") - finally: - context.detach(token) - - logs = log_exporter.get_finished_logs() - # Event_id is set but the log emission is gone - assert len(logs) == 0 - class TestSetBusinessContext: """Tests for set_business_context function.""" From 228f636dc40abc6759e543604f68275e2d768daa Mon Sep 17 00:00:00 2001 From: Deborah Jacob Date: Thu, 23 Apr 2026 19:00:36 -0700 Subject: [PATCH 3/3] docs: README, CHANGELOG, RELEASE + doc sweep - README: rewrite quickstart around botanu.event(), drop enable() mention - CHANGELOG: add Unreleased "Breaking" section for the event/step pivot and lean-mode removal; rewrite "Added" around the new API - RELEASE: fix smoke-test script to use botanu.event() - api/configuration.md: drop api_key field claim, fix to_baggage_dict description (5 always + 7 conditional, not "always 7") - api/event.md: drop auto-enable note - getting-started/configuration.md: drop enable() code examples, drop api_key field from BotanuConfig listing, drop "it calls enable()" note under zero-code init section - getting-started/quickstart.md: drop enable() mention - index.md: drop enable() mention - integration/auto-instrumentation.md: reframe around first event() call - integration/collector.md: drop enable() code example - integration/existing-otel.md: reframe the whole page around "on first botanu.event() call" instead of "when you call enable()"; drop brownfield/greenfield terminology for customer-facing copy - patterns/anti-patterns.md: remove "Calling enable() repeatedly" anti-pattern section - tracking/content-capture.md: drop enable() code example --- CHANGELOG.md | 67 ++----- README.md | 120 +++++++----- RELEASE.md | 5 +- docs/api/configuration.md | 105 +--------- docs/api/event.md | 4 - docs/getting-started/configuration.md | 35 +--- docs/getting-started/quickstart.md | 2 +- docs/index.md | 2 +- docs/integration/auto-instrumentation.md | 13 +- docs/integration/collector.md | 12 +- docs/integration/existing-otel.md | 238 ++++++----------------- docs/patterns/anti-patterns.md | 17 -- docs/tracking/content-capture.md | 11 +- 13 files changed, 180 insertions(+), 451 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0e6c57..f888b0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,61 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Added - -- **Security** - - OTLP bearer token is attached only when the endpoint host is botanu-owned - (`*.botanu.ai`) or a local dev host, preventing tenant API-key leakage - via a customer-supplied `OTEL_EXPORTER_OTLP_ENDPOINT`. - - Authorization / `x-api-key` / `botanu-api-key` headers and `user:pass@` - URL credentials are redacted in logs. -- **Brownfield OTel coexistence** - - `SampledSpanProcessor` preserves the host app's existing TracerProvider - sampling ratio when botanu is bootstrapped into a project that already - has OTel wired up. - - `register.py` entry point for explicit opt-in without decorator-side - provider mutation. - - Bootstrap detects a pre-configured provider and hands off instead of - overriding it. -- **Content capture for eval** - - Workflow-level input/output capture gated by `content_capture_rate` - config and a shared `content_sampler`. Writes - `botanu.eval.input_content` / `botanu.eval.output_content`. - - `set_input_content()` / `set_output_content()` on `LLMTracker` with the - same gate, plus matching helpers on data-tracking helpers. -- **Multi-step workflows** - - `@botanu_workflow(..., step=...)` parameter (stored in `RunContext`, - not yet emitted to span attributes — kept backward compatible until the - collector servicegraph work lands). -- **Resources** - - `ResourceEnricher` span processor for deployment attributes. -- **Release tooling** - - `scripts/pre_publish_check.py` red/green gate: builds sdist + wheel, - runs `twine check`, installs into a fresh venv, validates the public - API surface, runs an end-to-end decorator + `emit_outcome` smoke test. +### Breaking -### Fixed +- Primary API is now `botanu.event(...)` — works as context manager, async context manager, and decorator. The legacy `@botanu_workflow`, `workflow` alias, `run_botanu`, and `@botanu_outcome` decorators are removed. +- `emit_outcome` is keyword-only and no longer accepts a `status` argument. Authoritative event outcome is resolved server-side from SoR connectors, HITL reviews, or eval verdict rollup. +- Lean baggage propagation is removed. All seven baggage keys (plus any retry/deadline keys when set) always propagate. The `BOTANU_PROPAGATION_MODE` env var, the `propagation_mode` field on `BotanuConfig`, `BAGGAGE_KEYS_LEAN`, and the `lean_mode` parameter on `RunContextEnricher` / `RunContext.to_baggage_dict` are all gone. -- `SampledSpanProcessor.on_start` now gates on the same ratio decision as - `on_end`; forwarding `on_start` unconditionally while gating `on_end` - leaked span bookkeeping inside wrapped exporters (QUAL-C1). +### Added -### Initial release contents +- `botanu.event(...)` as context manager, async context manager, and decorator — a single API for marking business events. +- `botanu.step(name)` for nested phase spans inside an event. +- SDK initialises automatically on the first `botanu.event(...)` call. Customers no longer need to call any bootstrap function by hand. +- `botanu.set_correlation(**keys)` for SoR Tier-1 correlation. +- Security: OTLP bearer token is attached only when the endpoint host is botanu-owned (`*.botanu.ai`) or a local dev host, preventing tenant API-key leakage via a customer-supplied `OTEL_EXPORTER_OTLP_ENDPOINT`. Authorization / `x-api-key` / `botanu-api-key` headers and `user:pass@` URL credentials are redacted in logs. +- OTel coexistence: when the host app already has the OTel SDK wired up, botanu preserves the existing sampling ratio and adds itself alongside. `register.py` module for zero-code initialisation in containers / gunicorn / process runners. +- Content capture for eval, gated by `content_capture_rate`. Writes `botanu.eval.input_content` / `botanu.eval.output_content` with in-process PII scrub (regex by default; optional Microsoft Presidio via `pip install botanu[pii-nlp]`). +- `ResourceEnricher` span processor for deployment attributes. +- Release tooling: `scripts/pre_publish_check.py` — builds sdist + wheel, runs `twine check`, installs into a fresh venv, validates the public API surface, runs an end-to-end smoke test. -Carried forward from the pre-tag scaffolding (never published): +### Fixed -- `enable()` / `disable()` bootstrap, `@botanu_workflow`, - `@botanu_outcome`, `emit_outcome()`, `set_business_context()`, - `RunContextEnricher` — with UUIDv7 run_ids. -- LLM tracking aligned with OTel GenAI semconv: `track_llm_call()`, - `track_tool_call()`, token accounting, 15+ provider normalization. -- Data tracking: `track_db_operation()`, `track_storage_operation()`, - `track_messaging_operation()`; 30+ system normalizations. -- W3C Baggage propagation with `RunContext` (retry tracking + deadline). -- Cloud resource detectors via optional extras (`aws`, `gcp`, `azure`, - `container`, `cloud`). -- Auto-instrumentation bundled in the base install — HTTP clients, web - frameworks, databases, messaging, and GenAI providers; instrumentation - packages no-op when their target library is not installed. +- `SampledSpanProcessor.on_start` now gates on the same ratio decision as `on_end`; forwarding `on_start` unconditionally while gating `on_end` leaked span bookkeeping inside wrapped exporters. [Unreleased]: https://github.com/botanu-ai/botanu-sdk-python/commits/main diff --git a/README.md b/README.md index d2833d7..bd68aed 100644 --- a/README.md +++ b/README.md @@ -2,104 +2,120 @@ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](./LICENSE) - Event-level cost attribution for AI workflows, built on [OpenTelemetry](https://opentelemetry.io/). +An **event** is one business transaction — resolving a support ticket, processing an order, generating a report. Each event may involve multiple **runs** (LLM calls, retries, sub-workflows) across multiple services. By correlating every run to a stable `event_id`, Botanu gives you per-event cost attribution and outcome tracking without sampling artefacts. + +## Install +```bash +pip install botanu +``` -An **event** is one business transaction — resolving a support ticket, processing -an order, generating a report. Each event may involve multiple **runs** (LLM calls, -retries, sub-workflows) across multiple services. By correlating every run to a -stable `event_id`, Botanu gives you per-event cost attribution and outcome -tracking without sampling artifacts. +One install. Includes the OTel SDK, the OTLP exporter, and auto-instrumentation for 50+ libraries (OpenAI, Anthropic, Vertex, LangChain, httpx, requests, SQLAlchemy, psycopg2, Redis, Celery, Kafka, boto3, and more). -## Getting Started +## Quickstart + +Set your API key: ```bash -pip install botanu +export BOTANU_API_KEY= ``` -One install. Includes OTel SDK, OTLP exporter, and auto-instrumentation for -50+ libraries. +Wrap your agent: ```python -from botanu import enable, botanu_workflow, emit_outcome +import botanu + +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + agent.run(ticket) +``` -enable() # reads config from environment variables +That single wrap captures every LLM call, HTTP call, and DB call inside and stamps them with `event_id`, `customer_id`, and `workflow`. -@botanu_workflow("my-workflow", event_id="evt-001", customer_id="cust-42") -async def do_work(): - result = await do_something() - emit_outcome("success") - return result +### Decorator form + +```python +import botanu + +@botanu.event( + workflow="Support", + event_id=lambda ticket: ticket.id, + customer_id=lambda ticket: ticket.user_id, +) +def handle_ticket(ticket): + return agent.run(ticket) ``` -Entry points use `@botanu_workflow`. Every other service only needs `enable()`. -All configuration is via environment variables — zero hardcoded values in code. +Works for both sync and `async def` functions. + +### Multi-phase workflows + +```python +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + with botanu.step("retrieval"): + docs = vector_db.query(ticket.query) + with botanu.step("generation"): + response = llm.complete(docs) +``` -See the [Quick Start](./docs/getting-started/quickstart.md) guide for a full walkthrough. +See the [Quickstart](./docs/getting-started/quickstart.md) for the full five-minute walkthrough. ## Documentation -| Topic | Description | -|-------|-------------| -| [Installation](./docs/getting-started/installation.md) | Install and configure the SDK | -| [Quick Start](./docs/getting-started/quickstart.md) | Get up and running in 5 minutes | -| [Configuration](./docs/getting-started/configuration.md) | Environment variables and options | -| [Core Concepts](./docs/concepts/) | Events, runs, context propagation, architecture | -| [LLM Tracking](./docs/tracking/llm-tracking.md) | Track model calls and token usage | -| [Data Tracking](./docs/tracking/data-tracking.md) | Database, storage, and messaging | -| [Outcomes](./docs/tracking/outcomes.md) | Record business outcomes for ROI | -| [Auto-Instrumentation](./docs/integration/auto-instrumentation.md) | Supported libraries and frameworks | +| Topic | | +| --- | --- | +| [Installation](./docs/getting-started/installation.md) | Install and configure | +| [Quickstart](./docs/getting-started/quickstart.md) | Zero-to-first-trace in five minutes | +| [Configuration](./docs/getting-started/configuration.md) | Env vars, YAML, trusted-host auth | +| [Run Context](./docs/concepts/run-context.md) | Events, runs, retries, baggage | +| [Context Propagation](./docs/concepts/context-propagation.md) | Cross-service and queue propagation | +| [Architecture](./docs/concepts/architecture.md) | SDK + collector split | +| [LLM Tracking](./docs/tracking/llm-tracking.md) | Manual LLM instrumentation (usually not needed) | +| [Data Tracking](./docs/tracking/data-tracking.md) | DB, storage, messaging (usually not needed) | +| [Content Capture](./docs/tracking/content-capture.md) | Prompt/response capture for eval, with PII scrubbing | +| [Outcomes](./docs/tracking/outcomes.md) | Diagnostic annotations and server-side resolution | +| [Auto-Instrumentation](./docs/integration/auto-instrumentation.md) | Supported libraries | | [Kubernetes](./docs/integration/kubernetes.md) | Zero-code instrumentation at scale | -| [API Reference](./docs/api/) | Decorators, tracking API, configuration | -| [Best Practices](./docs/patterns/best-practices.md) | Recommended patterns | +| [Existing OTel / Datadog](./docs/integration/existing-otel.md) | Brownfield coexistence | +| [`event` / `step` API](./docs/api/event.md) | Primary API reference | +| [Best Practices](./docs/patterns/best-practices.md) | Patterns that work | +| [Anti-Patterns](./docs/patterns/anti-patterns.md) | Patterns that break cost attribution | ## Requirements -- Python 3.9+ -- OpenTelemetry Collector (recommended for production) +- Python 3.9 or newer +- An OpenTelemetry Collector (Botanu Cloud runs one for you; self-hosted is supported too) ## Contributing -We welcome contributions from the community. Please read our -[Contributing Guide](./CONTRIBUTING.md) before submitting a pull request. +Contributions are welcome. Read the [Contributing Guide](./CONTRIBUTING.md) before opening a pull request. -This project requires [DCO sign-off](https://developercertificate.org/) on all -commits: +All commits require [DCO sign-off](https://developercertificate.org/): ```bash git commit -s -m "Your commit message" ``` -Looking for a place to start? Check the -[good first issues](https://github.com/botanu-ai/botanu-sdk-python/labels/good%20first%20issue). +Looking for a place to start? See the [good first issues](https://github.com/botanu-ai/botanu-sdk-python/labels/good%20first%20issue). ## Community - [GitHub Discussions](https://github.com/botanu-ai/botanu-sdk-python/discussions) — questions, ideas, show & tell -- [GitHub Issues](https://github.com/botanu-ai/botanu-sdk-python/issues) — bug reports and feature requests +- [GitHub Issues](https://github.com/botanu-ai/botanu-sdk-python/issues) — bugs and feature requests ## Governance -See [GOVERNANCE.md](./GOVERNANCE.md) for details on roles, decision-making, -and the contributor ladder. - -Current maintainers are listed in [MAINTAINERS.md](./MAINTAINERS.md). +See [GOVERNANCE.md](./GOVERNANCE.md) for roles, decision-making, and the contributor ladder. Current maintainers are in [MAINTAINERS.md](./MAINTAINERS.md). ## Security -To report a security vulnerability, please use -[GitHub Security Advisories](https://github.com/botanu-ai/botanu-sdk-python/security/advisories/new) -or see [SECURITY.md](./SECURITY.md) for full details. **Do not file a public issue.** +Report security vulnerabilities via [GitHub Security Advisories](https://github.com/botanu-ai/botanu-sdk-python/security/advisories/new) or see [SECURITY.md](./SECURITY.md). **Do not file a public issue.** ## Code of Conduct -This project follows the -[LF Projects Code of Conduct](https://lfprojects.org/policies/code-of-conduct/). -See [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md). +This project follows the [LF Projects Code of Conduct](https://lfprojects.org/policies/code-of-conduct/). See [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md). ## License [Apache License 2.0](./LICENSE) - diff --git a/RELEASE.md b/RELEASE.md index 0686052..dda2174 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -94,8 +94,9 @@ python -c "import botanu; print(botanu.__version__)" # Run quick test python -c " -from botanu import enable, botanu_workflow -enable(service_name='test') +import botanu +with botanu.event(event_id='smoke-1', customer_id='smoke-cust', workflow='smoke-test'): + pass print('Botanu SDK loaded successfully!') " ``` diff --git a/docs/api/configuration.md b/docs/api/configuration.md index 27c83a6..963c9e5 100644 --- a/docs/api/configuration.md +++ b/docs/api/configuration.md @@ -173,114 +173,27 @@ Syntax: --- -## enable() +## `disable()` -Bootstrap function to initialise the SDK. +Flush pending spans and shut down the SDK cleanly. Typically called at application shutdown. ```python -from botanu import enable +import botanu -enable( - service_name: Optional[str] = None, - otlp_endpoint: Optional[str] = None, - environment: Optional[str] = None, - auto_instrumentation: bool = True, - propagators: Optional[List[str]] = None, - log_level: str = "INFO", - config: Optional[BotanuConfig] = None, - config_file: Optional[str] = None, -) -> bool -``` - -### Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `service_name` | `str` | From env | Service name | -| `otlp_endpoint` | `str` | From env | OTLP endpoint URL | -| `environment` | `str` | From env | Deployment environment | -| `auto_instrumentation` | `bool` | `True` | Enable auto-instrumentation | -| `propagators` | `list[str]` | `["tracecontext", "baggage"]` | Propagator list | -| `log_level` | `str` | `"INFO"` | Logging level | -| `config` | `BotanuConfig` | `None` | Pre-built configuration (overrides individual params) | -| `config_file` | `str` | `None` | Path to YAML config file | - -### Returns - -`True` if successfully initialised, `False` if already initialised. - -### Behaviour - -1. Creates/merges `BotanuConfig` -2. Configures `TracerProvider` with `RunContextEnricher` -3. Sets up OTLP exporter -4. Enables auto-instrumentation (if requested) -5. Configures W3C Baggage propagation - -### Examples - -#### Minimal - -```python -from botanu import enable - -enable(service_name="my-service") -``` - -#### With Config Object - -```python -from botanu import enable -from botanu.sdk.config import BotanuConfig - -config = BotanuConfig.from_yaml("config/botanu.yaml") -enable(config=config) -``` - -#### From environment only - -```python -from botanu import enable - -# Reads OTEL_SERVICE_NAME, OTEL_EXPORTER_OTLP_ENDPOINT, etc. -enable() +botanu.disable() ``` --- -## disable() +## `is_enabled()` -Disable the SDK and clean up resources. +Check if the SDK has been initialised. ```python -from botanu import disable +import botanu -disable() -> None -``` - -### Behaviour - -1. Flushes pending spans -2. Shuts down span processors -3. Disables instrumentation - ---- - -## is_enabled() - -Check if the SDK is currently enabled. - -```python -from botanu import is_enabled - -is_enabled() -> bool -``` - -### Example - -```python -if not is_enabled(): - enable(service_name="my-service") +if botanu.is_enabled(): + ... ``` --- diff --git a/docs/api/event.md b/docs/api/event.md index fc70dfb..898bc0e 100644 --- a/docs/api/event.md +++ b/docs/api/event.md @@ -77,10 +77,6 @@ with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): Each step stamps `botanu.step=` on its span and propagates it via baggage. -## Auto-enable - -`event()` calls `botanu.enable()` implicitly on first use. Explicit `botanu.enable(...)` is only needed to override config (custom endpoint, API key, content-capture rate). - ## See also - [Run Context](../concepts/run-context.md) diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index ffe2968..b64fbcb 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -24,21 +24,9 @@ For any other endpoint, the exporter ships without a bearer header. If your self ## Configuration precedence -1. Code arguments to `enable()` or `BotanuConfig(...)` -2. Environment variables (`BOTANU_*`, `OTEL_*`) -3. YAML config file (`botanu.yaml` or a path you pass) -4. Built-in defaults - -## Code - -```python -import botanu - -botanu.enable( - service_name="my-service", - otlp_endpoint="https://ingest.botanu.ai", -) -``` +1. Environment variables (`BOTANU_*`, `OTEL_*`) +2. YAML config file (`botanu.yaml` or a path you pass) +3. Built-in defaults ## Environment variables @@ -139,8 +127,6 @@ If you can't edit the entry point (third-party process runner, gunicorn preload) python -c "import botanu.register" -m your_app ``` -It calls `enable()` under the hood. - ## Auto-instrumentation ### Default packages @@ -159,19 +145,18 @@ It calls `enable()` under the hood. ### Customizing -```python -import botanu -from botanu.sdk.config import BotanuConfig +Set `BOTANU_AUTO_INSTRUMENT_PACKAGES` (comma-separated) or put the list in `botanu.yaml`: -config = BotanuConfig(auto_instrument_packages=["requests", "fastapi", "openai_v2"]) -botanu.enable(config=config) +```yaml +auto_instrument_packages: + - requests + - fastapi + - openai_v2 ``` ### Disabling -```python -botanu.enable(auto_instrumentation=False) -``` +Set `BOTANU_AUTO_INSTRUMENTATION=false` to skip auto-instrumentation entirely. Only do this if you've manually instrumented every library you care about. ## See also diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 8ba0742..a511cfb 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -37,7 +37,7 @@ with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): agent.run(ticket) ``` -Every LLM call, HTTP call, and database query inside the block is captured and stamped with `event_id`, `customer_id`, and `workflow`. There's no separate `enable()` call — the SDK initializes itself on the first `event` call. +Every LLM call, HTTP call, and database query inside the block is captured and stamped with `event_id`, `customer_id`, and `workflow`. ## Decorator form diff --git a/docs/index.md b/docs/index.md index 15652fe..ce116c8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,7 +17,7 @@ with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): agent.run(ticket) ``` -That single wrap captures every LLM call, HTTP call, and DB call the agent makes, and ties them to `event_id`. Authentication and endpoint setup come from `BOTANU_API_KEY` in the environment — no explicit `enable()` needed. +That single wrap captures every LLM call, HTTP call, and DB call the agent makes, and ties them to `event_id`. Authentication and endpoint setup come from `BOTANU_API_KEY` in the environment. See [Quick Start](getting-started/quickstart.md) for the full five-minute walkthrough. diff --git a/docs/integration/auto-instrumentation.md b/docs/integration/auto-instrumentation.md index be504bc..98e28ef 100644 --- a/docs/integration/auto-instrumentation.md +++ b/docs/integration/auto-instrumentation.md @@ -1,18 +1,19 @@ # Auto-Instrumentation -Botanu automatically instruments 50+ libraries with zero code changes. +botanu automatically instruments 50+ libraries with zero code changes. -## How It Works +## How it works -When you call `enable()`, the SDK detects which libraries are installed in your environment and instruments them automatically. Libraries that aren't installed are silently skipped. +On first `botanu.event(...)` call, the SDK detects which libraries are installed in your environment and instruments them automatically. Libraries that aren't installed are silently skipped. ```python -from botanu import enable +import botanu -enable() # auto-instruments everything that's installed +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + agent.run(ticket) ``` -No configuration needed. No import order requirements. Just call `enable()` at startup. +No configuration needed. No import order requirements. ## Supported Libraries diff --git a/docs/integration/collector.md b/docs/integration/collector.md index 2154555..3ecc063 100644 --- a/docs/integration/collector.md +++ b/docs/integration/collector.md @@ -30,22 +30,12 @@ No collector configuration is needed on your side. Just set the API key: export BOTANU_API_KEY= ``` -```python -from botanu import enable - -enable() # reads BOTANU_API_KEY from env -``` +The SDK reads this at initialisation and routes OTLP exports to `https://ingest.botanu.ai`. ### Override endpoint (advanced) For development or testing against a local collector: -```python -enable(otlp_endpoint="http://localhost:4318") -``` - -Or via environment variable: - ```bash export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 ``` diff --git a/docs/integration/existing-otel.md b/docs/integration/existing-otel.md index e74f49f..9d7373c 100644 --- a/docs/integration/existing-otel.md +++ b/docs/integration/existing-otel.md @@ -1,176 +1,104 @@ # Using botanu with an existing OTel / APM setup -botanu is designed to sit alongside an OTel / Datadog / Jaeger / Honeycomb / -New Relic setup you already have. You do not have to migrate off your current -APM — you call `enable()`, and botanu detects what is already configured and -adds itself without stealing spans or changing your sampled volume. - -This page explains exactly what `enable()` does in each of the three -configurations it detects, so you can reason about the outcome before you -install. +botanu is designed to sit alongside an OTel / Datadog / Jaeger / Honeycomb / New Relic setup you already have. You do not have to migrate off your current APM — on first use, the SDK detects what is already configured and adds itself without stealing spans or changing your sampled volume. ## TL;DR -- **Already on the OTel SDK?** `enable()` keeps your span processors, preserves - your sampling ratio for them, and adds the botanu exporter at 100%. - Your existing APM bill does not change. -- **Already on ddtrace (Datadog's Python SDK)?** `enable()` creates a - separate, parallel TracerProvider for botanu. ddtrace is untouched. -- **No existing tracing?** `enable()` creates a fresh provider and wires - everything up for you. +- **Already on the OTel SDK?** botanu keeps your span processors, preserves your sampling ratio for them, and adds the botanu exporter at 100%. Your existing APM bill does not change. +- **Already on ddtrace (Datadog's Python SDK)?** botanu runs on a separate, parallel TracerProvider. ddtrace is untouched. +- **No existing tracing?** botanu creates a fresh provider and wires everything up. -In all three cases the SDK is a single call: +Integration is a single line per service: ```python -from botanu import enable -enable() # reads config from env; no hard-coded values +import botanu + +with botanu.event(event_id=ticket.id, customer_id=user.id, workflow="Support"): + agent.run(ticket) ``` --- -## Detection — what `enable()` checks +## What botanu does on first use -When you call `enable()`, the SDK calls `trace.get_tracer_provider()` and -branches on what it finds. The logic lives in -[`src/botanu/sdk/bootstrap.py`](../../src/botanu/sdk/bootstrap.py). +On the first `botanu.event(...)` call, the SDK calls `trace.get_tracer_provider()` and branches on what it finds. The logic lives in [`src/botanu/sdk/bootstrap.py`](../../src/botanu/sdk/bootstrap.py). | What `get_tracer_provider()` returns | What botanu does | | --- | --- | -| `opentelemetry.sdk.trace.TracerProvider` (the real OTel SDK class) | Treats as **brownfield OTel**. Creates a new provider, migrates your span processors, wraps ratio-sampled processors in `SampledSpanProcessor`, adds botanu alongside, swaps the global provider. | -| `opentelemetry.trace.ProxyTracerProvider` (no real provider set) | Treats as **greenfield**. Creates a fresh provider with `ALWAYS_ON` sampling. | -| Anything else (e.g., `ddtrace.opentelemetry.TracerProvider`) | Treats as **unknown / parallel**. Creates a separate TracerProvider for botanu. Your existing tracer is untouched. | +| `opentelemetry.sdk.trace.TracerProvider` (the real OTel SDK class) | Creates a new provider, migrates your span processors, wraps ratio-sampled processors in `SampledSpanProcessor`, adds botanu alongside, swaps the global provider. | +| `opentelemetry.trace.ProxyTracerProvider` (no real provider set) | Creates a fresh provider with `ALWAYS_ON` sampling. | +| Anything else (e.g., `ddtrace.opentelemetry.TracerProvider`) | Creates a separate TracerProvider for botanu. Your existing tracer is untouched. | -botanu never mutates your existing provider in place. It either creates a -new one and swaps the global, or leaves yours entirely alone. +botanu never mutates your existing provider in place. It either creates a new one and swaps the global, or leaves yours entirely alone. --- -## Brownfield: existing OTel SDK +## If you already run the OTel SDK ### What botanu does -1. Reads the sampling ratio off your provider using - `_extract_sampler_ratio()`. This recognises `AlwaysOn`, `AlwaysOff`, - `TraceIdRatioBased`, and `ParentBased(...)` wrappers around those. -2. Collects the list of processors already attached to your provider. -3. Creates a **new** `TracerProvider` with `ALWAYS_ON`, keeping your - `Resource`. -4. For each of your existing processors: - - If your ratio was `< 1.0`, wraps the processor in a - `SampledSpanProcessor(proc, original_ratio)` so it continues to see - only the fraction of spans it used to. - - Otherwise attaches it as-is. -5. Adds `RunContextEnricher` (run_id / workflow / event_id baggage → span - attributes), `ResourceEnricher`, and botanu's own `BatchSpanProcessor` - to the new provider — **unwrapped**, so botanu sees 100%. -6. `trace.set_tracer_provider(new_provider)` — this becomes the global. -7. Logs one of: - - ```text - Botanu SDK: existing TracerProvider detected with 10% sampling. - Preserved your sampling ratio for existing exporters. - botanu captures 100%. No impact on your existing observability bill. - ``` - - or, for 100% samplers: - - ```text - Botanu SDK: existing TracerProvider detected. - Added botanu exporter alongside your existing setup. - ``` - -### Why flip to AlwaysOn internally? - -botanu needs 100% of spans to produce accurate cost attribution — you can't -extrapolate token counts or per-span costs from a 10% sample without -distortion. So the new provider is `ALWAYS_ON`. The resulting diagram: - -```text -App (sampler = AlwaysOn → every span created) - │ - ├─ SampledSpanProcessor(0.10) → your Datadog BatchSpanProcessor → Datadog (sees 10%) - │ ↑ same volume as before - │ - └─ botanu BatchSpanProcessor → botanu collector (sees 100%) -``` +1. Detects your existing `TracerProvider`. +2. Reads its sampler. If it's a ratio sampler (e.g., `TraceIdRatioBased(0.1)`), botanu records the ratio. +3. Creates a new provider with `ALWAYS_ON` sampling. +4. Migrates your existing span processors to the new provider. +5. Wraps each migrated processor in `SampledSpanProcessor(ratio)` so your exporters still see their original sampled subset. +6. Adds botanu's own exporter at 100% alongside. +7. Swaps the global provider. -`SampledSpanProcessor` is deterministic on `trace_id`, matching OTel's -`TraceIdRatioBasedSampler` algorithm — the same trace always gets the same -decision, so a trace that hits Datadog also hits 100% of botanu spans and -nothing is orphaned. +### Result -### Unknown sampler — safety path +- Your existing APM keeps getting the same sampled volume it always did. Your bill does not change. +- botanu captures 100% of spans for cost attribution. +- Every span — yours and auto-instrumented — carries the run context from [W3C Baggage](https://www.w3.org/TR/baggage/). -If `_extract_sampler_ratio()` cannot identify your sampler (custom -subclass, third-party library), botanu **does not assume 100%**. Instead it: +### Sampling preservation -1. Logs a warning with your sampler's class name. -2. Creates the new provider with your **original sampler** preserved. -3. Attaches your processors unwrapped — they see what they saw before. -4. Attaches botanu's processors also under your original sampler — meaning - botanu will see only the sampled subset too, not 100%. +| Your sampler | botanu sees | Your APM sees | +| --- | --- | --- | +| `AlwaysOn` | 100% | 100% | +| `TraceIdRatioBased(0.1)` | 100% | 10% | +| `ParentBased(TraceIdRatioBased(0.1))` | 100% | 10% | +| `AlwaysOff` | 100% | 0% | +| A custom sampler botanu can't introspect | the sampled subset only | unchanged | -This is deliberate. Silently defaulting an unknown sampler to 1.0 would -inflate your existing exporter's volume 10× or 100× and potentially blow up -your observability bill. The cost of the unknown path is that botanu's cost -numbers are computed on the sampled subset; accept that or migrate your -sampler to a known one (`AlwaysOn` / `TraceIdRatioBased` / -`ParentBased(TraceIdRatioBased(...))`). +The custom-sampler case is the one edge we can't preserve exactly — botanu logs a warning (`could not identify the sampling ratio of X`) and falls back to using your sampler. Cost attribution is still correct, just computed on fewer spans. --- -## Parallel: ddtrace +## If you already run ddtrace -`ddtrace` (Datadog's Python SDK) installs its own tracer that implements -the OTel API but extends a different base class than the OTel SDK. botanu -detects this via `isinstance(existing, TracerProvider)` returning `False` -and falls through to the parallel path. +`ddtrace` (Datadog's Python SDK) installs its own tracer that implements the OTel API but extends a different base class than the OTel SDK. botanu detects this via `isinstance(existing, TracerProvider)` returning `False` and falls through to the parallel path. In the parallel path: -- botanu creates its own `TracerProvider` and does **not** call - `trace.set_tracer_provider(...)`. -- ddtrace keeps handling spans for ddtrace decorators and for Datadog - auto-instrumentation. -- botanu's API (`botanu.event`, `track_llm_call`, etc.) gets its spans - from the botanu provider, which forwards to the botanu collector. +- botanu creates its own `TracerProvider` and does **not** call `trace.set_tracer_provider(...)`. +- ddtrace keeps handling spans for ddtrace decorators and for Datadog auto-instrumentation. +- botanu's API (`botanu.event`, `track_llm_call`, etc.) gets its spans from the botanu provider, which forwards to the botanu collector. The two tracing systems coexist. Nothing is stolen, nothing is wrapped. -A span will appear in Datadog if it was created inside ddtrace -instrumentation, and in botanu if it was created inside botanu -instrumentation. To cross-reference a trace between the two dashboards, use -`botanu.run_id` — it is set via W3C Baggage on every botanu span, and you -can write a small Datadog tag mapper to surface it on ddtrace spans too if -you want. +A span will appear in Datadog if it was created inside ddtrace instrumentation, and in botanu if it was created inside botanu instrumentation. To cross-reference a trace between the two dashboards, use `botanu.run_id` — it is set via W3C Baggage on every botanu span, and you can write a small Datadog tag mapper to surface it on ddtrace spans too if you want. ### Longer-term option If you eventually want a single tracing layer, the migration path is: 1. Today: dual tracing — ddtrace + botanu running in parallel. -2. Later: switch ddtrace off, move to the OTel SDK, configure the OTel - Datadog exporter. Now botanu's brownfield path kicks in and you're back - to one provider with two exporters. +2. Later: switch ddtrace off, move to the OTel SDK, configure the OTel Datadog exporter. Now botanu's provider-migration path kicks in and you're back to one provider with two exporters. -We do not require this and there is no deadline — the parallel setup is -supported indefinitely. +We do not require this and there is no deadline — the parallel setup is supported indefinitely. --- -## Greenfield: no existing tracing +## If you have no existing tracing -If `trace.get_tracer_provider()` returns a `ProxyTracerProvider`, nothing -is configured yet. botanu creates a fresh `TracerProvider` with -`ALWAYS_ON`, adds `RunContextEnricher` / `ResourceEnricher` / botanu's -exporter, and sets it as the global. This is the standard path for -first-time users. +If `trace.get_tracer_provider()` returns a `ProxyTracerProvider`, nothing is configured yet. botanu creates a fresh `TracerProvider` with `ALWAYS_ON`, adds `RunContextEnricher` / `ResourceEnricher` / botanu's exporter, and sets it as the global. This is the standard path for first-time users. --- ## Using the botanu API -Regardless of which path `enable()` takes, the API is the same: +Regardless of which path the SDK takes on initialisation, the API is the same: ```python import botanu @@ -192,8 +120,7 @@ Auto-instrumented spans (OpenAI SDK, HTTP clients, DB drivers) inside the event ## Manual integration (advanced, OTel SDK only) -If you want to wire botanu into an existing OTel SDK provider without -calling `enable()` at all, you can attach the processors yourself: +If you want to wire botanu into an existing OTel SDK provider without letting the SDK auto-initialise, you can attach the processors yourself: ```python from opentelemetry import trace @@ -206,10 +133,8 @@ from botanu.processors import RunContextEnricher, SampledSpanProcessor provider = trace.get_tracer_provider() assert isinstance(provider, TracerProvider), "manual integration requires the OTel SDK provider" -# 1. Enrich all spans with run_id / workflow / event_id from baggage. provider.add_span_processor(RunContextEnricher()) -# 2. Send a copy of every span to the botanu collector. botanu_exporter = OTLPSpanExporter( endpoint="https://ingest.botanu.ai:4318/v1/traces", headers={"Authorization": "Bearer "}, @@ -217,35 +142,27 @@ botanu_exporter = OTLPSpanExporter( provider.add_span_processor(BatchSpanProcessor(botanu_exporter)) ``` -**This does not work for ddtrace.** ddtrace's `TracerProvider` does not -expose `add_span_processor()`. If you are on ddtrace, use `enable()` and -let the SDK take the parallel path. +**This does not work for ddtrace.** ddtrace's `TracerProvider` does not expose `add_span_processor()`. If you are on ddtrace, let the SDK auto-initialise and take the parallel path. -Manual integration also skips the `SampledSpanProcessor` preservation -logic — if your provider uses ratio sampling and you need botanu to still -see 100%, you'd have to duplicate the bootstrap logic. Just call -`enable()`; that's what it's for. +Manual integration also skips the `SampledSpanProcessor` preservation logic — if your provider uses ratio sampling and you need botanu to still see 100%, you'd have to duplicate the bootstrap logic. Letting the SDK auto-initialise handles this for you. --- ## Verifying it worked -After installing and calling `enable()`, you should see exactly one of -these log lines: +After your first `botanu.event(...)` call runs, you should see exactly one of these log lines: | Log line | Means | | --- | --- | -| `existing TracerProvider detected with N% sampling. Preserved your sampling ratio` | Brownfield OTel SDK with known sampler. Your APM gets the same volume, botanu gets 100%. | -| `existing TracerProvider detected. Added botanu exporter alongside your existing setup` | Brownfield OTel SDK with AlwaysOn. Both exporters get 100%. | -| `could not identify the sampling ratio of X. Preserving the original sampler` | Brownfield OTel SDK with unknown sampler. Both exporters see the sampled subset. | -| (no brownfield line) | Greenfield or ddtrace parallel — check the `enable()` return value. | +| `existing TracerProvider detected with N% sampling. Preserved your sampling ratio` | OTel SDK with known sampler. Your APM gets the same volume, botanu gets 100%. | +| `existing TracerProvider detected. Added botanu exporter alongside your existing setup` | OTel SDK with AlwaysOn. Both exporters get 100%. | +| `could not identify the sampling ratio of X. Preserving the original sampler` | OTel SDK with unknown sampler. Both exporters see the sampled subset. | +| (none of the above) | No existing tracing, or ddtrace parallel — check `botanu.is_enabled()`. | Sanity checks in order: 1. Open your existing APM — confirm span volume is unchanged. -2. Open botanu — confirm spans are arriving. A run created by - `botanu.event(...)` carries `botanu.run_id`, `botanu.workflow`, - `botanu.event_id`. +2. Open botanu — confirm spans are arriving. A run created by `botanu.event(...)` carries `botanu.run_id`, `botanu.workflow`, `botanu.event_id`. 3. If both arrive, you're done. --- @@ -256,50 +173,21 @@ Sanity checks in order: Should not happen. Check, in order: -1. `enable()` was called exactly once at startup — if called twice, both - calls no-op after the first, so duplicate calls are safe but indicate a - confused startup order. -2. Your existing OTel provider was created **before** `enable()` runs. If - `enable()` runs first, brownfield detection doesn't see you and you'll - end up on the greenfield path, which blows away an OTel provider set - afterwards. -3. Your existing exporter was actually attached to the provider botanu - detected. If you have multiple providers (per-module, per-service), the - one `trace.get_tracer_provider()` returns is the one botanu wraps — - processors attached to others are unaffected (and therefore invisible - to botanu too). +1. Your existing OTel provider was created **before** the first `botanu.event(...)` call. If the SDK initialises first, it won't detect your provider and will replace it. +2. Your existing exporter was actually attached to the provider botanu detected. If you have multiple providers (per-module, per-service), the one `trace.get_tracer_provider()` returns is the one botanu wraps — processors attached to others are unaffected (and therefore invisible to botanu too). ### botanu shows 100% of spans but Datadog only shows 10% -That's the expected brownfield behavior with a ratio sampler. botanu -captures 100% for cost attribution; your existing exporter stays on its -original ratio so your bill and dashboards are unchanged. Look for the log -line starting `Preserved your sampling ratio`. +That's the expected behaviour with a ratio sampler. botanu captures 100% for cost attribution; your existing exporter stays on its original ratio so your bill and dashboards are unchanged. Look for the log line starting `Preserved your sampling ratio`. ### `run_id` is missing on auto-instrumented spans -1. Verify `enable()` was called (or `RunContextEnricher` was attached - manually). -2. Verify an entry-point function or block uses `botanu.event(...)` — the - baggage is set on entry and inherited by child spans from there. -3. Verify the W3C Baggage propagator is active: - `from opentelemetry import propagate; propagate.get_global_textmap()` - should include `baggage` in its composite. +1. Verify an entry-point function or block uses `botanu.event(...)` — the baggage is set on entry and inherited by child spans from there. +2. Verify the W3C Baggage propagator is active: `from opentelemetry import propagate; propagate.get_global_textmap()` should include `baggage` in its composite. ### `could not identify the sampling ratio` warning Your sampler is a type botanu doesn't recognise. Two options: -1. Accept it — botanu sees only the sampled subset. Cost attribution is - still correct, just computed on fewer spans. -2. Switch to `TraceIdRatioBased(...)`, `ParentBased(TraceIdRatioBased(...))`, - `AlwaysOn`, or `AlwaysOff` on your `TracerProvider`. botanu will then - take the normal brownfield path and preserve your ratio for existing - processors while capturing 100% for itself. - -## See also - -- [Collector](collector.md) — where botanu's spans go next -- [Auto-Instrumentation](auto-instrumentation.md) — the span sources -- [Configuration](../getting-started/configuration.md) — env vars, endpoint trust -- Source of truth: [`src/botanu/sdk/bootstrap.py`](../../src/botanu/sdk/bootstrap.py) and [`src/botanu/processors/sampled.py`](../../src/botanu/processors/sampled.py) +1. Accept it — botanu sees only the sampled subset. Cost attribution is still correct, just computed on fewer spans. +2. Switch to `TraceIdRatioBased(...)`, `ParentBased(TraceIdRatioBased(...))`, `AlwaysOn`, or `AlwaysOff` so botanu can preserve your ratio exactly. diff --git a/docs/patterns/anti-patterns.md b/docs/patterns/anti-patterns.md index 4b9f3af..a4431a8 100644 --- a/docs/patterns/anti-patterns.md +++ b/docs/patterns/anti-patterns.md @@ -112,23 +112,6 @@ with botanu.event(event_id=..., customer_id=..., workflow="Support"): botanu.emit_outcome(value_type="tickets_resolved", value_amount=1) ``` -## Calling `enable()` repeatedly - -`enable()` runs lazily on the first `botanu.event(...)` call. Calling it explicitly is only useful when you need to override config — a custom endpoint, an API key passed in code, a non-default content-capture rate. Calling `enable()` inside a request handler or library adds no value and can misconfigure the SDK. - -```python -def handle_request(req): - botanu.enable() - with botanu.event(...): - ... -``` - -```python -def handle_request(req): - with botanu.event(...): - ... -``` - ## Putting secrets or PII in baggage Baggage travels in plaintext on every outbound HTTP request and through third-party HTTP middleware. User tokens, API keys, and PII should flow through your application's normal auth layer, not through `set_baggage`. diff --git a/docs/tracking/content-capture.md b/docs/tracking/content-capture.md index a354063..f5ff903 100644 --- a/docs/tracking/content-capture.md +++ b/docs/tracking/content-capture.md @@ -13,16 +13,7 @@ and, via the verdict rollup, to accurate event-level outcome determination. ## The knob -One config field turns the whole thing on: - -```python -from botanu import enable -from botanu.sdk.config import BotanuConfig - -enable(config=BotanuConfig(content_capture_rate=0.10)) -``` - -Or via environment: +One env var turns the whole thing on: ```bash export BOTANU_CONTENT_CAPTURE_RATE=0.10