LCORE-1822: Enable OpenTelemetry delivery E2E test with mock OTLP collector

docker-compose-library.yaml

@@ -11,6 +11,8 @@ services:
     depends_on:
       mock-mcp:
         condition: service_healthy
+      mock-otel:
+        condition: service_healthy
     networks:
       - lightspeednet
     volumes:
@@ -67,12 +69,15 @@ services:
       - PDF_KV_RAG_PATH=/tmp/e2e-rag-work/pdf_kv_store.db
       # Prevent HuggingFace Hub update checks (HTTP 429 rate-limiting in CI from parallel jobs).
       - HF_HUB_OFFLINE=1
-      # OpenTelemetry configuration (tracing disabled by default)
-      - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-}
-      - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-}
-      - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-}
+      # OpenTelemetry configuration. Export is enabled by default and points at
+      # the mock OTLP/HTTP collector so the OpenTelemetry delivery E2E test works
+      # without per-scenario reconfiguration. Override any OTEL_* var (or set
+      # OTEL_SDK_DISABLED=true) to change or disable export.
+      - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-http://mock-otel:4318}
+      - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
+      - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-lightspeed-stack-e2e}
       - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default}
-      - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-true}
+      - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-false}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8080/liveness"]
       interval: 10s   # how often to run the check
@@ -113,6 +118,26 @@ services:
       retries: 3
       start_period: 2s
 
+  # Mock OTLP/HTTP collector for OpenTelemetry E2E tests.
+  # lightspeed-stack exports to it by default (see OTEL_* above) and waits for it
+  # to be healthy, so telemetry is delivered from startup. The port is bound to
+  # loopback only so it is not exposed beyond the host running the tests.
+  mock-otel:
+    build:
+      context: ./tests/e2e/mock_otel_collector
+      dockerfile: Dockerfile
+    container_name: mock-otel
+    ports:
+      - "127.0.0.1:4318:4318"
+    networks:
+      - lightspeednet
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4318/health')"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+      start_period: 2s
+
 
 networks:
   lightspeednet:

docker-compose.yaml

@@ -114,17 +114,22 @@ services:
       # Substituted in mounted lightspeed-stack.yaml (ogx.url); GitHub Actions sets E2E_LLAMA_HOSTNAME
       - E2E_OGX_HOSTNAME=${E2E_OGX_HOSTNAME:-${E2E_LLAMA_HOSTNAME:-ogx}}
       - E2E_OGX_PORT=${E2E_OGX_PORT:-${E2E_LLAMA_PORT:-8321}}
-      # OpenTelemetry configuration (tracing disabled by default)
-      - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-}
-      - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-}
-      - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-}
+      # OpenTelemetry configuration. Export is enabled by default and points at
+      # the mock OTLP/HTTP collector so the OpenTelemetry delivery E2E test works
+      # without per-scenario reconfiguration. Override any OTEL_* var (or set
+      # OTEL_SDK_DISABLED=true) to change or disable export.
+      - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-http://mock-otel:4318}
+      - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
+      - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-lightspeed-stack-e2e}
       - OTEL_ANONYMIZATION_SECRET=${OTEL_ANONYMIZATION_SECRET:-lightspeed-stack-otel-anonymization-dev-default}
-      - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-true}
+      - OTEL_SDK_DISABLED=${OTEL_SDK_DISABLED:-false}
     depends_on:
         ogx:
           condition: service_healthy
         mock-mcp:
           condition: service_healthy
+        mock-otel:
+          condition: service_healthy
     networks:
       - lightspeednet
     healthcheck:
@@ -167,6 +172,26 @@ services:
       retries: 3
       start_period: 2s
 
+  # Mock OTLP/HTTP collector for OpenTelemetry E2E tests.
+  # lightspeed-stack exports to it by default (see OTEL_* above) and waits for it
+  # to be healthy, so telemetry is delivered from startup. The port is bound to
+  # loopback only so it is not exposed beyond the host running the tests.
+  mock-otel:
+    build:
+      context: ./tests/e2e/mock_otel_collector
+      dockerfile: Dockerfile
+    container_name: mock-otel
+    ports:
+      - "127.0.0.1:4318:4318"
+    networks:
+      - lightspeednet
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4318/health')"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+      start_period: 2s
+
   # Mock TLS inference server for TLS E2E tests
   mock-tls-inference:
     build:

src/app/endpoints/responses.py

@@ -579,14 +579,18 @@ async def handle_responses_with_tracing(  # pylint: disable=too-many-locals
     )
     attachments_count = _count_request_attachments(original_request.input)
 
-    set_span_attributes(
-        root_span,
-        {
-            SpanAttributes.USER_ID: anonymize_value(user_id),
-            SpanAttributes.INPUT: anonymize_value(input_text),
-            SpanAttributes.REQUEST_ATTACHMENTS_COUNT: attachments_count,
-        },
-    )
+    span_attributes: dict[str, Any] = {
+        SpanAttributes.USER_ID: anonymize_value(user_id),
+        SpanAttributes.INPUT: anonymize_value(input_text),
+        SpanAttributes.REQUEST_ATTACHMENTS_COUNT: attachments_count,
+    }
+    # safety_identifier is a caller-supplied, non-PII identifier, so it is
+    # recorded verbatim (not anonymized) when present.
+    if original_request.safety_identifier is not None:
+        span_attributes[SpanAttributes.SAFETY_IDENTIFIER] = (
+            original_request.safety_identifier
+        )
+    set_span_attributes(root_span, span_attributes)
 
     await check_mcp_auth(configuration, mcp_headers, token, request.headers)
 

src/utils/otel_tracing.py

@@ -24,6 +24,7 @@ class SpanAttributes(StrEnum):
 
     SESSION_ID = "session.id"
     USER_ID = "user.id"  # anonymized
+    SAFETY_IDENTIFIER = "request.safety_identifier"  # caller-supplied identifier
     INPUT = "request.input"  # anonymized
     OUTPUT = "response.output"  # anonymized
     RESPONSE_ERROR = "response.error"

tests/e2e/features/opentelemetry.feature

@@ -1,11 +1,10 @@
-@cfg_authorized @OTel @skip
+@cfg_authorized @OTel @skip-in-prow
 Feature: OpenTelemetry observability tests
 
   Background:
     Given The service is started locally
       And The system is in default state
       And An OpenTelemetry service is running and listening for OTLP data
-      And The service is configured to export data to the OpenTelemetry service
       And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva
       And REST API service prefix is /v1
       And the Lightspeed stack configuration directory is "tests/e2e/configuration"
@@ -25,5 +24,4 @@ Feature: OpenTelemetry observability tests
       }
       """
     Then The status code of the response is 200
-     And The service exported an OpenTelemetry event containing e2e-otel-delivery-marker
      And The OpenTelemetry service received data containing e2e-otel-delivery-marker

tests/e2e/features/steps/opentelemetry.py

@@ -0,0 +1,87 @@
+"""Step definitions for the OpenTelemetry telemetry-delivery E2E scenario.
+
+The Lightspeed Core Stack exports spans/events to the mock OTLP/HTTP collector
+(the ``mock-otel`` Docker Compose service) from startup: the ``OTEL_*``
+environment variables that enable ``opentelemetry-instrument`` and point the
+exporter at the collector are baked into the Compose files, so no per-scenario
+reconfiguration is needed. These steps only reset the collector's buffer at the
+start of the scenario and assert that telemetry containing a scenario marker is
+delivered.
+"""
+
+import os
+import time
+
+import requests
+from behave import given, then  # pyright: ignore[reportAttributeAccessIssue]
+from behave.runner import Context
+
+from tests.e2e.utils.utils import wait_for_container_health
+
+# Compose service / container name for the mock collector (see docker-compose*.yaml).
+MOCK_OTEL_SERVICE = "mock-otel"
+
+# Host-side control API of the mock collector (published port from docker-compose).
+_MOCK_OTEL_HOST = os.getenv("E2E_OTEL_MOCK_HOST", "localhost")
+_MOCK_OTEL_PORT = os.getenv("E2E_OTEL_MOCK_PORT", "4318")
+MOCK_OTEL_CONTROL_BASE = f"http://{_MOCK_OTEL_HOST}:{_MOCK_OTEL_PORT}"
+
+# Delivery is asynchronous: the SDK batches spans before export. Poll generously.
+_DELIVERY_TIMEOUT_S = float(os.getenv("E2E_OTEL_DELIVERY_TIMEOUT_S", "45"))
+_DELIVERY_POLL_INTERVAL_S = 2.0
+
+
+def _reset_mock_collector() -> None:
+    """Clear any telemetry buffered by the mock collector from prior runs."""
+    response = requests.post(f"{MOCK_OTEL_CONTROL_BASE}/reset", timeout=5)
+    assert (
+        response.status_code == 200
+    ), f"Failed to reset mock OTEL collector: HTTP {response.status_code}"
+
+
+def _poll_collector_contains(marker: str) -> bool:
+    """Return True once the collector has buffered a payload containing ``marker``."""
+    url = f"{MOCK_OTEL_CONTROL_BASE}/received"
+    deadline = time.monotonic() + _DELIVERY_TIMEOUT_S
+    while time.monotonic() < deadline:
+        try:
+            response = requests.get(url, params={"contains": marker}, timeout=5)
+            if response.status_code == 200 and response.json().get("found"):
+                return True
+        except requests.RequestException:
+            pass
+        time.sleep(_DELIVERY_POLL_INTERVAL_S)
+    return False
+
+
+@given("An OpenTelemetry service is running and listening for OTLP data")
+def otel_service_running(context: Context) -> None:
+    """Wait for the mock OTLP collector to be healthy and clear its buffer.
+
+    The ``mock-otel`` Compose service starts with the rest of the stack and its
+    readiness is enforced by the Compose healthcheck, so this step waits for
+    that health status and resets any previously buffered telemetry so the
+    scenario starts from a clean slate. The Lightspeed Core Stack already
+    exports to the collector via the ``OTEL_*`` variables set in the Compose
+    files.
+    """
+    wait_for_container_health(MOCK_OTEL_SERVICE)
+    _reset_mock_collector()
+    context.otel_collector_ready = True
+
+
+@then("The OpenTelemetry service received data containing {marker}")
+def collector_received_data(context: Context, marker: str) -> None:
+    """Assert the mock collector buffered telemetry containing ``marker``.
+
+    Verifies delivery from the collector's perspective; polls to tolerate the
+    SDK's batched, asynchronous export.
+    """
+    assert getattr(
+        context, "otel_collector_ready", False
+    ), "The OpenTelemetry service must be started before asserting on delivery"
+    marker = marker.strip()
+    assert _poll_collector_contains(marker), (
+        f"Mock OTEL collector did not receive data containing {marker!r} "
+        f"within {_DELIVERY_TIMEOUT_S:.0f}s"
+    )

tests/e2e/mock_otel_collector/Dockerfile

@@ -0,0 +1,5 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY server.py .
+EXPOSE 4318
+CMD ["python", "server.py"]

tests/e2e/mock_otel_collector/README.md

@@ -0,0 +1,53 @@
+# Mock OTEL collector
+
+A minimal OTLP/HTTP collector used by the OpenTelemetry E2E scenario
+(`tests/e2e/features/opentelemetry.feature`) to verify that the Lightspeed Core
+Stack delivers spans/events to a telemetry backend.
+
+It is a stdlib-only `http.server` that buffers the raw OTLP export bodies in
+memory and exposes a small control API so Behave steps can assert what was
+received. See `server.py` for the full endpoint list.
+
+## Endpoints
+
+| Method & path        | Purpose                                                        |
+| -------------------- | ------------------------------------------------------------- |
+| `POST /v1/*`         | Receive an OTLP export (traces/logs/metrics); body buffered.  |
+| `GET /received`      | Report the count of buffered exports.                         |
+| `GET /received?contains=<text>` | Report whether `<text>` appears in any payload.   |
+| `POST /reset`        | Clear the buffer (called at scenario start).                  |
+| `GET /health`        | Liveness probe (`{"status": "ok"}`).                         |
+
+Substring queries search the raw request bytes. OTLP protobuf encodes string
+fields as UTF-8, so a plaintext marker embedded in a span attribute value is
+found without decoding protobuf.
+
+## Running
+
+Locally:
+
+```bash
+python server.py [port]   # default port 4318
+```
+
+In E2E it runs as the `mock-otel` Docker Compose service on the `lightspeednet`
+network. It starts with the rest of the stack (`docker compose up -d`) and
+`lightspeed-stack` lists it under `depends_on` (waiting for it to become
+healthy), so telemetry is delivered from startup. The
+`An OpenTelemetry service is running and listening for OTLP data` step only waits
+for it to become healthy and resets its buffer.
+
+## Pointing the service at it
+
+The Lightspeed Core Stack exports via HTTP/protobuf when launched with the OTEL
+SDK enabled. The Compose files set these by default so export is always on in
+E2E (override or set `OTEL_SDK_DISABLED=true` to change or disable it):
+
+```bash
+OTEL_SDK_DISABLED=false
+OTEL_EXPORTER_OTLP_ENDPOINT=http://mock-otel:4318
+OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+```
+
+The `scripts/entrypoint.sh` gate launches the service under
+`opentelemetry-instrument` whenever `OTEL_SDK_DISABLED=false`.