From 810b8c89e8fe2f022be3f67190910355a46c03f1 Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Tue, 14 Jul 2026 01:09:24 +0530 Subject: [PATCH 1/7] feat(api/observability): add OpenTelemetry bootstrap package + unit tests --- apps/api/plane/observability/__init__.py | 3 + apps/api/plane/observability/logging.py | 37 +++ apps/api/plane/observability/setup.py | 217 ++++++++++++++ .../tests/unit/observability/__init__.py | 3 + .../tests/unit/observability/conftest.py | 37 +++ .../tests/unit/observability/test_logging.py | 80 +++++ .../tests/unit/observability/test_setup.py | 283 ++++++++++++++++++ apps/api/requirements/base.txt | 10 + 8 files changed, 670 insertions(+) create mode 100644 apps/api/plane/observability/__init__.py create mode 100644 apps/api/plane/observability/logging.py create mode 100644 apps/api/plane/observability/setup.py create mode 100644 apps/api/plane/tests/unit/observability/__init__.py create mode 100644 apps/api/plane/tests/unit/observability/conftest.py create mode 100644 apps/api/plane/tests/unit/observability/test_logging.py create mode 100644 apps/api/plane/tests/unit/observability/test_setup.py diff --git a/apps/api/plane/observability/__init__.py b/apps/api/plane/observability/__init__.py new file mode 100644 index 00000000000..fcc34a703d7 --- /dev/null +++ b/apps/api/plane/observability/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. diff --git a/apps/api/plane/observability/logging.py b/apps/api/plane/observability/logging.py new file mode 100644 index 00000000000..c57c015679a --- /dev/null +++ b/apps/api/plane/observability/logging.py @@ -0,0 +1,37 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +"""Trace-context log correlation for the API. + +`TraceContextFilter` adds trace_id / span_id / trace_flags / service_name +attributes to every LogRecord. Empty values when no span is active. + +The filter is wired in via Django's LOGGING dict (see +plane/settings/local.py and plane/settings/production.py) and attached to +every handler, so it works regardless of logger propagation settings. +configure_otel() does not install it at runtime — Django's dictConfig +would wipe a runtime-installed filter when settings are applied. +""" + +import logging +import os + +from opentelemetry import trace + + +class TraceContextFilter(logging.Filter): + """Inject trace_id, span_id, trace_flags, service_name into LogRecord.""" + + def filter(self, record: logging.LogRecord) -> bool: + ctx = trace.get_current_span().get_span_context() + if ctx.is_valid: + record.trace_id = format(ctx.trace_id, "032x") + record.span_id = format(ctx.span_id, "016x") + record.trace_flags = int(ctx.trace_flags) + else: + record.trace_id = "" + record.span_id = "" + record.trace_flags = 0 + record.service_name = os.environ.get("OTEL_SERVICE_NAME", "plane-api") + return True diff --git a/apps/api/plane/observability/setup.py b/apps/api/plane/observability/setup.py new file mode 100644 index 00000000000..55e6559acc8 --- /dev/null +++ b/apps/api/plane/observability/setup.py @@ -0,0 +1,217 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +"""OpenTelemetry bootstrap for the Plane API. + +Single entry point: configure_otel(). Idempotent. No-op unless OTEL_ENABLED +is truthy and OTEL_EXPORTER_OTLP_ENDPOINT is set. +""" + +import logging +import os + +from opentelemetry import metrics, trace +from opentelemetry.instrumentation.celery import CeleryInstrumentor +from opentelemetry.instrumentation.django import DjangoInstrumentor +from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor +from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor +from opentelemetry.instrumentation.redis import RedisInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +logger = logging.getLogger(__name__) + +_CONFIGURED = False + +# Kept at module scope so flush_otel() can force-flush them on worker shutdown +# (prefork children exit via os._exit and skip atexit). +_TRACER_PROVIDER: "TracerProvider | None" = None +_METER_PROVIDER: "MeterProvider | None" = None + +# Accepted "on" tokens — kept in sync with the pi/node services so OTEL_ENABLED +# behaves identically across every runtime. +_TRUTHY_VALUES = ("1", "true", "yes", "on") + +_NOISY_OTEL_LOGGERS = ( + "opentelemetry", + "opentelemetry.exporter.otlp", + "opentelemetry.exporter.otlp.proto.grpc", + "opentelemetry.exporter.otlp.proto.http", + "opentelemetry.instrumentation", + "opentelemetry.sdk.trace.export", +) + +_HTTP_PROTOCOLS = ("http/protobuf", "http") + + +def _is_enabled() -> bool: + return os.environ.get("OTEL_ENABLED", "0").strip().lower() in _TRUTHY_VALUES + + +def _has_endpoint() -> bool: + return bool(os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()) + + +def _protocol() -> str: + return os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").lower() + + +def _apply_defaults() -> None: + """Set defaults for standard OTEL env vars. Operator overrides win.""" + os.environ.setdefault("OTEL_SERVICE_NAME", "plane-api") + os.environ.setdefault("OTEL_TRACES_SAMPLER", "parentbased_traceidratio") + os.environ.setdefault("OTEL_TRACES_SAMPLER_ARG", "0.1") + os.environ.setdefault("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc") + + +def _build_resource() -> Resource: + """Build the OTEL Resource. + + Resource.create() reads OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES from + the environment natively. We additionally set deployment.environment.name + from a dedicated OTEL_ENVIRONMENT (or SENTRY_ENVIRONMENT) var. Note the + current semconv key is `deployment.environment.name` (not the legacy + `deployment.environment`). + """ + attributes: dict[str, str] = {} + environment = os.environ.get("OTEL_ENVIRONMENT") or os.environ.get("SENTRY_ENVIRONMENT") or "" + if environment: + attributes["deployment.environment.name"] = environment + return Resource.create(attributes) + + +def _create_span_exporter(): + """Return the OTLP span exporter that matches OTEL_EXPORTER_OTLP_PROTOCOL.""" + if _protocol() in _HTTP_PROTOCOLS: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter as HttpSpanExporter, + ) + + return HttpSpanExporter() + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as GrpcSpanExporter, + ) + + return GrpcSpanExporter() + + +def _create_metric_exporter(): + """Return the OTLP metric exporter that matches OTEL_EXPORTER_OTLP_PROTOCOL.""" + if _protocol() in _HTTP_PROTOCOLS: + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter as HttpMetricExporter, + ) + + return HttpMetricExporter() + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter as GrpcMetricExporter, + ) + + return GrpcMetricExporter() + + +def _setup_tracing(resource: Resource) -> None: + global _TRACER_PROVIDER + provider = TracerProvider(resource=resource) + provider.add_span_processor(BatchSpanProcessor(_create_span_exporter())) + trace.set_tracer_provider(provider) + _TRACER_PROVIDER = provider + + +def _setup_metrics(resource: Resource) -> None: + global _METER_PROVIDER + reader = PeriodicExportingMetricReader(_create_metric_exporter()) + provider = MeterProvider(resource=resource, metric_readers=[reader]) + metrics.set_meter_provider(provider) + _METER_PROVIDER = provider + + +def _instrument_libraries() -> None: + """Patch Django + Celery + downstream client libraries. + + DjangoInstrumentor emits HTTP server spans + http.server.* metrics. + CeleryInstrumentor emits a span per task with celery.action, task_name, + task_id, state, and propagates traceparent across the queue so a + request that enqueues a task is linked to that task's execution span. + The others add child spans so latency can be decomposed (SQL query, + Redis op, outbound HTTP). + """ + DjangoInstrumentor().instrument() + CeleryInstrumentor().instrument() + PsycopgInstrumentor().instrument(enable_commenter=False) + RedisInstrumentor().instrument() + RequestsInstrumentor().instrument() + HTTPXClientInstrumentor().instrument() + + +def _quiet_otel_loggers() -> None: + """Pin OTEL SDK / exporter loggers to WARNING. + + Without this, a misconfigured or unreachable collector floods stdout + with per-export INFO/ERROR messages on every batch flush. + """ + for name in _NOISY_OTEL_LOGGERS: + logging.getLogger(name).setLevel(logging.WARNING) + + +def configure_otel() -> None: + """Configure OpenTelemetry tracing + metrics. + + No-op unless OTEL_ENABLED is truthy and OTEL_EXPORTER_OTLP_ENDPOINT is set. + Idempotent — safe to call from wsgi.py, asgi.py, manage.py, and celery.py. + + Log correlation is wired separately via Django's LOGGING dict (see + plane/settings/local.py and production.py) and, for Celery, via the + after_setup_logger handlers in plane/celery.py. + """ + global _CONFIGURED + if _CONFIGURED: + return + if not _is_enabled(): + return + if not _has_endpoint(): + logger.warning("OTEL_ENABLED=1 but OTEL_EXPORTER_OTLP_ENDPOINT is not set; OpenTelemetry bootstrap skipped") + return + + _apply_defaults() + resource = _build_resource() + _setup_tracing(resource) + _setup_metrics(resource) + _instrument_libraries() + _quiet_otel_loggers() + + _CONFIGURED = True + logger.info( + "OpenTelemetry configured: service=%s, endpoint=%s, protocol=%s, sampler=%s(%s)", + os.environ.get("OTEL_SERVICE_NAME"), + os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT"), + os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL"), + os.environ.get("OTEL_TRACES_SAMPLER"), + os.environ.get("OTEL_TRACES_SAMPLER_ARG"), + ) + + +def flush_otel(timeout_millis: int = 3000) -> None: + """Best-effort bounded flush of buffered spans + metrics. + + Celery prefork children exit via os._exit and never run atexit hooks, so the + worker_process_shutdown signal calls this to keep the tail of each child's + spans/metrics from being dropped on --max-tasks-per-child recycling and warm + shutdown. Never raises and is bounded by the timeout, so it cannot stall + child replacement. + """ + if _TRACER_PROVIDER is not None: + try: + _TRACER_PROVIDER.force_flush(timeout_millis=timeout_millis) + except Exception as exc: + logger.warning("OTel trace flush failed: %s", exc) + if _METER_PROVIDER is not None: + try: + _METER_PROVIDER.force_flush(timeout_millis=timeout_millis) + except Exception as exc: + logger.warning("OTel metric flush failed: %s", exc) diff --git a/apps/api/plane/tests/unit/observability/__init__.py b/apps/api/plane/tests/unit/observability/__init__.py new file mode 100644 index 00000000000..fcc34a703d7 --- /dev/null +++ b/apps/api/plane/tests/unit/observability/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. diff --git a/apps/api/plane/tests/unit/observability/conftest.py b/apps/api/plane/tests/unit/observability/conftest.py new file mode 100644 index 00000000000..f140e7a9d2f --- /dev/null +++ b/apps/api/plane/tests/unit/observability/conftest.py @@ -0,0 +1,37 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +"""Test fixtures for the observability package. + +`configure_otel()` is gated by environment variables and a module-level +`_CONFIGURED` flag. Tests must start from a clean slate every time, so: + +- `_CONFIGURED` is reset before every test (otherwise the second test + would always be a no-op because the first one flipped the flag). +- Every `OTEL_*` env var is stripped before the test and restored after, + so what `_apply_defaults()` writes in one test doesn't leak into the + next (it uses `os.environ.setdefault`, which bypasses monkeypatch's + bookkeeping). +""" + +import os + +import pytest + + +@pytest.fixture(autouse=True) +def isolate_otel_state(monkeypatch): + from plane.observability import setup as otel_setup + + monkeypatch.setattr(otel_setup, "_CONFIGURED", False, raising=False) + + saved = {k: v for k, v in os.environ.items() if k.startswith("OTEL_")} + for key in list(saved.keys()): + del os.environ[key] + + yield + + for key in [k for k in os.environ if k.startswith("OTEL_")]: + del os.environ[key] + os.environ.update(saved) diff --git a/apps/api/plane/tests/unit/observability/test_logging.py b/apps/api/plane/tests/unit/observability/test_logging.py new file mode 100644 index 00000000000..67eb64fcd10 --- /dev/null +++ b/apps/api/plane/tests/unit/observability/test_logging.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +"""Tests for plane.observability.logging.TraceContextFilter.""" + +import logging + +import pytest +from opentelemetry.sdk.trace import TracerProvider + +from plane.observability.logging import TraceContextFilter + + +def _make_record() -> logging.LogRecord: + return logging.LogRecord( + name="t", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="x", + args=(), + exc_info=None, + ) + + +@pytest.mark.unit +def test_filter_writes_empty_ids_when_no_active_span(): + record = _make_record() + TraceContextFilter().filter(record) + assert record.trace_id == "" + assert record.span_id == "" + assert record.trace_flags == 0 + + +@pytest.mark.unit +def test_filter_writes_ids_when_span_active(): + # Use a local SDK TracerProvider — do NOT call set_tracer_provider so + # the global state isn't mutated for sibling tests. + tracer = TracerProvider().get_tracer(__name__) + with tracer.start_as_current_span("unit-span"): + record = _make_record() + TraceContextFilter().filter(record) + + assert len(record.trace_id) == 32 + assert len(record.span_id) == 16 + assert int(record.trace_id, 16) != 0 + assert int(record.span_id, 16) != 0 + + +@pytest.mark.unit +def test_filter_always_returns_true(): + # Filter is a "decorator" — it enriches the record but never drops it. + assert TraceContextFilter().filter(_make_record()) is True + + +@pytest.mark.unit +def test_service_name_uses_env_when_set(monkeypatch): + monkeypatch.setenv("OTEL_SERVICE_NAME", "custom-service") + record = _make_record() + TraceContextFilter().filter(record) + assert record.service_name == "custom-service" + + +@pytest.mark.unit +def test_service_name_falls_back_to_plane_api(): + record = _make_record() + TraceContextFilter().filter(record) + assert record.service_name == "plane-api" + + +@pytest.mark.unit +def test_trace_flags_is_int_for_log_record_compat(): + # python-json-logger serializes ints natively; ensure trace_flags is + # an int and not a TraceFlags enum object that would render as a repr. + tracer = TracerProvider().get_tracer(__name__) + with tracer.start_as_current_span("unit-span"): + record = _make_record() + TraceContextFilter().filter(record) + assert isinstance(record.trace_flags, int) diff --git a/apps/api/plane/tests/unit/observability/test_setup.py b/apps/api/plane/tests/unit/observability/test_setup.py new file mode 100644 index 00000000000..9c99564b157 --- /dev/null +++ b/apps/api/plane/tests/unit/observability/test_setup.py @@ -0,0 +1,283 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +"""Tests for plane.observability.setup.configure_otel and helpers. + +Strategy: mock at the boundary. The OTLP exporter factories and the +opentelemetry global setters (`set_tracer_provider`, `set_meter_provider`) +are patched so tests don't open real gRPC connections or mutate global +OTEL state. Instrumentor classes are patched at the same point — the +real DjangoInstrumentor is a module-level singleton that would survive +the test process and pollute other suites. +""" + +import logging +import os +from contextlib import ExitStack +from unittest.mock import patch + +import pytest + +from plane.observability.setup import ( + _create_metric_exporter, + _create_span_exporter, + configure_otel, +) + + +_MOCK_TARGETS = ( + "plane.observability.setup._create_span_exporter", + "plane.observability.setup._create_metric_exporter", + "opentelemetry.trace.set_tracer_provider", + "opentelemetry.metrics.set_meter_provider", + "plane.observability.setup.DjangoInstrumentor", + "plane.observability.setup.CeleryInstrumentor", + "plane.observability.setup.PsycopgInstrumentor", + "plane.observability.setup.RedisInstrumentor", + "plane.observability.setup.RequestsInstrumentor", + "plane.observability.setup.HTTPXClientInstrumentor", +) + + +def _full_enabled_env(monkeypatch): + """Set the minimal env that makes configure_otel run end-to-end.""" + monkeypatch.setenv("OTEL_ENABLED", "1") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") + + +def _patched_configure_otel(): + """Run configure_otel with the full boundary mocked. + + Returns a dict of mock objects keyed by their target string so callers + can assert on them. + """ + stack = ExitStack() + mocks = {target: stack.enter_context(patch(target)) for target in _MOCK_TARGETS} + try: + configure_otel() + finally: + stack.close() + return mocks + + +# --------------------------------------------------------------------------- +# Gating +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_disabled_by_default_does_nothing(): + # No env set (fixture clears OTEL_* automatically). + with patch("opentelemetry.trace.set_tracer_provider") as set_tp, patch( + "opentelemetry.metrics.set_meter_provider" + ) as set_mp: + configure_otel() + set_tp.assert_not_called() + set_mp.assert_not_called() + + +@pytest.mark.unit +def test_explicit_zero_is_treated_as_disabled(monkeypatch): + monkeypatch.setenv("OTEL_ENABLED", "0") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") + with patch("opentelemetry.trace.set_tracer_provider") as set_tp: + configure_otel() + set_tp.assert_not_called() + + +@pytest.mark.unit +def test_enabled_without_endpoint_warns_and_returns(monkeypatch, caplog): + monkeypatch.setenv("OTEL_ENABLED", "1") + with patch("opentelemetry.trace.set_tracer_provider") as set_tp: + with caplog.at_level(logging.WARNING, logger="plane.observability.setup"): + configure_otel() + set_tp.assert_not_called() + assert any( + "OTEL_EXPORTER_OTLP_ENDPOINT is not set" in r.message + for r in caplog.records + ) + + +@pytest.mark.unit +@pytest.mark.parametrize("truthy_value", ["1", "true", "TRUE", "yes", "YES", "on", "ON", " on "]) +def test_truthy_values_for_otel_enabled(truthy_value, monkeypatch): + monkeypatch.setenv("OTEL_ENABLED", truthy_value) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") + + mocks = _patched_configure_otel() + mocks["opentelemetry.trace.set_tracer_provider"].assert_called_once() + + +# --------------------------------------------------------------------------- +# Provider + instrumentation wiring +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_enabled_with_endpoint_sets_providers_and_instruments(monkeypatch): + _full_enabled_env(monkeypatch) + mocks = _patched_configure_otel() + + mocks["plane.observability.setup._create_span_exporter"].assert_called_once() + mocks["plane.observability.setup._create_metric_exporter"].assert_called_once() + mocks["opentelemetry.trace.set_tracer_provider"].assert_called_once() + mocks["opentelemetry.metrics.set_meter_provider"].assert_called_once() + + for instrumentor_target in ( + "plane.observability.setup.DjangoInstrumentor", + "plane.observability.setup.CeleryInstrumentor", + "plane.observability.setup.PsycopgInstrumentor", + "plane.observability.setup.RedisInstrumentor", + "plane.observability.setup.RequestsInstrumentor", + "plane.observability.setup.HTTPXClientInstrumentor", + ): + mocks[instrumentor_target].return_value.instrument.assert_called_once() + + +@pytest.mark.unit +def test_psycopg_instrumentor_called_with_enable_commenter_false(monkeypatch): + _full_enabled_env(monkeypatch) + mocks = _patched_configure_otel() + mocks[ + "plane.observability.setup.PsycopgInstrumentor" + ].return_value.instrument.assert_called_once_with(enable_commenter=False) + + +# --------------------------------------------------------------------------- +# Defaults / overrides +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_applies_defaults_for_unset_otel_vars(monkeypatch): + _full_enabled_env(monkeypatch) + _patched_configure_otel() + + assert os.environ.get("OTEL_SERVICE_NAME") == "plane-api" + assert os.environ.get("OTEL_TRACES_SAMPLER") == "parentbased_traceidratio" + assert os.environ.get("OTEL_TRACES_SAMPLER_ARG") == "0.1" + assert os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") == "grpc" + + +@pytest.mark.unit +def test_operator_overrides_are_preserved(monkeypatch): + _full_enabled_env(monkeypatch) + monkeypatch.setenv("OTEL_SERVICE_NAME", "custom-name") + monkeypatch.setenv("OTEL_TRACES_SAMPLER_ARG", "0.5") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + + _patched_configure_otel() + + assert os.environ["OTEL_SERVICE_NAME"] == "custom-name" + assert os.environ["OTEL_TRACES_SAMPLER_ARG"] == "0.5" + assert os.environ["OTEL_EXPORTER_OTLP_PROTOCOL"] == "http/protobuf" + + +# --------------------------------------------------------------------------- +# Quiet loggers +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_quiet_otel_loggers_sets_warning_level(monkeypatch): + _full_enabled_env(monkeypatch) + _patched_configure_otel() + + assert logging.getLogger("opentelemetry").level == logging.WARNING + assert logging.getLogger("opentelemetry.exporter.otlp").level == logging.WARNING + assert logging.getLogger("opentelemetry.instrumentation").level == logging.WARNING + + +# --------------------------------------------------------------------------- +# Idempotency +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_idempotent_when_called_twice(monkeypatch): + _full_enabled_env(monkeypatch) + + stack = ExitStack() + mocks = {target: stack.enter_context(patch(target)) for target in _MOCK_TARGETS} + try: + configure_otel() + configure_otel() + finally: + stack.close() + + assert mocks["plane.observability.setup._create_span_exporter"].call_count == 1 + assert mocks["opentelemetry.trace.set_tracer_provider"].call_count == 1 + assert ( + mocks["plane.observability.setup.DjangoInstrumentor"] + .return_value.instrument.call_count + == 1 + ) + + +@pytest.mark.unit +def test_idempotent_when_disabled(): + # When disabled, configure_otel returns BEFORE flipping _CONFIGURED. + # Calling twice is still a no-op and the flag stays False. + from plane.observability import setup as otel_setup + + configure_otel() + configure_otel() + assert otel_setup._CONFIGURED is False + + +# --------------------------------------------------------------------------- +# Protocol-aware exporter selection +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_create_span_exporter_uses_grpc_by_default(): + with patch( + "opentelemetry.exporter.otlp.proto.grpc.trace_exporter.OTLPSpanExporter" + ) as Grpc, patch( + "opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter" + ) as Http: + _create_span_exporter() + Grpc.assert_called_once() + Http.assert_not_called() + + +@pytest.mark.unit +@pytest.mark.parametrize("protocol", ["http/protobuf", "http", "HTTP", "Http/Protobuf"]) +def test_create_span_exporter_uses_http_when_protocol_is_http(protocol, monkeypatch): + monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", protocol) + with patch( + "opentelemetry.exporter.otlp.proto.grpc.trace_exporter.OTLPSpanExporter" + ) as Grpc, patch( + "opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter" + ) as Http: + _create_span_exporter() + Http.assert_called_once() + Grpc.assert_not_called() + + +@pytest.mark.unit +def test_create_metric_exporter_uses_grpc_by_default(): + with patch( + "opentelemetry.exporter.otlp.proto.grpc.metric_exporter.OTLPMetricExporter" + ) as Grpc, patch( + "opentelemetry.exporter.otlp.proto.http.metric_exporter.OTLPMetricExporter" + ) as Http: + _create_metric_exporter() + Grpc.assert_called_once() + Http.assert_not_called() + + +@pytest.mark.unit +@pytest.mark.parametrize("protocol", ["http/protobuf", "http"]) +def test_create_metric_exporter_uses_http_when_protocol_is_http(protocol, monkeypatch): + monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", protocol) + with patch( + "opentelemetry.exporter.otlp.proto.grpc.metric_exporter.OTLPMetricExporter" + ) as Grpc, patch( + "opentelemetry.exporter.otlp.proto.http.metric_exporter.OTLPMetricExporter" + ) as Http: + _create_metric_exporter() + Http.assert_called_once() + Grpc.assert_not_called() diff --git a/apps/api/requirements/base.txt b/apps/api/requirements/base.txt index 89b15694f39..832b2e0511a 100644 --- a/apps/api/requirements/base.txt +++ b/apps/api/requirements/base.txt @@ -71,6 +71,16 @@ opentelemetry-sdk==1.28.1 opentelemetry-instrumentation-django==0.49b1 opentelemetry-exporter-otlp==1.28.1 opentelemetry-exporter-otlp-proto-grpc==1.28.1 +# Required for http.server.* spans + metrics on the ASGI serving path (uvicorn worker). +# DjangoInstrumentor declares it only as an optional ASGI extra; without it +# _is_asgi_supported=False and every ASGI request is skipped (no server span/metric). +opentelemetry-instrumentation-asgi==0.49b1 +opentelemetry-exporter-otlp-proto-http==1.28.1 +opentelemetry-instrumentation-celery==0.49b1 +opentelemetry-instrumentation-psycopg==0.49b1 +opentelemetry-instrumentation-redis==0.49b1 +opentelemetry-instrumentation-requests==0.49b1 +opentelemetry-instrumentation-httpx==0.49b1 # OpenAPI Specification drf-spectacular==0.28.0 # html sanitizer From cd3f0042fb46cac1b412fd6cde3e61463a22ddb2 Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Tue, 14 Jul 2026 01:10:26 +0530 Subject: [PATCH 2/7] feat(api/observability): bootstrap OpenTelemetry in wsgi/asgi/manage entry points --- apps/api/manage.py | 7 +++++++ apps/api/plane/asgi.py | 14 ++++++++------ apps/api/plane/wsgi.py | 10 ++++++++-- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/apps/api/manage.py b/apps/api/manage.py index a79268b37c8..6f69f442529 100644 --- a/apps/api/manage.py +++ b/apps/api/manage.py @@ -8,6 +8,13 @@ if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production") + + # Bootstrap OpenTelemetry before Django imports so runserver and + # management commands are instrumented too. No-op unless OTEL_ENABLED=1. + from plane.observability.setup import configure_otel + + configure_otel() + try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/apps/api/plane/asgi.py b/apps/api/plane/asgi.py index 9d3bd6b07a0..e7db9b2d312 100644 --- a/apps/api/plane/asgi.py +++ b/apps/api/plane/asgi.py @@ -4,15 +4,17 @@ import os -from channels.routing import ProtocolTypeRouter -from django.core.asgi import get_asgi_application +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production") -django_asgi_app = get_asgi_application() +# Bootstrap OpenTelemetry before Django wires up so DjangoInstrumentor can +# patch the ASGI handler. No-op unless OTEL_ENABLED=1. +from plane.observability.setup import configure_otel # noqa: E402 +configure_otel() + +from channels.routing import ProtocolTypeRouter # noqa: E402 +from django.core.asgi import get_asgi_application # noqa: E402 -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production") # Initialize Django ASGI application early to ensure the AppRegistry # is populated before importing code that may import ORM models. - - application = ProtocolTypeRouter({"http": get_asgi_application()}) diff --git a/apps/api/plane/wsgi.py b/apps/api/plane/wsgi.py index 4c8a7916364..1b4faf48738 100644 --- a/apps/api/plane/wsgi.py +++ b/apps/api/plane/wsgi.py @@ -11,8 +11,14 @@ import os -from django.core.wsgi import get_wsgi_application - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production") +# Bootstrap OpenTelemetry before Django wires up so DjangoInstrumentor can +# patch the WSGI handler. No-op unless OTEL_ENABLED=1. +from plane.observability.setup import configure_otel # noqa: E402 + +configure_otel() + +from django.core.wsgi import get_wsgi_application # noqa: E402 + application = get_wsgi_application() From bd331a64d3a28e63f0c1eb1dd7d19f8cb6839848 Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Tue, 14 Jul 2026 01:11:32 +0530 Subject: [PATCH 3/7] feat(api/observability): instrument Celery workers + trace-correlate worker logs --- apps/api/plane/celery.py | 64 ++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/apps/api/plane/celery.py b/apps/api/plane/celery.py index 8ae7c7b7051..3a1b8614900 100644 --- a/apps/api/plane/celery.py +++ b/apps/api/plane/celery.py @@ -10,7 +10,11 @@ # Third party imports from celery import Celery from pythonjsonlogger.json import JsonFormatter -from celery.signals import after_setup_logger, after_setup_task_logger +from celery.signals import ( + after_setup_logger, + after_setup_task_logger, + worker_process_shutdown, +) from celery.schedules import crontab, schedule # Module imports @@ -19,6 +23,54 @@ # Set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production") +# Bootstrap OpenTelemetry before Celery wires up so CeleryInstrumentor can +# patch task execution. No-op unless OTEL_ENABLED=1. +from plane.observability.setup import configure_otel, flush_otel # noqa: E402 +from plane.observability.logging import TraceContextFilter # noqa: E402 + +configure_otel() + +# Whether to trace-correlate worker logs. Matches the Django LOGGING gate in +# plane/settings/{local,production}.py; the bootstrap in configure_otel() uses +# its own (superset) token check. +_OTEL_LOG_ENABLED = os.environ.get("OTEL_ENABLED", "0").strip().lower() in ("1", "true", "yes") + +# Base JSON log fmt (unchanged off-path); the OTel variant appends the +# trace-context fields that TraceContextFilter populates. +_CELERY_LOG_FMT = '"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s' +_CELERY_OTEL_LOG_FMT = ( + '"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s ' + "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s" +) + + +@worker_process_shutdown.connect +def flush_otel_on_worker_shutdown(*args, **kwargs): + """Flush buffered spans/metrics when a prefork child exits. + + Prefork children exit via os._exit and skip atexit, so without this the tail + of each child's telemetry is dropped on --max-tasks-per-child recycling and + warm shutdown. No-op (bounded, never raises) unless OTel was configured. + """ + flush_otel() + + +def _build_celery_log_handler() -> logging.Handler: + """Build the worker's JSON StreamHandler. + + Off-path: identical to the historical handler (same fmt). When OTel logging + is enabled, use the extended fmt and attach TraceContextFilter so worker + log lines carry trace_id/span_id/service_name, matching the Django request + path. + """ + fmt = _CELERY_OTEL_LOG_FMT if _OTEL_LOG_ENABLED else _CELERY_LOG_FMT + handler = logging.StreamHandler() + handler.setFormatter(fmt=JsonFormatter(fmt)) + if _OTEL_LOG_ENABLED: + handler.addFilter(TraceContextFilter()) + return handler + + ri = redis_instance() # Configurable metrics push interval (in minutes) @@ -98,18 +150,12 @@ def _get_metrics_push_interval_minutes() -> int: # Setup logging @after_setup_logger.connect def setup_loggers(logger, *args, **kwargs): - formatter = JsonFormatter('"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s') - handler = logging.StreamHandler() - handler.setFormatter(fmt=formatter) - logger.addHandler(handler) + logger.addHandler(_build_celery_log_handler()) @after_setup_task_logger.connect def setup_task_loggers(logger, *args, **kwargs): - formatter = JsonFormatter('"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s') - handler = logging.StreamHandler() - handler.setFormatter(fmt=formatter) - logger.addHandler(handler) + logger.addHandler(_build_celery_log_handler()) # Load task modules from all registered Django app configs. From 5514160067ecc5d3d2fefec1e3f703b6e022447a Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Tue, 14 Jul 2026 01:12:47 +0530 Subject: [PATCH 4/7] feat(api/observability): trace-correlate JSON logs via LOGGING dict when OTEL_ENABLED --- apps/api/plane/settings/local.py | 16 ++++++++++++++++ apps/api/plane/settings/production.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/apps/api/plane/settings/local.py b/apps/api/plane/settings/local.py index a346eca141b..cd726d8e33a 100644 --- a/apps/api/plane/settings/local.py +++ b/apps/api/plane/settings/local.py @@ -87,3 +87,19 @@ }, }, } + +# OpenTelemetry APM: only when OTEL_ENABLED=1 do we extend the JSON +# formatter and attach TraceContextFilter to every handler. Attaching at +# the handler level (rather than the root logger) is required because +# most plane.* loggers have propagate=False; runtime mutation also wouldn't +# survive Django's dictConfig. Off path leaves the log schema unchanged. +if os.environ.get("OTEL_ENABLED", "0").lower() in ("1", "true", "yes"): + LOGGING["formatters"]["json"]["fmt"] = ( + "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " + "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s" + ) + LOGGING["filters"] = { + "trace_context": {"()": "plane.observability.logging.TraceContextFilter"}, + } + for _handler in LOGGING["handlers"].values(): + _handler["filters"] = ["trace_context"] diff --git a/apps/api/plane/settings/production.py b/apps/api/plane/settings/production.py index 1dbf43196a8..d9c5ed781bc 100644 --- a/apps/api/plane/settings/production.py +++ b/apps/api/plane/settings/production.py @@ -97,3 +97,19 @@ }, }, } + +# OpenTelemetry APM: only when OTEL_ENABLED=1 do we extend the JSON +# formatter and attach TraceContextFilter to every handler. Attaching at +# the handler level (rather than the root logger) is required because +# most plane.* loggers have propagate=False; runtime mutation also wouldn't +# survive Django's dictConfig. Off path leaves the log schema unchanged. +if os.environ.get("OTEL_ENABLED", "0").lower() in ("1", "true", "yes"): + LOGGING["formatters"]["json"]["fmt"] = ( + "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " + "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s" + ) + LOGGING["filters"] = { + "trace_context": {"()": "plane.observability.logging.TraceContextFilter"}, + } + for _handler in LOGGING["handlers"].values(): + _handler["filters"] = ["trace_context"] From b8b5f5d932c1b6fb53531cab59edb83e971fc816 Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Tue, 14 Jul 2026 01:15:12 +0530 Subject: [PATCH 5/7] chore(deploy): wire OpenTelemetry env into community API deployment templates --- apps/api/.env.example | 19 +++++++++++++++++++ deployments/aio/community/variables.env | 9 +++++++++ deployments/cli/community/docker-compose.yml | 16 +++++++++++++--- deployments/cli/community/variables.env | 9 +++++++++ 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/apps/api/.env.example b/apps/api/.env.example index 4c84bd68373..ca517c32ebb 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -73,3 +73,22 @@ MINIO_ENDPOINT_SSL=0 # API key rate limit API_KEY_RATE_LIMIT="60/minute" + +# ---------------------------------------------------------------------------- +# OpenTelemetry APM (self-hoster observability). +# Off by default. Flip OTEL_ENABLED to 1 to opt in. +# Point at any OTLP-compatible collector (otel-collector, Jaeger, Tempo, +# Datadog Agent, Honeycomb, Grafana Cloud, etc.). +# See docs/otel-api-observability/README.md for the full guide. +# ---------------------------------------------------------------------------- +OTEL_ENABLED=0 +# OTEL_SERVICE_NAME=plane-api +# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +# OTEL_EXPORTER_OTLP_PROTOCOL=grpc +# OTEL_TRACES_SAMPLER=parentbased_traceidratio +# OTEL_TRACES_SAMPLER_ARG=0.1 +# Deployment environment tag. Set OTEL_ENVIRONMENT (falls back to +# SENTRY_ENVIRONMENT) and the API emits deployment.environment.name. +# OTEL_ENVIRONMENT=prod +# OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=prod,service.version=v0.34.0 +# OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20 diff --git a/deployments/aio/community/variables.env b/deployments/aio/community/variables.env index 0a74aa2a007..894e7c39d9b 100644 --- a/deployments/aio/community/variables.env +++ b/deployments/aio/community/variables.env @@ -72,3 +72,12 @@ WEBHOOK_ALLOWED_IPS= # SSRF check. Useful for trusted internal services whose container/service IPs are # dynamic (e.g. "silo,silo.namespace.svc.cluster.local") WEBHOOK_ALLOWED_HOSTS= + +# OpenTelemetry APM (self-hoster observability). Off by default. +# See docs/otel-api-observability/README.md. +OTEL_ENABLED=0 +# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +# OTEL_EXPORTER_OTLP_PROTOCOL=grpc +# OTEL_EXPORTER_OTLP_HEADERS= +# OTEL_SERVICE_NAME=plane-api +# OTEL_ENVIRONMENT=prod diff --git a/deployments/cli/community/docker-compose.yml b/deployments/cli/community/docker-compose.yml index cd41be85f0f..d82902374ce 100644 --- a/deployments/cli/community/docker-compose.yml +++ b/deployments/cli/community/docker-compose.yml @@ -61,6 +61,16 @@ x-app-env: &app-env WEBHOOK_ALLOWED_IPS: ${WEBHOOK_ALLOWED_IPS:-} WEBHOOK_ALLOWED_HOSTS: ${WEBHOOK_ALLOWED_HOSTS:-} +# OpenTelemetry APM for the Django API (api, worker, beat-worker). +# Read at container RUNTIME; off by default. See docs/otel-api-observability/README.md. +x-otel-env: &otel-env + OTEL_ENABLED: ${OTEL_ENABLED:-0} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-grpc} + OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS:-} + OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-plane-api} + OTEL_ENVIRONMENT: ${OTEL_ENVIRONMENT:-} + services: web: image: makeplane/plane-frontend:${APP_RELEASE:-stable} @@ -115,7 +125,7 @@ services: volumes: - logs_api:/code/plane/logs environment: - <<: [*app-env, *db-env, *redis-env, *minio-env, *aws-s3-env, *proxy-env] + <<: [*app-env, *db-env, *redis-env, *minio-env, *aws-s3-env, *proxy-env, *otel-env] depends_on: - plane-db - plane-redis @@ -131,7 +141,7 @@ services: volumes: - logs_worker:/code/plane/logs environment: - <<: [*app-env, *db-env, *redis-env, *minio-env, *aws-s3-env, *proxy-env] + <<: [*app-env, *db-env, *redis-env, *minio-env, *aws-s3-env, *proxy-env, *otel-env] depends_on: - api - plane-db @@ -148,7 +158,7 @@ services: volumes: - logs_beat-worker:/code/plane/logs environment: - <<: [*app-env, *db-env, *redis-env, *minio-env, *aws-s3-env, *proxy-env] + <<: [*app-env, *db-env, *redis-env, *minio-env, *aws-s3-env, *proxy-env, *otel-env] depends_on: - api - plane-db diff --git a/deployments/cli/community/variables.env b/deployments/cli/community/variables.env index f141d01fb7d..3c04a3a3c5e 100644 --- a/deployments/cli/community/variables.env +++ b/deployments/cli/community/variables.env @@ -96,3 +96,12 @@ WEBHOOK_ALLOWED_IPS= # SSRF check. Useful for trusted internal services whose container/service IPs are # dynamic (e.g. "silo,silo.namespace.svc.cluster.local") WEBHOOK_ALLOWED_HOSTS= + +# OpenTelemetry APM (self-hoster observability). Off by default. +# See docs/otel-api-observability/README.md. +OTEL_ENABLED=0 +# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +# OTEL_EXPORTER_OTLP_PROTOCOL=grpc +# OTEL_EXPORTER_OTLP_HEADERS= +# OTEL_SERVICE_NAME=plane-api +# OTEL_ENVIRONMENT=prod From 38306e3893afef625862e7477847c6792ed67087 Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Tue, 14 Jul 2026 01:15:45 +0530 Subject: [PATCH 6/7] docs(otel): self-hoster guide + reference collector config --- docs/otel-api-observability/README.md | 47 ++++++++++++++ .../otel-collector.yaml | 63 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 docs/otel-api-observability/README.md create mode 100644 docs/otel-api-observability/otel-collector.yaml diff --git a/docs/otel-api-observability/README.md b/docs/otel-api-observability/README.md new file mode 100644 index 00000000000..59397bad11c --- /dev/null +++ b/docs/otel-api-observability/README.md @@ -0,0 +1,47 @@ +# OpenTelemetry for Plane self-hosters + +Plane's API server can emit OpenTelemetry traces, HTTP metrics, and trace-correlated logs over OTLP. Point it at any OTEL-compatible backend (Jaeger, Tempo, Datadog Agent, Honeycomb, Grafana Cloud, …) and start debugging slow endpoints. + +## Quickstart + +1. Run an OTEL Collector pointing at your backend of choice. See [`otel-collector.yaml`](./otel-collector.yaml) in this folder for a starting config. +2. Set two environment variables on the API container (and, for task telemetry, the worker and beat-worker containers): + ```bash + OTEL_ENABLED=1 + OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 + ``` +3. Restart the API/worker containers. That's it. + +## What you get + +- A server span per HTTP request, with `http.route`, `http.method`, `http.status_code`, `http.target`, and duration. +- A span per Celery task with `celery.action` / `celery.task_name` / `celery.state`. Traceparent is propagated through the queue, so a request that enqueues a task is linked to that task's execution span in the same trace. +- Child spans for every Postgres query, Redis op, and outbound `requests` / `httpx` call inside that request or task. +- HTTP metrics: `http.server.duration` histogram, `http.server.active_requests`, `http.server.request.size`, `http.server.response.size`. +- JSON logs on stdout with `trace_id` / `span_id` / `service_name` fields **added only when `OTEL_ENABLED=1`**. Off path leaves the existing log schema untouched. Point the collector's `filelog` receiver at your container log directory to link logs ↔ traces. + +## Environment variables + +| Var | Default | Purpose | +| ----------------------------- | -------------------------- | ----------------------------------------------------------------------- | +| `OTEL_ENABLED` | `0` | Plane gate. Must be `1` (or `true`/`yes`/`on`). | +| `OTEL_SERVICE_NAME` | `plane-api` | Service identifier in your APM backend | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | _(required)_ | Your collector's OTLP receiver | +| `OTEL_EXPORTER_OTLP_PROTOCOL` | `grpc` | `grpc` or `http/protobuf` | +| `OTEL_EXPORTER_OTLP_HEADERS` | _(unset)_ | For SaaS backends needing auth headers | +| `OTEL_ENVIRONMENT` | _(unset)_ | Sets `deployment.environment.name` (falls back to `SENTRY_ENVIRONMENT`) | +| `OTEL_TRACES_SAMPLER` | `parentbased_traceidratio` | Standard OTEL sampler | +| `OTEL_TRACES_SAMPLER_ARG` | `0.1` | 10 % head sampling. Set to `1.0` to capture every request. | +| `OTEL_RESOURCE_ATTRIBUTES` | _(unset)_ | Extra resource attrs: `service.version=...` | + +If `OTEL_ENABLED=1` but `OTEL_EXPORTER_OTLP_ENDPOINT` is unset, the API logs a single WARNING at boot and continues without instrumentation — no silent local-host default. + +## What's not instrumented yet + +- The `live` (Node.js) collaboration server. It has no OTEL bootstrap yet and isn't covered by this Django-side setup. + +## Troubleshooting + +- **No spans showing up.** Confirm `OTEL_ENABLED=1` is in the API container's env, not just the host shell. Check API logs for the `OpenTelemetry configured` INFO line at boot. +- **`connection refused` floods stop appearing.** Good — they were dropped batches. The `opentelemetry.*` loggers are pinned to WARNING but the underlying gRPC retries still happen. Fix the collector reachability or unset `OTEL_ENABLED`. +- **`trace_id` is empty in logs.** Either you're outside a request/task or the sampler dropped the trace. Drop `OTEL_TRACES_SAMPLER_ARG` to `1.0` while debugging. diff --git a/docs/otel-api-observability/otel-collector.yaml b/docs/otel-api-observability/otel-collector.yaml new file mode 100644 index 00000000000..6d58f08a290 --- /dev/null +++ b/docs/otel-api-observability/otel-collector.yaml @@ -0,0 +1,63 @@ +# Reference OpenTelemetry Collector config for Plane self-hosters. +# Run with: otelcol --config=otel-collector.yaml +# Replace the `debug` exporter with one targeting your backend. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + filelog: + include: + - /var/lib/docker/containers/*/*-json.log + operators: + - type: json_parser + - type: move + from: attributes.trace_id + to: trace_id + - type: move + from: attributes.span_id + to: span_id + +processors: + batch: + timeout: 10s + send_batch_size: 1024 + +exporters: + # Replace `debug` with your real backend. Examples (commented): + # + # otlphttp/tempo: + # endpoint: http://tempo:4318 + # + # datadog: + # api: + # key: ${DD_API_KEY} + # + # otlphttp/honeycomb: + # endpoint: https://api.honeycomb.io + # headers: + # x-honeycomb-team: ${HONEYCOMB_API_KEY} + # + # prometheusremotewrite: + # endpoint: https://prometheus.example.com/api/v1/write + debug: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [debug] + logs: + receivers: [filelog] + processors: [batch] + exporters: [debug] From 9a7d84076156cc3f159d71cef652325a796042a2 Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Tue, 14 Jul 2026 13:27:40 +0530 Subject: [PATCH 7/7] refactor(api/observability): unify OTEL_ENABLED gate, isolate instrumentors, pass sampler vars through compose - Extract shared is_otel_enabled() + extend_logging_config() into plane.observability.logging so the bootstrap, Celery, and Django LOGGING gates all accept the documented tokens (incl. 'on') identically. - Isolate each instrumentor in _instrument_libraries() so one failure can't block startup. - Pass OTEL_TRACES_SAMPLER / _ARG / OTEL_RESOURCE_ATTRIBUTES through the x-otel-env compose anchor (documented in .env.example). - Reset _TRACER_PROVIDER/_METER_PROVIDER in the test fixture. --- apps/api/plane/celery.py | 10 ++--- apps/api/plane/observability/logging.py | 39 +++++++++++++++++++ apps/api/plane/observability/setup.py | 35 ++++++++++------- apps/api/plane/settings/local.py | 20 +++------- apps/api/plane/settings/production.py | 20 +++------- .../tests/unit/observability/conftest.py | 2 + deployments/cli/community/docker-compose.yml | 3 ++ 7 files changed, 79 insertions(+), 50 deletions(-) diff --git a/apps/api/plane/celery.py b/apps/api/plane/celery.py index 3a1b8614900..92215c7c80a 100644 --- a/apps/api/plane/celery.py +++ b/apps/api/plane/celery.py @@ -26,14 +26,14 @@ # Bootstrap OpenTelemetry before Celery wires up so CeleryInstrumentor can # patch task execution. No-op unless OTEL_ENABLED=1. from plane.observability.setup import configure_otel, flush_otel # noqa: E402 -from plane.observability.logging import TraceContextFilter # noqa: E402 +from plane.observability.logging import TraceContextFilter, is_otel_enabled # noqa: E402 configure_otel() -# Whether to trace-correlate worker logs. Matches the Django LOGGING gate in -# plane/settings/{local,production}.py; the bootstrap in configure_otel() uses -# its own (superset) token check. -_OTEL_LOG_ENABLED = os.environ.get("OTEL_ENABLED", "0").strip().lower() in ("1", "true", "yes") +# Whether to trace-correlate worker logs. Uses the same shared is_otel_enabled() +# gate as the bootstrap (setup.configure_otel) and the Django LOGGING gate, so +# every documented OTEL_ENABLED token behaves identically across processes. +_OTEL_LOG_ENABLED = is_otel_enabled() # Base JSON log fmt (unchanged off-path); the OTel variant appends the # trace-context fields that TraceContextFilter populates. diff --git a/apps/api/plane/observability/logging.py b/apps/api/plane/observability/logging.py index c57c015679a..9e260c9d0b5 100644 --- a/apps/api/plane/observability/logging.py +++ b/apps/api/plane/observability/logging.py @@ -19,6 +19,21 @@ from opentelemetry import trace +# Accepted OTEL_ENABLED tokens — the single source of truth shared by the +# bootstrap gate (setup.configure_otel), the Celery worker-log gate, and the +# Django LOGGING gate, so enabling via any documented token (see the README) +# behaves identically everywhere. +_TRUTHY_VALUES = ("1", "true", "yes", "on") + +# Trace-context fields appended to the JSON log formatter when OTel is enabled; +# populated by TraceContextFilter. +_TRACE_LOG_FIELDS = "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s" + + +def is_otel_enabled() -> bool: + """Return True when OTEL_ENABLED is set to a recognized truthy token.""" + return os.environ.get("OTEL_ENABLED", "0").strip().lower() in _TRUTHY_VALUES + class TraceContextFilter(logging.Filter): """Inject trace_id, span_id, trace_flags, service_name into LogRecord.""" @@ -35,3 +50,27 @@ def filter(self, record: logging.LogRecord) -> bool: record.trace_flags = 0 record.service_name = os.environ.get("OTEL_SERVICE_NAME", "plane-api") return True + + +def extend_logging_config(logging_config: dict) -> None: + """Trace-correlate a Django LOGGING dict in place. No-op unless OTel is enabled. + + When enabled, extends the JSON formatter's fmt with the trace-context fields + and attaches TraceContextFilter to every handler. Attaching at the handler + level (rather than the root logger) is required because most plane.* loggers + set propagate=False; runtime mutation also wouldn't survive Django's + dictConfig. The off path leaves the log schema byte-for-byte unchanged. + + Shared by settings/local.py and settings/production.py so the two stay in + lockstep. + """ + if not is_otel_enabled(): + return + logging_config["formatters"]["json"]["fmt"] = ( + "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " + _TRACE_LOG_FIELDS + ) + logging_config.setdefault("filters", {})["trace_context"] = { + "()": "plane.observability.logging.TraceContextFilter", + } + for handler in logging_config["handlers"].values(): + handler["filters"] = ["trace_context"] diff --git a/apps/api/plane/observability/setup.py b/apps/api/plane/observability/setup.py index 55e6559acc8..09790c29964 100644 --- a/apps/api/plane/observability/setup.py +++ b/apps/api/plane/observability/setup.py @@ -24,6 +24,8 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from plane.observability.logging import is_otel_enabled + logger = logging.getLogger(__name__) _CONFIGURED = False @@ -33,10 +35,6 @@ _TRACER_PROVIDER: "TracerProvider | None" = None _METER_PROVIDER: "MeterProvider | None" = None -# Accepted "on" tokens — kept in sync with the pi/node services so OTEL_ENABLED -# behaves identically across every runtime. -_TRUTHY_VALUES = ("1", "true", "yes", "on") - _NOISY_OTEL_LOGGERS = ( "opentelemetry", "opentelemetry.exporter.otlp", @@ -49,10 +47,6 @@ _HTTP_PROTOCOLS = ("http/protobuf", "http") -def _is_enabled() -> bool: - return os.environ.get("OTEL_ENABLED", "0").strip().lower() in _TRUTHY_VALUES - - def _has_endpoint() -> bool: return bool(os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()) @@ -140,13 +134,24 @@ def _instrument_libraries() -> None: request that enqueues a task is linked to that task's execution span. The others add child spans so latency can be decomposed (SQL query, Redis op, outbound HTTP). + + Each instrumentor is isolated: a single failure (version mismatch, missing + optional dependency) is logged and skipped so it neither blocks the other + instrumentors nor takes down application startup. """ - DjangoInstrumentor().instrument() - CeleryInstrumentor().instrument() - PsycopgInstrumentor().instrument(enable_commenter=False) - RedisInstrumentor().instrument() - RequestsInstrumentor().instrument() - HTTPXClientInstrumentor().instrument() + instrumentors = ( + (DjangoInstrumentor, {}), + (CeleryInstrumentor, {}), + (PsycopgInstrumentor, {"enable_commenter": False}), + (RedisInstrumentor, {}), + (RequestsInstrumentor, {}), + (HTTPXClientInstrumentor, {}), + ) + for instrumentor_cls, kwargs in instrumentors: + try: + instrumentor_cls().instrument(**kwargs) + except Exception as exc: + logger.warning("Failed to instrument %s: %s", instrumentor_cls.__name__, exc) def _quiet_otel_loggers() -> None: @@ -172,7 +177,7 @@ def configure_otel() -> None: global _CONFIGURED if _CONFIGURED: return - if not _is_enabled(): + if not is_otel_enabled(): return if not _has_endpoint(): logger.warning("OTEL_ENABLED=1 but OTEL_EXPORTER_OTLP_ENDPOINT is not set; OpenTelemetry bootstrap skipped") diff --git a/apps/api/plane/settings/local.py b/apps/api/plane/settings/local.py index cd726d8e33a..7148011a1c1 100644 --- a/apps/api/plane/settings/local.py +++ b/apps/api/plane/settings/local.py @@ -88,18 +88,8 @@ }, } -# OpenTelemetry APM: only when OTEL_ENABLED=1 do we extend the JSON -# formatter and attach TraceContextFilter to every handler. Attaching at -# the handler level (rather than the root logger) is required because -# most plane.* loggers have propagate=False; runtime mutation also wouldn't -# survive Django's dictConfig. Off path leaves the log schema unchanged. -if os.environ.get("OTEL_ENABLED", "0").lower() in ("1", "true", "yes"): - LOGGING["formatters"]["json"]["fmt"] = ( - "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " - "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s" - ) - LOGGING["filters"] = { - "trace_context": {"()": "plane.observability.logging.TraceContextFilter"}, - } - for _handler in LOGGING["handlers"].values(): - _handler["filters"] = ["trace_context"] +# OpenTelemetry APM: trace-correlate the JSON logs when OTEL is enabled. +# No-op (log schema unchanged) otherwise. Shared with settings/production.py. +from plane.observability.logging import extend_logging_config # noqa: E402 + +extend_logging_config(LOGGING) diff --git a/apps/api/plane/settings/production.py b/apps/api/plane/settings/production.py index d9c5ed781bc..0702d841918 100644 --- a/apps/api/plane/settings/production.py +++ b/apps/api/plane/settings/production.py @@ -98,18 +98,8 @@ }, } -# OpenTelemetry APM: only when OTEL_ENABLED=1 do we extend the JSON -# formatter and attach TraceContextFilter to every handler. Attaching at -# the handler level (rather than the root logger) is required because -# most plane.* loggers have propagate=False; runtime mutation also wouldn't -# survive Django's dictConfig. Off path leaves the log schema unchanged. -if os.environ.get("OTEL_ENABLED", "0").lower() in ("1", "true", "yes"): - LOGGING["formatters"]["json"]["fmt"] = ( - "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " - "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s" - ) - LOGGING["filters"] = { - "trace_context": {"()": "plane.observability.logging.TraceContextFilter"}, - } - for _handler in LOGGING["handlers"].values(): - _handler["filters"] = ["trace_context"] +# OpenTelemetry APM: trace-correlate the JSON logs when OTEL is enabled. +# No-op (log schema unchanged) otherwise. Shared with settings/local.py. +from plane.observability.logging import extend_logging_config # noqa: E402 + +extend_logging_config(LOGGING) diff --git a/apps/api/plane/tests/unit/observability/conftest.py b/apps/api/plane/tests/unit/observability/conftest.py index f140e7a9d2f..023f1abc4b4 100644 --- a/apps/api/plane/tests/unit/observability/conftest.py +++ b/apps/api/plane/tests/unit/observability/conftest.py @@ -25,6 +25,8 @@ def isolate_otel_state(monkeypatch): from plane.observability import setup as otel_setup monkeypatch.setattr(otel_setup, "_CONFIGURED", False, raising=False) + monkeypatch.setattr(otel_setup, "_TRACER_PROVIDER", None, raising=False) + monkeypatch.setattr(otel_setup, "_METER_PROVIDER", None, raising=False) saved = {k: v for k, v in os.environ.items() if k.startswith("OTEL_")} for key in list(saved.keys()): diff --git a/deployments/cli/community/docker-compose.yml b/deployments/cli/community/docker-compose.yml index d82902374ce..7c613cc3bd5 100644 --- a/deployments/cli/community/docker-compose.yml +++ b/deployments/cli/community/docker-compose.yml @@ -70,6 +70,9 @@ x-otel-env: &otel-env OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS:-} OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-plane-api} OTEL_ENVIRONMENT: ${OTEL_ENVIRONMENT:-} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-} + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-} services: web: