feat: OpenTelemetry (OTel) observability for the community API

apps/api/.env.example

@@ -73,3 +73,22 @@ MINIO_ENDPOINT_SSL=0
 
 # API key rate limit
 API_KEY_RATE_LIMIT="60/minute"
+
+# ----------------------------------------------------------------------------
+# OpenTelemetry APM (self-hoster observability).
+# Off by default. Flip OTEL_ENABLED to 1 to opt in.
+# Point at any OTLP-compatible collector (otel-collector, Jaeger, Tempo,
+# Datadog Agent, Honeycomb, Grafana Cloud, etc.).
+# See docs/otel-api-observability/README.md for the full guide.
+# ----------------------------------------------------------------------------
+OTEL_ENABLED=0
+# OTEL_SERVICE_NAME=plane-api
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+# OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+# OTEL_TRACES_SAMPLER=parentbased_traceidratio
+# OTEL_TRACES_SAMPLER_ARG=0.1
+# Deployment environment tag. Set OTEL_ENVIRONMENT (falls back to
+# SENTRY_ENVIRONMENT) and the API emits deployment.environment.name.
+# OTEL_ENVIRONMENT=prod
+# OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=prod,service.version=v0.34.0
+# OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20<token>

apps/api/manage.py

@@ -8,6 +8,13 @@
 
 if __name__ == "__main__":
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production")
+
+    # Bootstrap OpenTelemetry before Django imports so runserver and
+    # management commands are instrumented too. No-op unless OTEL_ENABLED=1.
+    from plane.observability.setup import configure_otel
+
+    configure_otel()
+
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

apps/api/plane/asgi.py

@@ -4,15 +4,17 @@
 
 import os
 
-from channels.routing import ProtocolTypeRouter
-from django.core.asgi import get_asgi_application
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production")
 
-django_asgi_app = get_asgi_application()
+# Bootstrap OpenTelemetry before Django wires up so DjangoInstrumentor can
+# patch the ASGI handler. No-op unless OTEL_ENABLED=1.
+from plane.observability.setup import configure_otel  # noqa: E402
 
+configure_otel()
+
+from channels.routing import ProtocolTypeRouter  # noqa: E402
+from django.core.asgi import get_asgi_application  # noqa: E402
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production")
 # Initialize Django ASGI application early to ensure the AppRegistry
 # is populated before importing code that may import ORM models.
-
-
 application = ProtocolTypeRouter({"http": get_asgi_application()})

apps/api/plane/celery.py

@@ -10,7 +10,11 @@
 # Third party imports
 from celery import Celery
 from pythonjsonlogger.json import JsonFormatter
-from celery.signals import after_setup_logger, after_setup_task_logger
+from celery.signals import (
+    after_setup_logger,
+    after_setup_task_logger,
+    worker_process_shutdown,
+)
 from celery.schedules import crontab, schedule
 
 # Module imports
@@ -19,6 +23,54 @@
 # Set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production")
 
+# Bootstrap OpenTelemetry before Celery wires up so CeleryInstrumentor can
+# patch task execution. No-op unless OTEL_ENABLED=1.
+from plane.observability.setup import configure_otel, flush_otel  # noqa: E402
+from plane.observability.logging import TraceContextFilter, is_otel_enabled  # noqa: E402
+
+configure_otel()
+
+# Whether to trace-correlate worker logs. Uses the same shared is_otel_enabled()
+# gate as the bootstrap (setup.configure_otel) and the Django LOGGING gate, so
+# every documented OTEL_ENABLED token behaves identically across processes.
+_OTEL_LOG_ENABLED = is_otel_enabled()
+
+# Base JSON log fmt (unchanged off-path); the OTel variant appends the
+# trace-context fields that TraceContextFilter populates.
+_CELERY_LOG_FMT = '"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s'
+_CELERY_OTEL_LOG_FMT = (
+    '"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s '
+    "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s"
+)
+
+
+@worker_process_shutdown.connect
+def flush_otel_on_worker_shutdown(*args, **kwargs):
+    """Flush buffered spans/metrics when a prefork child exits.
+
+    Prefork children exit via os._exit and skip atexit, so without this the tail
+    of each child's telemetry is dropped on --max-tasks-per-child recycling and
+    warm shutdown. No-op (bounded, never raises) unless OTel was configured.
+    """
+    flush_otel()
+
+
+def _build_celery_log_handler() -> logging.Handler:
+    """Build the worker's JSON StreamHandler.
+
+    Off-path: identical to the historical handler (same fmt). When OTel logging
+    is enabled, use the extended fmt and attach TraceContextFilter so worker
+    log lines carry trace_id/span_id/service_name, matching the Django request
+    path.
+    """
+    fmt = _CELERY_OTEL_LOG_FMT if _OTEL_LOG_ENABLED else _CELERY_LOG_FMT
+    handler = logging.StreamHandler()
+    handler.setFormatter(fmt=JsonFormatter(fmt))
+    if _OTEL_LOG_ENABLED:
+        handler.addFilter(TraceContextFilter())
+    return handler
+
+
 ri = redis_instance()
 
 # Configurable metrics push interval (in minutes)
@@ -98,18 +150,12 @@ def _get_metrics_push_interval_minutes() -> int:
 # Setup logging
 @after_setup_logger.connect
 def setup_loggers(logger, *args, **kwargs):
-    formatter = JsonFormatter('"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s')
-    handler = logging.StreamHandler()
-    handler.setFormatter(fmt=formatter)
-    logger.addHandler(handler)
+    logger.addHandler(_build_celery_log_handler())
 
 
 @after_setup_task_logger.connect
 def setup_task_loggers(logger, *args, **kwargs):
-    formatter = JsonFormatter('"%(levelname)s %(asctime)s %(module)s %(name)s %(message)s')
-    handler = logging.StreamHandler()
-    handler.setFormatter(fmt=formatter)
-    logger.addHandler(handler)
+    logger.addHandler(_build_celery_log_handler())
 
 
 # Load task modules from all registered Django app configs.

apps/api/plane/observability/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2023-present Plane Software, Inc. and contributors
+# SPDX-License-Identifier: AGPL-3.0-only
+# See the LICENSE file for details.

apps/api/plane/observability/logging.py

@@ -0,0 +1,76 @@
+# Copyright (c) 2023-present Plane Software, Inc. and contributors
+# SPDX-License-Identifier: AGPL-3.0-only
+# See the LICENSE file for details.
+
+"""Trace-context log correlation for the API.
+
+`TraceContextFilter` adds trace_id / span_id / trace_flags / service_name
+attributes to every LogRecord. Empty values when no span is active.
+
+The filter is wired in via Django's LOGGING dict (see
+plane/settings/local.py and plane/settings/production.py) and attached to
+every handler, so it works regardless of logger propagation settings.
+configure_otel() does not install it at runtime — Django's dictConfig
+would wipe a runtime-installed filter when settings are applied.
+"""
+
+import logging
+import os
+
+from opentelemetry import trace
+
+# Accepted OTEL_ENABLED tokens — the single source of truth shared by the
+# bootstrap gate (setup.configure_otel), the Celery worker-log gate, and the
+# Django LOGGING gate, so enabling via any documented token (see the README)
+# behaves identically everywhere.
+_TRUTHY_VALUES = ("1", "true", "yes", "on")
+
+# Trace-context fields appended to the JSON log formatter when OTel is enabled;
+# populated by TraceContextFilter.
+_TRACE_LOG_FIELDS = "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s"
+
+
+def is_otel_enabled() -> bool:
+    """Return True when OTEL_ENABLED is set to a recognized truthy token."""
+    return os.environ.get("OTEL_ENABLED", "0").strip().lower() in _TRUTHY_VALUES
+
+
+class TraceContextFilter(logging.Filter):
+    """Inject trace_id, span_id, trace_flags, service_name into LogRecord."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        ctx = trace.get_current_span().get_span_context()
+        if ctx.is_valid:
+            record.trace_id = format(ctx.trace_id, "032x")
+            record.span_id = format(ctx.span_id, "016x")
+            record.trace_flags = int(ctx.trace_flags)
+        else:
+            record.trace_id = ""
+            record.span_id = ""
+            record.trace_flags = 0
+        record.service_name = os.environ.get("OTEL_SERVICE_NAME", "plane-api")
+        return True
+
+
+def extend_logging_config(logging_config: dict) -> None:
+    """Trace-correlate a Django LOGGING dict in place. No-op unless OTel is enabled.
+
+    When enabled, extends the JSON formatter's fmt with the trace-context fields
+    and attaches TraceContextFilter to every handler. Attaching at the handler
+    level (rather than the root logger) is required because most plane.* loggers
+    set propagate=False; runtime mutation also wouldn't survive Django's
+    dictConfig. The off path leaves the log schema byte-for-byte unchanged.
+
+    Shared by settings/local.py and settings/production.py so the two stay in
+    lockstep.
+    """
+    if not is_otel_enabled():
+        return
+    logging_config["formatters"]["json"]["fmt"] = (
+        "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " + _TRACE_LOG_FIELDS
+    )
+    logging_config.setdefault("filters", {})["trace_context"] = {
+        "()": "plane.observability.logging.TraceContextFilter",
+    }
+    for handler in logging_config["handlers"].values():
+        handler["filters"] = ["trace_context"]