diff --git a/.changelog/5384.changed b/.changelog/5384.changed new file mode 100644 index 00000000000..b45a47e9b78 --- /dev/null +++ b/.changelog/5384.changed @@ -0,0 +1 @@ +`opentelemetry-exporter-otlp-proto-http`: refactor to use shared opentelemetry-exporter-otlp-common and opentelemetry-exporter-http-transport packages diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/pyproject.toml b/exporter/opentelemetry-exporter-otlp-proto-http/pyproject.toml index db7915af4d7..205caf4a048 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/pyproject.toml +++ b/exporter/opentelemetry-exporter-otlp-proto-http/pyproject.toml @@ -31,6 +31,8 @@ dependencies = [ "opentelemetry-proto == 1.44.0.dev", "opentelemetry-sdk ~= 1.44.0.dev", "opentelemetry-exporter-otlp-proto-common == 1.44.0.dev", + "opentelemetry-exporter-otlp-common == 0.65b0.dev", + "opentelemetry-exporter-http-transport[requests,urllib3] == 0.65b0.dev", "requests ~= 2.7", "typing-extensions >= 4.5.0", ] diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_common/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_common/__init__.py index 57bd7ca065a..b4d902e4da1 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_common/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_common/__init__.py @@ -12,14 +12,6 @@ from opentelemetry.util._importlib_metadata import entry_points -def _is_retryable(resp: requests.Response) -> bool: - if resp.status_code == 408: - return True - if resp.status_code >= 500 and resp.status_code <= 599: - return True - return False - - def _load_session_from_envvar( cred_envvar: Literal[ "OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER", diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_internal.py b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_internal.py new file mode 100644 index 00000000000..666c1f30f61 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_internal.py @@ -0,0 +1,186 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import logging +import os +from collections.abc import Mapping +from typing import TYPE_CHECKING, Literal + +import requests + +from opentelemetry.exporter.http.transport._requests import ( + RequestsHTTPTransport, +) +from opentelemetry.exporter.http.transport._urllib3 import ( + Urllib3HTTPTransport, +) +from opentelemetry.exporter.otlp.common._http import ( + Compression as CommonCompression, +) +from opentelemetry.exporter.otlp.proto.http import ( + _OTLP_HTTP_HEADERS, + Compression, +) +from opentelemetry.exporter.otlp.proto.http._common import ( + _load_session_from_envvar, +) +from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_KEY, + OTEL_EXPORTER_OTLP_COMPRESSION, + OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_HEADERS, + OTEL_EXPORTER_OTLP_TIMEOUT, +) +from opentelemetry.util.re import parse_env_headers + +if TYPE_CHECKING: + from opentelemetry.exporter.http.transport._base import BaseHTTPTransport + +_logger = logging.getLogger(__name__) + +_DEFAULT_ENDPOINT = "http://localhost:4318" +_DEFAULT_TIMEOUT = 10 + +_FROM_LEGACY_COMPRESSION: dict[Compression, CommonCompression] = { + Compression.NoCompression: CommonCompression.NONE, + Compression.Deflate: CommonCompression.DEFLATE, + Compression.Gzip: CommonCompression.GZIP, +} + + +def _normalize_compression( + compression: Compression | CommonCompression | None, +) -> CommonCompression | None: + if compression is None: + return None + if isinstance(compression, Compression): + return _FROM_LEGACY_COMPRESSION[compression] + return compression + + +def _resolve_endpoint( + endpoint_env_var: str, + default_path: Literal["v1/traces", "v1/metrics", "v1/logs"], +) -> str: + if endpoint := os.environ.get(endpoint_env_var): + return endpoint + + base_endpoint = ( + os.environ.get(OTEL_EXPORTER_OTLP_ENDPOINT) or _DEFAULT_ENDPOINT + ) + + return f"{base_endpoint.removesuffix('/')}/{default_path}" + + +def _resolve_headers( + headers: Mapping[str, str] | None, + headers_env_var: str, +) -> dict[str, str]: + headers_ = {k.lower(): v for k, v in _OTLP_HTTP_HEADERS.items()} + env_headers = parse_env_headers( + os.environ.get(headers_env_var) + or os.environ.get(OTEL_EXPORTER_OTLP_HEADERS, ""), + liberal=True, + ) + headers_.update(env_headers) + if headers: + headers_.update({key.lower(): value for key, value in headers.items()}) + return headers_ + + +def _resolve_timeout( + timeout_env_var: str, +) -> float: + raw = ( + os.environ.get(timeout_env_var) + or os.environ.get(OTEL_EXPORTER_OTLP_TIMEOUT) + or _DEFAULT_TIMEOUT + ) + + try: + return float(raw) + except ValueError: + _logger.warning( + "Invalid timeout value %r, using default of %s seconds", + raw, + _DEFAULT_TIMEOUT, + ) + return float(_DEFAULT_TIMEOUT) + + +def _resolve_compression(compression_env_var: str) -> CommonCompression: + value = ( + ( + os.environ.get(compression_env_var) + or os.environ.get(OTEL_EXPORTER_OTLP_COMPRESSION) + or "none" + ) + .lower() + .strip() + ) + + try: + return CommonCompression.from_str(value) + except ValueError: + _logger.warning("Unsupported compression type: %s", value) + return CommonCompression.NONE + + +_CredentialEnvVar = Literal[ + "OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER", + "OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER", + "OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER", +] + + +def _resolve_session( + session: requests.Session | None, + credential_env_var: _CredentialEnvVar, +) -> requests.Session | None: + """Resolve the one canonical session used by both the exporter and its transport. + + Returns ``None`` when no explicit session was supplied and no credential + provider is configured, in which case the exporter falls back to a + lighter-weight urllib3-backed transport instead of a requests session. + """ + return session or _load_session_from_envvar(credential_env_var) + + +def _build_transport( + certificate_file: str | bool | None, + client_key_file: str | None, + client_certificate_file: str | None, + certificate_env_var: str, + client_key_env_var: str, + client_certificate_env_var: str, + session: requests.Session | None, +) -> BaseHTTPTransport: + verify: bool | str = ( + certificate_file + or os.environ.get(certificate_env_var) + or os.environ.get(OTEL_EXPORTER_OTLP_CERTIFICATE) + or True + ) + client_key_file = ( + client_key_file + or os.environ.get(client_key_env_var) + or os.environ.get(OTEL_EXPORTER_OTLP_CLIENT_KEY) + ) + client_certificate_file = ( + client_certificate_file + or os.environ.get(client_certificate_env_var) + or os.environ.get(OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE) + ) + cert = ( + (client_certificate_file, client_key_file) + if client_certificate_file and client_key_file + else client_certificate_file + ) + + if session is not None: + return RequestsHTTPTransport(verify=verify, cert=cert, session=session) + return Urllib3HTTPTransport(verify=verify, cert=cert) diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py index 56906b96501..411a1be4c4a 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/_log_exporter/__init__.py @@ -1,32 +1,33 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -import gzip +from __future__ import annotations + import logging import os -import random -import threading -import zlib -from collections.abc import Sequence -from io import BytesIO -from os import environ -from time import time +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, overload from urllib.parse import urlparse import requests -from requests.exceptions import ConnectionError +from opentelemetry.exporter.otlp.common._http import ( + Compression as CommonCompression, +) +from opentelemetry.exporter.otlp.common._http import OTLPHTTPClient from opentelemetry.exporter.otlp.proto.common._exporter_metrics import ( create_exporter_metrics, ) from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs -from opentelemetry.exporter.otlp.proto.http import ( - _OTLP_HTTP_HEADERS, - Compression, -) -from opentelemetry.exporter.otlp.proto.http._common import ( - _is_retryable, - _load_session_from_envvar, +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http._internal import ( + _build_transport, + _normalize_compression, + _resolve_compression, + _resolve_endpoint, + _resolve_headers, + _resolve_session, + _resolve_timeout, ) from opentelemetry.metrics import MeterProvider from opentelemetry.sdk._logs import ReadableLogRecord @@ -37,12 +38,6 @@ from opentelemetry.sdk._shared_internal import DuplicateFilter from opentelemetry.sdk.environment_variables import ( _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER, - OTEL_EXPORTER_OTLP_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION, - OTEL_EXPORTER_OTLP_ENDPOINT, - OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE, OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY, @@ -50,7 +45,6 @@ OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, OTEL_EXPORTER_OTLP_LOGS_HEADERS, OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, - OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, ) from opentelemetry.semconv._incubating.attributes.otel_attributes import ( @@ -59,7 +53,9 @@ from opentelemetry.semconv.attributes.http_attributes import ( HTTP_RESPONSE_STATUS_CODE, ) -from opentelemetry.util.re import parse_env_headers + +if TYPE_CHECKING: + from opentelemetry.exporter.http.transport._base import BaseHTTPTransport _logger = logging.getLogger(__name__) # This prevents logs generated when a log fails to be written to generate another log which fails to be written etc. etc. @@ -70,77 +66,83 @@ DEFAULT_ENDPOINT = "http://localhost:4318/" DEFAULT_LOGS_EXPORT_PATH = "v1/logs" DEFAULT_TIMEOUT = 10 # in seconds -_MAX_RETRYS = 6 class OTLPLogExporter(LogRecordExporter): + @overload def __init__( self, endpoint: str | None = None, certificate_file: str | None = None, client_key_file: str | None = None, client_certificate_file: str | None = None, - headers: dict[str, str] | None = None, + headers: Mapping[str, str] | None = None, timeout: float | None = None, - compression: Compression | None = None, + compression: Compression | CommonCompression | None = None, session: requests.Session | None = None, *, meter_provider: MeterProvider | None = None, - ): - self._shutdown_is_occuring = threading.Event() - self._endpoint = endpoint or environ.get( - OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, - _append_logs_path( - environ.get(OTEL_EXPORTER_OTLP_ENDPOINT, DEFAULT_ENDPOINT) - ), + ) -> None: ... + + @overload + def __init__( + self, + endpoint: str | None = None, + certificate_file: None = None, + client_key_file: None = None, + client_certificate_file: None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None = None, + compression: Compression | CommonCompression | None = None, + session: requests.Session | None = None, + *, + meter_provider: MeterProvider | None = None, + _transport: BaseHTTPTransport, + ) -> None: ... + + def __init__( + self, + endpoint: str | None = None, + certificate_file: str | None = None, + client_key_file: str | None = None, + client_certificate_file: str | None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None = None, + compression: Compression | CommonCompression | None = None, + session: requests.Session | None = None, + *, + meter_provider: MeterProvider | None = None, + _transport: BaseHTTPTransport | None = None, + ) -> None: + self._endpoint = endpoint or _resolve_endpoint( + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, DEFAULT_LOGS_EXPORT_PATH ) - # Keeping these as instance variables because they are used in tests - self._certificate_file = certificate_file or environ.get( - OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE, - environ.get(OTEL_EXPORTER_OTLP_CERTIFICATE, True), + self._compression = _normalize_compression( + compression + ) or _resolve_compression(OTEL_EXPORTER_OTLP_LOGS_COMPRESSION) + self._session = _resolve_session( + session, _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER ) - self._client_key_file = client_key_file or environ.get( + transport = _transport or _build_transport( + certificate_file, + client_key_file, + client_certificate_file, + OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE, OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY, - environ.get(OTEL_EXPORTER_OTLP_CLIENT_KEY, None), - ) - self._client_certificate_file = client_certificate_file or environ.get( OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE, - environ.get(OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, None), + session=self._session, ) - self._client_cert = ( - (self._client_certificate_file, self._client_key_file) - if self._client_certificate_file and self._client_key_file - else self._client_certificate_file + self._client = OTLPHTTPClient( + transport=transport, + endpoint=self._endpoint, + kind="logs", + timeout=timeout + if timeout is not None + else _resolve_timeout(OTEL_EXPORTER_OTLP_LOGS_TIMEOUT), + compression=self._compression, + headers=_resolve_headers(headers, OTEL_EXPORTER_OTLP_LOGS_HEADERS), + logger=_logger, ) - headers_string = environ.get( - OTEL_EXPORTER_OTLP_LOGS_HEADERS, - environ.get(OTEL_EXPORTER_OTLP_HEADERS, ""), - ) - self._headers = headers or parse_env_headers( - headers_string, liberal=True - ) - self._timeout = timeout or float( - environ.get( - OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, - environ.get(OTEL_EXPORTER_OTLP_TIMEOUT, DEFAULT_TIMEOUT), - ) - ) - self._compression = compression or _compression_from_env() - self._session = ( - session - or _load_session_from_envvar( - _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER - ) - or requests.Session() - ) - self._session.headers.update(self._headers) - self._session.headers.update(_OTLP_HTTP_HEADERS) - # let users override our defaults - self._session.headers.update(self._headers) - if self._compression is not Compression.NoCompression: - self._session.headers.update( - {"Content-Encoding": self._compression.value} - ) self._shutdown = False self._metrics = create_exporter_metrics( @@ -154,43 +156,6 @@ def __init__( == "true", ) - def _export( - self, serialized_data: bytes, timeout_sec: float | None = None - ): - data = serialized_data - if self._compression == Compression.Gzip: - gzip_data = BytesIO() - with gzip.GzipFile(fileobj=gzip_data, mode="w") as gzip_stream: - gzip_stream.write(serialized_data) - data = gzip_data.getvalue() - elif self._compression == Compression.Deflate: - data = zlib.compress(serialized_data) - - if timeout_sec is None: - timeout_sec = self._timeout - - # By default, keep-alive is enabled in Session's request - # headers. Backends may choose to close the connection - # while a post happens which causes an unhandled - # exception. This try/except will retry the post on such exceptions - try: - resp = self._session.post( - url=self._endpoint, - data=data, - verify=self._certificate_file, - timeout=timeout_sec, - cert=self._client_cert, - ) - except ConnectionError: - resp = self._session.post( - url=self._endpoint, - data=data, - verify=self._certificate_file, - timeout=timeout_sec, - cert=self._client_cert, - ) - return resp - def export( self, batch: Sequence[ReadableLogRecord] ) -> LogRecordExportResult: @@ -199,70 +164,26 @@ def export( return LogRecordExportResult.FAILURE with self._metrics.export_operation(len(batch)) as result: - serialized_data = encode_logs(batch).SerializeToString() - deadline_sec = time() + self._timeout - for retry_num in range(_MAX_RETRYS): - # multiplying by a random number between .8 and 1.2 introduces a +/20% jitter to each backoff. - backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2) - export_error: Exception | None = None - try: - resp = self._export(serialized_data, deadline_sec - time()) - if resp.ok: - return LogRecordExportResult.SUCCESS - except requests.exceptions.RequestException as error: - reason = error - export_error = error - retryable = isinstance(error, ConnectionError) - status_code = None - else: - reason = resp.reason - retryable = _is_retryable(resp) - status_code = resp.status_code - - if not retryable: - _logger.error( - "Failed to export logs batch code: %s, reason: %s", - status_code, - reason, - ) - error_attrs = ( - {HTTP_RESPONSE_STATUS_CODE: status_code} - if status_code is not None - else None - ) - result.error = export_error - result.error_attrs = error_attrs - return LogRecordExportResult.FAILURE - - if ( - retry_num + 1 == _MAX_RETRYS - or backoff_seconds > (deadline_sec - time()) - or self._shutdown - ): - _logger.error( - "Failed to export logs batch due to timeout, " - "max retries or shutdown." - ) - error_attrs = ( - {HTTP_RESPONSE_STATUS_CODE: status_code} - if status_code is not None - else None - ) - result.error = export_error - result.error_attrs = error_attrs - return LogRecordExportResult.FAILURE - _logger.warning( - "Transient error %s encountered while exporting logs batch, retrying in %.2fs.", - reason, - backoff_seconds, + try: + serialized_data = encode_logs(batch).SerializeToString() + # pylint: disable-next=broad-exception-caught + except Exception as error: + _logger.error("Failed to encode logs batch: %s", error) + result.error = error + return LogRecordExportResult.FAILURE + + export_result = self._client.export(serialized_data) + if not export_result.success: + result.error = export_result.error + result.error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: export_result.status_code} + if export_result.status_code is not None + else None ) - shutdown = self._shutdown_is_occuring.wait(backoff_seconds) - if shutdown: - _logger.warning("Shutdown in progress, aborting retry.") - break - return LogRecordExportResult.FAILURE + return LogRecordExportResult.FAILURE + return LogRecordExportResult.SUCCESS - def force_flush(self, timeout_millis: float = 10_000) -> bool: + def force_flush(self, timeout_millis: int = 10_000) -> bool: """Nothing is buffered in this exporter, so this method does nothing.""" return True @@ -271,23 +192,4 @@ def shutdown(self): _logger.warning("Exporter already shutdown, ignoring call") return self._shutdown = True - self._shutdown_is_occuring.set() - self._session.close() - - -def _compression_from_env() -> Compression: - compression = ( - environ.get( - OTEL_EXPORTER_OTLP_LOGS_COMPRESSION, - environ.get(OTEL_EXPORTER_OTLP_COMPRESSION, "none"), - ) - .lower() - .strip() - ) - return Compression(compression) - - -def _append_logs_path(endpoint: str) -> str: - if endpoint.endswith("/"): - return endpoint + DEFAULT_LOGS_EXPORT_PATH - return endpoint + f"/{DEFAULT_LOGS_EXPORT_PATH}" + self._client.shutdown() diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/metric_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/metric_exporter/__init__.py index eb1e69cfe4f..91098577f2e 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/metric_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/metric_exporter/__init__.py @@ -2,45 +2,46 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -import gzip import logging import os -import random -import threading -import zlib -from collections.abc import Callable, Iterable -from io import BytesIO -from os import environ -from time import time +from collections.abc import Callable, Iterable, Mapping from typing import ( # noqa: F401 + TYPE_CHECKING, Any, Optional, + overload, ) from urllib.parse import urlparse import requests -from requests.exceptions import ConnectionError from typing_extensions import deprecated +from opentelemetry.exporter.otlp.common._aggregation import ( + _get_aggregation, + _get_temporality, +) +from opentelemetry.exporter.otlp.common._http import ( + Compression as CommonCompression, +) +from opentelemetry.exporter.otlp.common._http import OTLPHTTPClient from opentelemetry.exporter.otlp.proto.common._exporter_metrics import ( create_exporter_metrics, ) from opentelemetry.exporter.otlp.proto.common._internal import ( _get_resource_data, ) -from opentelemetry.exporter.otlp.proto.common._internal.metrics_encoder import ( - OTLPMetricExporterMixin, -) from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( encode_metrics, ) -from opentelemetry.exporter.otlp.proto.http import ( - _OTLP_HTTP_HEADERS, - Compression, -) -from opentelemetry.exporter.otlp.proto.http._common import ( - _is_retryable, - _load_session_from_envvar, +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http._internal import ( + _build_transport, + _normalize_compression, + _resolve_compression, + _resolve_endpoint, + _resolve_headers, + _resolve_session, + _resolve_timeout, ) from opentelemetry.metrics import MeterProvider from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( # noqa: F401 @@ -60,12 +61,6 @@ ) from opentelemetry.sdk.environment_variables import ( _OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER, - OTEL_EXPORTER_OTLP_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION, - OTEL_EXPORTER_OTLP_ENDPOINT, - OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE, OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY, @@ -73,7 +68,6 @@ OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, OTEL_EXPORTER_OTLP_METRICS_HEADERS, OTEL_EXPORTER_OTLP_METRICS_TIMEOUT, - OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, ) from opentelemetry.sdk.metrics._internal.aggregation import Aggregation @@ -95,7 +89,9 @@ from opentelemetry.semconv.attributes.http_attributes import ( HTTP_RESPONSE_STATUS_CODE, ) -from opentelemetry.util.re import parse_env_headers + +if TYPE_CHECKING: + from opentelemetry.exporter.http.transport._base import BaseHTTPTransport _logger = logging.getLogger(__name__) @@ -104,19 +100,19 @@ DEFAULT_ENDPOINT = "http://localhost:4318/" DEFAULT_METRICS_EXPORT_PATH = "v1/metrics" DEFAULT_TIMEOUT = 10 # in seconds -_MAX_RETRYS = 6 -class OTLPMetricExporter(MetricExporter, OTLPMetricExporterMixin): +class OTLPMetricExporter(MetricExporter): + @overload def __init__( self, endpoint: str | None = None, certificate_file: str | None = None, client_key_file: str | None = None, client_certificate_file: str | None = None, - headers: dict[str, str] | None = None, + headers: Mapping[str, str] | None = None, timeout: float | None = None, - compression: Compression | None = None, + compression: Compression | CommonCompression | None = None, session: requests.Session | None = None, preferred_temporality: dict[type, AggregationTemporality] | None = None, @@ -124,7 +120,46 @@ def __init__( max_export_batch_size: int | None = None, *, meter_provider: MeterProvider | None = None, - ): + ) -> None: ... + + @overload + def __init__( + self, + endpoint: str | None = None, + certificate_file: None = None, + client_key_file: None = None, + client_certificate_file: None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None = None, + compression: Compression | CommonCompression | None = None, + session: requests.Session | None = None, + preferred_temporality: dict[type, AggregationTemporality] + | None = None, + preferred_aggregation: dict[type, Aggregation] | None = None, + max_export_batch_size: int | None = None, + *, + meter_provider: MeterProvider | None = None, + _transport: BaseHTTPTransport, + ) -> None: ... + + def __init__( + self, + endpoint: str | None = None, + certificate_file: str | None = None, + client_key_file: str | None = None, + client_certificate_file: str | None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None = None, + compression: Compression | CommonCompression | None = None, + session: requests.Session | None = None, + preferred_temporality: dict[type, AggregationTemporality] + | None = None, + preferred_aggregation: dict[type, Aggregation] | None = None, + max_export_batch_size: int | None = None, + *, + meter_provider: MeterProvider | None = None, + _transport: BaseHTTPTransport | None = None, + ) -> None: """OTLP HTTP metrics exporter Args: @@ -146,62 +181,42 @@ def __init__( If not set there is no limit to the number of data points in a request. If it is set and the number of data points exceeds the max, the request will be split. """ - self._shutdown_in_progress = threading.Event() - self._endpoint = endpoint or environ.get( - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, - _append_metrics_path( - environ.get(OTEL_EXPORTER_OTLP_ENDPOINT, DEFAULT_ENDPOINT) - ), + MetricExporter.__init__( + self, + preferred_temporality=_get_temporality(preferred_temporality), + preferred_aggregation=_get_aggregation(preferred_aggregation), ) - self._certificate_file = certificate_file or environ.get( - OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE, - environ.get(OTEL_EXPORTER_OTLP_CERTIFICATE, True), + self._endpoint = endpoint or _resolve_endpoint( + OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, DEFAULT_METRICS_EXPORT_PATH ) - self._client_key_file = client_key_file or environ.get( - OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY, - environ.get(OTEL_EXPORTER_OTLP_CLIENT_KEY, None), + self._compression = _normalize_compression( + compression + ) or _resolve_compression(OTEL_EXPORTER_OTLP_METRICS_COMPRESSION) + self._session = _resolve_session( + session, + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER, ) - self._client_certificate_file = client_certificate_file or environ.get( + transport = _transport or _build_transport( + certificate_file, + client_key_file, + client_certificate_file, + OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE, + OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY, OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE, - environ.get(OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, None), + session=self._session, ) - self._client_cert = ( - (self._client_certificate_file, self._client_key_file) - if self._client_certificate_file and self._client_key_file - else self._client_certificate_file - ) - headers_string = environ.get( - OTEL_EXPORTER_OTLP_METRICS_HEADERS, - environ.get(OTEL_EXPORTER_OTLP_HEADERS, ""), - ) - self._headers = headers or parse_env_headers( - headers_string, liberal=True - ) - self._timeout = timeout or float( - environ.get( - OTEL_EXPORTER_OTLP_METRICS_TIMEOUT, - environ.get(OTEL_EXPORTER_OTLP_TIMEOUT, DEFAULT_TIMEOUT), - ) - ) - self._compression = compression or _compression_from_env() - self._session = ( - session - or _load_session_from_envvar( - _OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER - ) - or requests.Session() - ) - self._session.headers.update(self._headers) - self._session.headers.update(_OTLP_HTTP_HEADERS) - # let users override our defaults - self._session.headers.update(self._headers) - if self._compression is not Compression.NoCompression: - self._session.headers.update( - {"Content-Encoding": self._compression.value} - ) - - self._common_configuration( - preferred_temporality, preferred_aggregation + self._client = OTLPHTTPClient( + transport=transport, + endpoint=self._endpoint, + kind="metrics", + timeout=timeout + if timeout is not None + else _resolve_timeout(OTEL_EXPORTER_OTLP_METRICS_TIMEOUT), + compression=self._compression, + headers=_resolve_headers( + headers, OTEL_EXPORTER_OTLP_METRICS_HEADERS + ), + logger=_logger, ) self._max_export_batch_size: int | None = max_export_batch_size self._shutdown = False @@ -217,121 +232,24 @@ def __init__( == "true", ) - def _export( - self, serialized_data: bytes, timeout_sec: float | None = None - ): - data = serialized_data - if self._compression == Compression.Gzip: - gzip_data = BytesIO() - with gzip.GzipFile(fileobj=gzip_data, mode="w") as gzip_stream: - gzip_stream.write(serialized_data) - data = gzip_data.getvalue() - elif self._compression == Compression.Deflate: - data = zlib.compress(serialized_data) - - if timeout_sec is None: - timeout_sec = self._timeout - - # By default, keep-alive is enabled in Session's request - # headers. Backends may choose to close the connection - # while a post happens which causes an unhandled - # exception. This try/except will retry the post on such exceptions - try: - resp = self._session.post( - url=self._endpoint, - data=data, - verify=self._certificate_file, - timeout=timeout_sec, - cert=self._client_cert, - ) - except ConnectionError: - resp = self._session.post( - url=self._endpoint, - data=data, - verify=self._certificate_file, - timeout=timeout_sec, - cert=self._client_cert, - ) - return resp - - def _export_with_retries( + def _export_one( self, export_request: ExportMetricsServiceRequest, - deadline_sec: float, num_items: int, ) -> MetricExportResult: - """Export serialized data with retry logic until success, non-transient error, or exponential backoff maxed out. - - Args: - export_request: ExportMetricsServiceRequest object containing metrics data to export - deadline_sec: timestamp deadline for the export - - Returns: - MetricExportResult: SUCCESS if export succeeded, FAILURE otherwise - """ with self._metrics.export_operation(num_items) as result: - serialized_data = export_request.SerializeToString() - deadline_sec = time() + self._timeout - for retry_num in range(_MAX_RETRYS): - # multiplying by a random number between .8 and 1.2 introduces a +/20% jitter to each backoff. - backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2) - export_error: Exception | None = None - try: - resp = self._export(serialized_data, deadline_sec - time()) - if resp.ok: - return MetricExportResult.SUCCESS - except requests.exceptions.RequestException as error: - reason = error - export_error = error - retryable = isinstance(error, ConnectionError) - status_code = None - else: - reason = resp.reason - retryable = _is_retryable(resp) - status_code = resp.status_code - - if not retryable: - _logger.error( - "Failed to export metrics batch code: %s, reason: %s", - status_code, - reason, - ) - error_attrs = ( - {HTTP_RESPONSE_STATUS_CODE: status_code} - if status_code is not None - else None - ) - result.error = export_error - result.error_attrs = error_attrs - return MetricExportResult.FAILURE - if ( - retry_num + 1 == _MAX_RETRYS - or backoff_seconds > (deadline_sec - time()) - or self._shutdown - ): - _logger.error( - "Failed to export metrics batch due to timeout, " - "max retries or shutdown." - ) - error_attrs = ( - {HTTP_RESPONSE_STATUS_CODE: status_code} - if status_code is not None - else None - ) - result.error = export_error - result.error_attrs = error_attrs - return MetricExportResult.FAILURE - - _logger.warning( - "Transient error %s encountered while exporting metrics batch, retrying in %.2fs.", - reason, - backoff_seconds, + export_result = self._client.export( + export_request.SerializeToString() + ) + if not export_result.success: + result.error = export_result.error + result.error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: export_result.status_code} + if export_result.status_code is not None + else None ) - shutdown = self._shutdown_in_progress.wait(backoff_seconds) - if shutdown: - _logger.warning("Shutdown in progress, aborting retry.") - break - return MetricExportResult.FAILURE + return MetricExportResult.FAILURE + return MetricExportResult.SUCCESS def export( self, @@ -349,14 +267,16 @@ def export( for metric in scope_metrics.metrics: num_items += len(metric.data.data_points) - export_request = encode_metrics(metrics_data) - deadline_sec = time() + self._timeout + try: + export_request = encode_metrics(metrics_data) + # pylint: disable-next=broad-exception-caught + except Exception as error: + _logger.error("Failed to encode metrics batch: %s", error) + return MetricExportResult.FAILURE # If no batch size configured, export as single batch with retries as configured if self._max_export_batch_size is None: - return self._export_with_retries( - export_request, deadline_sec, num_items - ) + return self._export_one(export_request, num_items) # Else, export in batches of configured size batched_export_requests = _split_metrics_data( @@ -364,11 +284,7 @@ def export( ) for split_metrics_data in batched_export_requests: - export_result = self._export_with_retries( - split_metrics_data, - deadline_sec, - num_items, - ) + export_result = self._export_one(split_metrics_data, num_items) if export_result != MetricExportResult.SUCCESS: return MetricExportResult.FAILURE @@ -380,12 +296,7 @@ def shutdown(self, timeout_millis: float = 30_000, **kwargs) -> None: _logger.warning("Exporter already shutdown, ignoring call") return self._shutdown = True - self._shutdown_in_progress.set() - self._session.close() - - @property - def _exporting(self) -> str: - return "metrics" + self._client.shutdown() def force_flush(self, timeout_millis: float = 10_000) -> bool: """Nothing is buffered in this exporter, so this method does nothing.""" @@ -722,21 +633,3 @@ def get_resource_data( name: str, ) -> list[PB2Resource]: return _get_resource_data(sdk_resource_scope_data, resource_class, name) - - -def _compression_from_env() -> Compression: - compression = ( - environ.get( - OTEL_EXPORTER_OTLP_METRICS_COMPRESSION, - environ.get(OTEL_EXPORTER_OTLP_COMPRESSION, "none"), - ) - .lower() - .strip() - ) - return Compression(compression) - - -def _append_metrics_path(endpoint: str) -> str: - if endpoint.endswith("/"): - return endpoint + DEFAULT_METRICS_EXPORT_PATH - return endpoint + f"/{DEFAULT_METRICS_EXPORT_PATH}" diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/trace_exporter/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/trace_exporter/__init__.py index 2be240103c0..05a426be005 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/trace_exporter/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/src/opentelemetry/exporter/otlp/proto/http/trace_exporter/__init__.py @@ -1,45 +1,39 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -import gzip +from __future__ import annotations + import logging import os -import random -import threading -import zlib -from collections.abc import Sequence -from io import BytesIO -from os import environ -from time import time +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING, overload from urllib.parse import urlparse import requests -from requests.exceptions import ConnectionError +from opentelemetry.exporter.otlp.common._http import ( + Compression as CommonCompression, +) +from opentelemetry.exporter.otlp.common._http import OTLPHTTPClient from opentelemetry.exporter.otlp.proto.common._exporter_metrics import ( create_exporter_metrics, ) from opentelemetry.exporter.otlp.proto.common.trace_encoder import ( encode_spans, ) -from opentelemetry.exporter.otlp.proto.http import ( - _OTLP_HTTP_HEADERS, - Compression, -) -from opentelemetry.exporter.otlp.proto.http._common import ( - _is_retryable, - _load_session_from_envvar, +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http._internal import ( + _build_transport, + _normalize_compression, + _resolve_compression, + _resolve_endpoint, + _resolve_headers, + _resolve_session, + _resolve_timeout, ) from opentelemetry.metrics import MeterProvider from opentelemetry.sdk.environment_variables import ( _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER, - OTEL_EXPORTER_OTLP_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION, - OTEL_EXPORTER_OTLP_ENDPOINT, - OTEL_EXPORTER_OTLP_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, @@ -57,7 +51,9 @@ from opentelemetry.semconv.attributes.http_attributes import ( HTTP_RESPONSE_STATUS_CODE, ) -from opentelemetry.util.re import parse_env_headers + +if TYPE_CHECKING: + from opentelemetry.exporter.http.transport._base import BaseHTTPTransport _logger = logging.getLogger(__name__) @@ -66,76 +62,85 @@ DEFAULT_ENDPOINT = "http://localhost:4318/" DEFAULT_TRACES_EXPORT_PATH = "v1/traces" DEFAULT_TIMEOUT = 10 # in seconds -_MAX_RETRYS = 6 class OTLPSpanExporter(SpanExporter): + @overload def __init__( self, endpoint: str | None = None, certificate_file: str | None = None, client_key_file: str | None = None, client_certificate_file: str | None = None, - headers: dict[str, str] | None = None, + headers: Mapping[str, str] | None = None, timeout: float | None = None, - compression: Compression | None = None, + compression: Compression | CommonCompression | None = None, session: requests.Session | None = None, *, meter_provider: MeterProvider | None = None, - ): - self._shutdown_in_progress = threading.Event() - self._endpoint = endpoint or environ.get( - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, - _append_trace_path( - environ.get(OTEL_EXPORTER_OTLP_ENDPOINT, DEFAULT_ENDPOINT) - ), + ) -> None: ... + + @overload + def __init__( + self, + endpoint: str | None = None, + certificate_file: None = None, + client_key_file: None = None, + client_certificate_file: None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None = None, + compression: Compression | CommonCompression | None = None, + session: requests.Session | None = None, + *, + meter_provider: MeterProvider | None = None, + _transport: BaseHTTPTransport, + ) -> None: ... + + def __init__( + self, + endpoint: str | None = None, + certificate_file: str | None = None, + client_key_file: str | None = None, + client_certificate_file: str | None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None = None, + compression: Compression | CommonCompression | None = None, + session: requests.Session | None = None, + *, + meter_provider: MeterProvider | None = None, + _transport: BaseHTTPTransport | None = None, + ) -> None: + self._endpoint = endpoint or _resolve_endpoint( + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, DEFAULT_TRACES_EXPORT_PATH ) - self._certificate_file = certificate_file or environ.get( - OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, - environ.get(OTEL_EXPORTER_OTLP_CERTIFICATE, True), + self._compression = _normalize_compression( + compression + ) or _resolve_compression(OTEL_EXPORTER_OTLP_TRACES_COMPRESSION) + self._session = _resolve_session( + session, _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER ) - self._client_key_file = client_key_file or environ.get( + transport = _transport or _build_transport( + certificate_file, + client_key_file, + client_certificate_file, + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, - environ.get(OTEL_EXPORTER_OTLP_CLIENT_KEY, None), - ) - self._client_certificate_file = client_certificate_file or environ.get( OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, - environ.get(OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, None), - ) - self._client_cert = ( - (self._client_certificate_file, self._client_key_file) - if self._client_certificate_file and self._client_key_file - else self._client_certificate_file + session=self._session, ) - headers_string = environ.get( - OTEL_EXPORTER_OTLP_TRACES_HEADERS, - environ.get(OTEL_EXPORTER_OTLP_HEADERS, ""), - ) - self._headers = headers or parse_env_headers( - headers_string, liberal=True - ) - self._timeout = timeout or float( - environ.get( - OTEL_EXPORTER_OTLP_TRACES_TIMEOUT, - environ.get(OTEL_EXPORTER_OTLP_TIMEOUT, DEFAULT_TIMEOUT), - ) - ) - self._compression = compression or _compression_from_env() - self._session = ( - session - or _load_session_from_envvar( - _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER - ) - or requests.Session() + self._client = OTLPHTTPClient( + transport=transport, + endpoint=self._endpoint, + kind="spans", + timeout=timeout + if timeout is not None + else _resolve_timeout(OTEL_EXPORTER_OTLP_TRACES_TIMEOUT), + compression=self._compression, + headers=_resolve_headers( + headers, OTEL_EXPORTER_OTLP_TRACES_HEADERS + ), + logger=_logger, ) - self._session.headers.update(self._headers) - self._session.headers.update(_OTLP_HTTP_HEADERS) - # let users override our defaults - self._session.headers.update(self._headers) - if self._compression is not Compression.NoCompression: - self._session.headers.update( - {"Content-Encoding": self._compression.value} - ) self._shutdown = False self._metrics = create_exporter_metrics( @@ -149,138 +154,40 @@ def __init__( == "true", ) - def _export( - self, serialized_data: bytes, timeout_sec: float | None = None - ): - data = serialized_data - if self._compression == Compression.Gzip: - gzip_data = BytesIO() - with gzip.GzipFile(fileobj=gzip_data, mode="w") as gzip_stream: - gzip_stream.write(serialized_data) - data = gzip_data.getvalue() - elif self._compression == Compression.Deflate: - data = zlib.compress(serialized_data) - - if timeout_sec is None: - timeout_sec = self._timeout - - # By default, keep-alive is enabled in Session's request - # headers. Backends may choose to close the connection - # while a post happens which causes an unhandled - # exception. This try/except will retry the post on such exceptions - try: - resp = self._session.post( - url=self._endpoint, - data=data, - verify=self._certificate_file, - timeout=timeout_sec, - cert=self._client_cert, - ) - except ConnectionError: - resp = self._session.post( - url=self._endpoint, - data=data, - verify=self._certificate_file, - timeout=timeout_sec, - cert=self._client_cert, - ) - return resp - def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: if self._shutdown: _logger.warning("Exporter already shutdown, ignoring batch") return SpanExportResult.FAILURE with self._metrics.export_operation(len(spans)) as result: - serialized_data = encode_spans(spans).SerializePartialToString() - deadline_sec = time() + self._timeout - for retry_num in range(_MAX_RETRYS): - # multiplying by a random number between .8 and 1.2 introduces a +/20% jitter to each backoff. - backoff_seconds = 2**retry_num * random.uniform(0.8, 1.2) - export_error: Exception | None = None - try: - resp = self._export(serialized_data, deadline_sec - time()) - if resp.ok: - return SpanExportResult.SUCCESS - except requests.exceptions.RequestException as error: - reason = error - export_error = error - retryable = isinstance(error, ConnectionError) - status_code = None - else: - reason = resp.reason - retryable = _is_retryable(resp) - status_code = resp.status_code - - if not retryable: - _logger.error( - "Failed to export span batch code: %s, reason: %s", - status_code, - reason, - ) - error_attrs = ( - {HTTP_RESPONSE_STATUS_CODE: status_code} - if status_code is not None - else None - ) - result.error = export_error - result.error_attrs = error_attrs - return SpanExportResult.FAILURE - - if ( - retry_num + 1 == _MAX_RETRYS - or backoff_seconds > (deadline_sec - time()) - or self._shutdown - ): - _logger.error( - "Failed to export span batch due to timeout, " - "max retries or shutdown." - ) - error_attrs = ( - {HTTP_RESPONSE_STATUS_CODE: status_code} - if status_code is not None - else None - ) - result.error = export_error - result.error_attrs = error_attrs - return SpanExportResult.FAILURE - _logger.warning( - "Transient error %s encountered while exporting span batch, retrying in %.2fs.", - reason, - backoff_seconds, + try: + serialized_data = encode_spans( + spans + ).SerializePartialToString() + # pylint: disable-next=broad-exception-caught + except Exception as error: + _logger.error("Failed to encode span batch: %s", error) + result.error = error + return SpanExportResult.FAILURE + + export_result = self._client.export(serialized_data) + if not export_result.success: + result.error = export_result.error + result.error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: export_result.status_code} + if export_result.status_code is not None + else None ) - shutdown = self._shutdown_in_progress.wait(backoff_seconds) - if shutdown: - _logger.warning("Shutdown in progress, aborting retry.") - break - return SpanExportResult.FAILURE + return SpanExportResult.FAILURE + return SpanExportResult.SUCCESS def shutdown(self): if self._shutdown: _logger.warning("Exporter already shutdown, ignoring call") return self._shutdown = True - self._shutdown_in_progress.set() - self._session.close() + self._client.shutdown() def force_flush(self, timeout_millis: int = 30000) -> bool: """Nothing is buffered in this exporter, so this method does nothing.""" return True - - -def _compression_from_env() -> Compression: - compression = ( - environ.get( - OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, - environ.get(OTEL_EXPORTER_OTLP_COMPRESSION, "none"), - ) - .lower() - .strip() - ) - return Compression(compression) - - -def _append_trace_path(endpoint: str) -> str: - if endpoint.endswith("/"): - return endpoint + DEFAULT_TRACES_EXPORT_PATH - return endpoint + f"/{DEFAULT_TRACES_EXPORT_PATH}" diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/test-requirements.txt b/exporter/opentelemetry-exporter-otlp-proto-http/test-requirements.txt index 1ebae084c42..560bde3f671 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/test-requirements.txt +++ b/exporter/opentelemetry-exporter-otlp-proto-http/test-requirements.txt @@ -4,6 +4,7 @@ charset-normalizer==3.3.2 googleapis-common-protos==1.75.0 idna==3.7 iniconfig==2.0.0 +mocket==3.14.1 packaging==24.0 pluggy==1.6.0 protobuf==7.35.0 @@ -19,6 +20,8 @@ wrapt==1.16.0 -e opentelemetry-api -e tests/opentelemetry-test-utils -e exporter/opentelemetry-exporter-otlp-proto-common +-e exporter/opentelemetry-exporter-otlp-common +-e exporter/opentelemetry-exporter-http-transport[requests,urllib3] -e opentelemetry-proto -e opentelemetry-sdk -e opentelemetry-semantic-conventions diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/tests/__init__.py b/exporter/opentelemetry-exporter-otlp-proto-http/tests/__init__.py index e69de29bb2d..e05ad351f69 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/tests/__init__.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/tests/__init__.py @@ -0,0 +1,32 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from unittest.mock import Mock, patch + + +@contextmanager +def _mock_clock( + shutdown_event: Mock | None = None, +) -> Iterator[Callable[[float], None]]: + _now = [0.0] + + def advance(delta: float) -> None: + _now[0] += delta + + def get_time() -> float: + return _now[0] + + if shutdown_event is not None: + + def _wait(duration: float) -> bool: + advance(duration) + return False + + shutdown_event.wait.side_effect = _wait + + with patch( + "opentelemetry.exporter.otlp.common._http.time.time", + side_effect=get_time, + ): + yield advance diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/tests/metrics/test_otlp_metrics_exporter.py b/exporter/opentelemetry-exporter-otlp-proto-http/tests/metrics/test_otlp_metrics_exporter.py index 84a11e8ae90..fb7fa85ab95 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/tests/metrics/test_otlp_metrics_exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/tests/metrics/test_otlp_metrics_exporter.py @@ -1,33 +1,39 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -# pylint: disable=too-many-lines +# pylint: disable=too-many-lines,protected-access,too-many-public-methods + import threading import time from logging import WARNING from os import environ from unittest import TestCase -from unittest.mock import ANY, MagicMock, Mock, patch +from unittest.mock import Mock, patch import requests -from requests import Session -from requests.exceptions import ConnectionError -from requests.models import Response +from mocket import Mocket, Mocketizer, mocketize +from mocket.mocks.mockhttp import Entry, Response +from opentelemetry.exporter.http.transport._requests import ( + RequestsHTTPTransport, +) +from opentelemetry.exporter.http.transport._urllib3 import ( + Urllib3HTTPTransport, +) +from opentelemetry.exporter.otlp.common._http import ( + Compression as CommonCompression, +) from opentelemetry.exporter.otlp.proto.common.metrics_encoder import ( encode_metrics, ) from opentelemetry.exporter.otlp.proto.http import Compression from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( - DEFAULT_COMPRESSION, DEFAULT_ENDPOINT, DEFAULT_METRICS_EXPORT_PATH, - DEFAULT_TIMEOUT, OTLPMetricExporter, _get_split_resource_metrics_pb2, _split_metrics_data, ) -from opentelemetry.exporter.otlp.proto.http.version import __version__ from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( ExportMetricsServiceRequest, ) @@ -41,22 +47,9 @@ ) from opentelemetry.sdk.environment_variables import ( _OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER, - OTEL_EXPORTER_OTLP_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION, - OTEL_EXPORTER_OTLP_ENDPOINT, - OTEL_EXPORTER_OTLP_HEADERS, - OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE, - OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY, - OTEL_EXPORTER_OTLP_METRICS_COMPRESSION, OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION, OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, - OTEL_EXPORTER_OTLP_METRICS_HEADERS, OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE, - OTEL_EXPORTER_OTLP_METRICS_TIMEOUT, - OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, ) from opentelemetry.sdk.metrics import ( @@ -86,19 +79,22 @@ ) from opentelemetry.test.metrictestutil import _generate_sum from opentelemetry.test.mock_test_classes import IterEntryPoint +from tests import _mock_clock -OS_ENV_ENDPOINT = "os.env.base" -OS_ENV_CERTIFICATE = "os/env/base.crt" -OS_ENV_CLIENT_CERTIFICATE = "os/env/client-cert.pem" -OS_ENV_CLIENT_KEY = "os/env/client-key.pem" -OS_ENV_HEADERS = "envHeader1=val1,envHeader2=val2,User-agent=Overridden" -OS_ENV_TIMEOUT = "30" +_TEST_ENDPOINT = "http://localhost:4318/v1/metrics" + + +def _decode_body(body: bytes) -> ExportMetricsServiceRequest: + return ExportMetricsServiceRequest.FromString(body) # pylint: disable=protected-access,too-many-public-methods class TestOTLPMetricExporter(TestCase): - # pylint: disable=too-many-public-methods def setUp(self): + env_patcher = patch.dict("os.environ", {}, clear=True) + env_patcher.start() + self.addCleanup(env_patcher.stop) + self.metric_reader = InMemoryMetricReader() self.meter_provider = MeterProvider( metric_readers=[self.metric_reader] @@ -128,212 +124,175 @@ def setUp(self): ), } - def test_constructor_default(self): - exporter = OTLPMetricExporter() - - self.assertEqual( - exporter._endpoint, DEFAULT_ENDPOINT + DEFAULT_METRICS_EXPORT_PATH - ) - self.assertEqual(exporter._certificate_file, True) - self.assertEqual(exporter._client_certificate_file, None) - self.assertEqual(exporter._client_key_file, None) - self.assertEqual(exporter._timeout, DEFAULT_TIMEOUT) - self.assertIs(exporter._compression, DEFAULT_COMPRESSION) - self.assertEqual(exporter._headers, {}) - self.assertIsInstance(exporter._session, Session) - self.assertIn("User-Agent", exporter._session.headers) + def assert_standard_metric_attrs(self, attributes): self.assertEqual( - exporter._session.headers.get("Content-Type"), - "application/x-protobuf", + attributes["otel.component.type"], "otlp_http_metric_exporter" ) - self.assertEqual( - exporter._session.headers.get("User-Agent"), - "OTel-OTLP-Exporter-Python/" + __version__, + self.assertTrue( + attributes["otel.component.name"].startswith( + "otlp_http_metric_exporter/" + ) ) + self.assertEqual(attributes["server.address"], "localhost") + self.assertEqual(attributes["server.port"], 4318) - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_CERTIFICATE: OS_ENV_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: OS_ENV_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY: OS_ENV_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION: Compression.Gzip.value, - OTEL_EXPORTER_OTLP_ENDPOINT: OS_ENV_ENDPOINT, - OTEL_EXPORTER_OTLP_HEADERS: OS_ENV_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT: OS_ENV_TIMEOUT, - OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE: "metrics/certificate.env", - OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE: "metrics/client-cert.pem", - OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY: "metrics/client-key.pem", - OTEL_EXPORTER_OTLP_METRICS_COMPRESSION: Compression.Deflate.value, - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "https://metrics.endpoint.env", - OTEL_EXPORTER_OTLP_METRICS_HEADERS: "metricsEnv1=val1,metricsEnv2=val2,metricEnv3===val3==,User-agent=metrics-user-agent", - OTEL_EXPORTER_OTLP_METRICS_TIMEOUT: "40", - _OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER: "credential_provider", - }, - ) - @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") - def test_exporter_metrics_env_take_priority(self, mock_entry_points): - credential = Session() - - def f(): - return credential + @staticmethod + def _create_metrics_data_multiple_data_points( + num_data_points: int, + ) -> MetricsData: + """Helper to create MetricsData with specified number of data points for testing batch splitting.""" + metrics = [] + for idx in range(num_data_points): + metrics.append(_generate_sum(f"sum_int_{idx}", 33)) - mock_entry_points.configure_mock( - return_value=[IterEntryPoint("custom_credential", f)] + return MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Resource( + attributes={"a": 1, "b": False}, + schema_url="resource_schema_url", + ), + scope_metrics=[ + ScopeMetrics( + scope=SDKInstrumentationScope( + name="first_name", + version="first_version", + schema_url="insrumentation_scope_schema_url", + ), + metrics=metrics, + schema_url="instrumentation_scope_schema_url", + ) + ], + schema_url="resource_schema_url", + ) + ] ) + + # -- construction / transport selection -------------------------------- + + def test_constructor_default_uses_urllib3_transport(self): exporter = OTLPMetricExporter() - self.assertEqual(exporter._endpoint, "https://metrics.endpoint.env") - self.assertEqual(exporter._certificate_file, "metrics/certificate.env") - self.assertEqual( - exporter._client_certificate_file, "metrics/client-cert.pem" - ) - self.assertEqual(exporter._client_key_file, "metrics/client-key.pem") - self.assertEqual(exporter._timeout, 40) - self.assertIs(exporter._compression, Compression.Deflate) - self.assertEqual( - exporter._headers, - { - "metricsenv1": "val1", - "metricsenv2": "val2", - "metricenv3": "==val3==", - "user-agent": "metrics-user-agent", - }, - ) - self.assertIsInstance(exporter._session, Session) self.assertEqual( - exporter._session.headers.get("User-Agent"), - "metrics-user-agent", + exporter._endpoint, DEFAULT_ENDPOINT + DEFAULT_METRICS_EXPORT_PATH ) - self.assertEqual( - exporter._session.headers.get("Content-Type"), - "application/x-protobuf", + self.assertIs(exporter._compression, CommonCompression.NONE) + self.assertIsNone(exporter._session) + self.assertIsInstance( + exporter._client._transport, Urllib3HTTPTransport ) - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_CERTIFICATE: OS_ENV_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: OS_ENV_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY: OS_ENV_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION: Compression.Gzip.value, - OTEL_EXPORTER_OTLP_ENDPOINT: OS_ENV_ENDPOINT, - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "https://metrics.endpoint.env", - OTEL_EXPORTER_OTLP_HEADERS: OS_ENV_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT: OS_ENV_TIMEOUT, - }, - ) - def test_exporter_constructor_take_priority(self): - exporter = OTLPMetricExporter( - endpoint="example.com/1234", - certificate_file="path/to/service.crt", - client_key_file="path/to/client-key.pem", - client_certificate_file="path/to/client-cert.pem", - headers={"testHeader1": "value1", "testHeader2": "value2"}, - timeout=20, - compression=Compression.NoCompression, - session=Session(), - ) + def test_explicit_session_uses_requests_transport(self): + session = requests.Session() + exporter = OTLPMetricExporter(session=session) - self.assertEqual(exporter._endpoint, "example.com/1234") - self.assertEqual(exporter._certificate_file, "path/to/service.crt") - self.assertEqual( - exporter._client_certificate_file, "path/to/client-cert.pem" - ) - self.assertEqual(exporter._client_key_file, "path/to/client-key.pem") - self.assertEqual(exporter._timeout, 20) - self.assertIs(exporter._compression, Compression.NoCompression) - self.assertEqual( - exporter._headers, - {"testHeader1": "value1", "testHeader2": "value2"}, + self.assertIs(exporter._session, session) + self.assertIsInstance( + exporter._client._transport, RequestsHTTPTransport ) - self.assertIsInstance(exporter._session, Session) - - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_CERTIFICATE: OS_ENV_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: OS_ENV_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY: OS_ENV_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION: Compression.Gzip.value, - OTEL_EXPORTER_OTLP_HEADERS: OS_ENV_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT: OS_ENV_TIMEOUT, - }, - ) - def test_exporter_env(self): - exporter = OTLPMetricExporter() + self.assertIs(exporter._client._transport._session, session) - self.assertEqual(exporter._certificate_file, OS_ENV_CERTIFICATE) - self.assertEqual( - exporter._client_certificate_file, OS_ENV_CLIENT_CERTIFICATE + @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") + def test_credential_provider_uses_requests_transport( + self, mock_entry_point + ): + credential = requests.Session() + mock_entry_point.configure_mock( + return_value=[ + IterEntryPoint("custom_credential", lambda: credential) + ] ) - self.assertEqual(exporter._client_key_file, OS_ENV_CLIENT_KEY) - self.assertEqual(exporter._timeout, int(OS_ENV_TIMEOUT)) - self.assertIs(exporter._compression, Compression.Gzip) - self.assertEqual( - exporter._headers, + with patch.dict( + environ, { - "envheader1": "val1", - "envheader2": "val2", - "user-agent": "Overridden", + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER: "custom_credential", }, + ): + exporter = OTLPMetricExporter() + + self.assertIs(exporter._session, credential) + self.assertIsInstance( + exporter._client._transport, RequestsHTTPTransport ) - @patch.dict( - "os.environ", - {OTEL_EXPORTER_OTLP_ENDPOINT: OS_ENV_ENDPOINT}, - ) + @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") + def test_exception_raised_when_entrypoint_returns_wrong_type( + self, mock_entry_point + ): + mock_entry_point.configure_mock( + return_value=[IterEntryPoint("bad_credential", lambda: 1)] + ) + with ( + patch.dict( + environ, + { + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_METRICS_CREDENTIAL_PROVIDER: "bad_credential", + }, + ), + self.assertRaises(RuntimeError), + ): + OTLPMetricExporter() + + def test_compression_dual_enum_acceptance(self): + for compression in (Compression.Gzip, CommonCompression.GZIP): + with self.subTest(compression=compression): + exporter = OTLPMetricExporter(compression=compression) + self.assertIs(exporter._compression, CommonCompression.GZIP) + def test_exporter_env_endpoint_without_slash(self): - exporter = OTLPMetricExporter() + with patch.dict( + environ, {OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "os.env.base"} + ): + exporter = OTLPMetricExporter() + self.assertEqual(exporter._endpoint, "os.env.base") + + @mocketize + def test_custom_transport(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + custom_transport = Urllib3HTTPTransport() + + with patch( + "opentelemetry.exporter.otlp.proto.http.metric_exporter._build_transport" + ) as mock_build_transport: + exporter = OTLPMetricExporter( + endpoint=_TEST_ENDPOINT, _transport=custom_transport + ) - self.assertEqual( - exporter._endpoint, - OS_ENV_ENDPOINT + f"/{DEFAULT_METRICS_EXPORT_PATH}", - ) + mock_build_transport.assert_not_called() + self.assertIs(exporter._client._transport, custom_transport) - @patch.dict( - "os.environ", - {OTEL_EXPORTER_OTLP_ENDPOINT: OS_ENV_ENDPOINT + "/"}, - ) - def test_exporter_env_endpoint_with_slash(self): - exporter = OTLPMetricExporter() + result = exporter.export(self.metrics["sum_int"]) + self.assertEqual(result, MetricExportResult.SUCCESS) - self.assertEqual( - exporter._endpoint, - OS_ENV_ENDPOINT + f"/{DEFAULT_METRICS_EXPORT_PATH}", - ) + # -- export / wire format ------------------------------------------------ - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_HEADERS: "envHeader1=val1,envHeader2=val2,missingValue" - }, - ) - def test_headers_parse_from_env(self): - with self.assertLogs(level="WARNING") as cm: - _ = OTLPMetricExporter() + @mocketize + def test_serialization(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPMetricExporter(endpoint=_TEST_ENDPOINT) + transport = exporter._client._transport + with patch.object( + transport, "request", wraps=transport.request + ) as mock_request: self.assertEqual( - cm.records[0].message, - ( - "Header format invalid! Header values in environment " - "variables must be URL encoded per the OpenTelemetry " - "Protocol Exporter specification or a comma separated " - "list of name=value occurrences: missingValue" - ), + exporter.export(self.metrics["sum_int"]), + MetricExportResult.SUCCESS, ) + serialized_data = encode_metrics(self.metrics["sum_int"]) + sent_data = mock_request.call_args.kwargs["data"] + self.assertEqual(_decode_body(sent_data), serialized_data) + @patch.dict( "os.environ", {OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: " true "} ) - @patch.object(Session, "post") - def test_success(self, mock_post): - resp = Response() - resp.status_code = 200 - mock_post.return_value = resp - - exporter = OTLPMetricExporter() - exporter.set_meter_provider(self.meter_provider) + @mocketize + def test_success_records_metrics(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPMetricExporter( + endpoint=_TEST_ENDPOINT, meter_provider=self.meter_provider + ) self.assertEqual( exporter.export(self.metrics["sum_int"]), @@ -342,39 +301,26 @@ def test_success(self, mock_post): metrics_data = self.metric_reader.get_metrics_data() scope_metrics = metrics_data.resource_metrics[0].scope_metrics[0] - self.assertEqual(scope_metrics.scope.name, "opentelemetry-sdk") metrics = sorted(scope_metrics.metrics, key=lambda m: m.name) self.assertEqual(len(metrics), 3) - self.assertEqual( - metrics[0].name, "otel.sdk.exporter.metric_data_point.exported" - ) - self.assert_standard_metric_attrs( - metrics[0].data.data_points[0].attributes - ) - self.assertEqual( - metrics[1].name, "otel.sdk.exporter.metric_data_point.inflight" - ) - self.assert_standard_metric_attrs( - metrics[1].data.data_points[0].attributes - ) - self.assertEqual( - metrics[2].name, "otel.sdk.exporter.operation.duration" - ) - self.assert_standard_metric_attrs( - metrics[2].data.data_points[0].attributes - ) + names = [m.name for m in metrics] + self.assertIn("otel.sdk.exporter.metric_data_point.exported", names) + self.assertIn("otel.sdk.exporter.metric_data_point.inflight", names) + self.assertIn("otel.sdk.exporter.operation.duration", names) + for metric in metrics: + self.assert_standard_metric_attrs( + metric.data.data_points[0].attributes + ) @patch.dict( "os.environ", {OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true"} ) - @patch.object(Session, "post") - def test_failure(self, mock_post): - resp = Response() - resp.status_code = 401 - mock_post.return_value = resp - - exporter = OTLPMetricExporter() - exporter.set_meter_provider(self.meter_provider) + @mocketize + def test_failure_records_metrics(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=401) + exporter = OTLPMetricExporter( + endpoint=_TEST_ENDPOINT, meter_provider=self.meter_provider + ) self.assertEqual( exporter.export(self.metrics["sum_int"]), @@ -383,152 +329,375 @@ def test_failure(self, mock_post): metrics_data = self.metric_reader.get_metrics_data() scope_metrics = metrics_data.resource_metrics[0].scope_metrics[0] - self.assertEqual(scope_metrics.scope.name, "opentelemetry-sdk") metrics = sorted(scope_metrics.metrics, key=lambda m: m.name) - self.assertEqual(len(metrics), 3) - self.assertEqual( - metrics[0].name, "otel.sdk.exporter.metric_data_point.exported" + duration_metric = next( + m + for m in metrics + if m.name == "otel.sdk.exporter.operation.duration" ) - self.assert_standard_metric_attrs( - metrics[0].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[0].data.data_points[0].attributes + self.assertEqual( + duration_metric.data.data_points[0].attributes[ + "http.response.status_code" + ], + 401, ) - self.assertNotIn( - "http.response.status_code", - metrics[0].data.data_points[0].attributes, + + # -- batch splitting integration ------------------------------------------ + + @mocketize + def test_export_max_export_batch_size_single_batch_integration(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + + # 2 data points, batch size of 3: fits in one batch + metrics_data = self._create_metrics_data_multiple_data_points(2) + exporter = OTLPMetricExporter( + endpoint=_TEST_ENDPOINT, max_export_batch_size=3 ) - self.assertEqual( - metrics[1].name, "otel.sdk.exporter.metric_data_point.inflight" + transport = exporter._client._transport + + with patch.object( + transport, "request", wraps=transport.request + ) as mock_request: + result = exporter.export(metrics_data) + + self.assertEqual(result, MetricExportResult.SUCCESS) + self.assertEqual(mock_request.call_count, 1) + + request = _decode_body(mock_request.call_args.kwargs["data"]) + self.assertEqual(len(request.resource_metrics), 1) + metrics = request.resource_metrics[0].scope_metrics[0].metrics + metric_names = {metric.name for metric in metrics} + self.assertEqual(metric_names, {"sum_int_0", "sum_int_1"}) + + @mocketize + def test_export_max_export_batch_size_multiple_batches_integration(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=200), + Response(status=200), ) - self.assert_standard_metric_attrs( - metrics[1].data.data_points[0].attributes + + # 3 data points, batch size of 2: requires 2 batches + metrics_data = self._create_metrics_data_multiple_data_points(3) + exporter = OTLPMetricExporter( + endpoint=_TEST_ENDPOINT, max_export_batch_size=2 ) - self.assertNotIn( - "error.type", metrics[1].data.data_points[0].attributes + transport = exporter._client._transport + + with patch.object( + transport, "request", wraps=transport.request + ) as mock_request: + result = exporter.export(metrics_data) + + self.assertEqual(result, MetricExportResult.SUCCESS) + self.assertEqual(mock_request.call_count, 2) + + first_request = _decode_body( + mock_request.call_args_list[0].kwargs["data"] ) - self.assertNotIn( - "http.response.status_code", - metrics[1].data.data_points[0].attributes, + first_metrics = ( + first_request.resource_metrics[0].scope_metrics[0].metrics ) self.assertEqual( - metrics[2].name, "otel.sdk.exporter.operation.duration" + {m.name for m in first_metrics}, {"sum_int_0", "sum_int_1"} ) - self.assert_standard_metric_attrs( - metrics[2].data.data_points[0].attributes + + second_request = _decode_body( + mock_request.call_args_list[1].kwargs["data"] ) - self.assertNotIn( - "error.type", metrics[2].data.data_points[0].attributes + second_metrics = ( + second_request.resource_metrics[0].scope_metrics[0].metrics ) - self.assertEqual( - metrics[2] - .data.data_points[0] - .attributes["http.response.status_code"], - 401, + self.assertEqual(len(second_metrics), 1) + self.assertEqual(second_metrics[0].name, "sum_int_2") + + @mocketize + def test_export_max_export_batch_size_retry_scenarios_integration(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=200), + Response(status=400), ) - @patch.object(Session, "post") - def test_serialization(self, mock_post): - resp = Response() - resp.status_code = 200 - mock_post.return_value = resp + # 3 data points, batch size of 2: requires 2 batches + metrics_data = self._create_metrics_data_multiple_data_points(3) + exporter = OTLPMetricExporter( + endpoint=_TEST_ENDPOINT, max_export_batch_size=2 + ) - exporter = OTLPMetricExporter() + result = exporter.export(metrics_data) + self.assertEqual(result, MetricExportResult.FAILURE) + self.assertEqual(len(Mocket.request_list()), 2) - self.assertEqual( - exporter.export(self.metrics["sum_int"]), - MetricExportResult.SUCCESS, + @mocketize + def test_export_max_export_batch_size_retryable_failure_integration(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=200), + Response(status=503), + Response(status=200), ) - serialized_data = encode_metrics(self.metrics["sum_int"]) - mock_post.assert_called_once_with( - url=exporter._endpoint, - data=serialized_data.SerializeToString(), - verify=exporter._certificate_file, - timeout=ANY, # Timeout is a float based on real time, can't put an exact value here. - cert=exporter._client_cert, + # 3 data points, batch size of 2: requires 2 batches + metrics_data = self._create_metrics_data_multiple_data_points(3) + exporter = OTLPMetricExporter( + endpoint=_TEST_ENDPOINT, max_export_batch_size=2, timeout=2.0 ) - def test_split_metrics_data_many_data_points(self): - metrics_data = ExportMetricsServiceRequest( - resource_metrics=[ - _resource_metrics( - index=1, - scope_metrics=[ - _scope_metrics( - index=1, - metrics=[ - _gauge( - index=1, - data_points=[ - _number_data_point(11), - _number_data_point(12), - _number_data_point(13), - ], - ), - ], - ), - ], - ), - ] - ) - split_metrics_data: list[ExportMetricsServiceRequest] = list( - # pylint: disable=protected-access - _split_metrics_data( - metrics_data=metrics_data, - max_export_batch_size=2, - ) - ) + result = exporter.export(metrics_data) + self.assertEqual(result, MetricExportResult.SUCCESS) + # First batch + retry of second batch + self.assertEqual(len(Mocket.request_list()), 3) - self.assertEqual( - [ - ExportMetricsServiceRequest( - resource_metrics=[ - _resource_metrics( - index=1, - scope_metrics=[ - _scope_metrics( - index=1, - metrics=[ - _gauge( - index=1, - data_points=[ - _number_data_point(11), - _number_data_point(12), - ], - ), - ], - ), - ], - ), - ] - ), - ExportMetricsServiceRequest( - resource_metrics=[ - _resource_metrics( - index=1, - scope_metrics=[ - _scope_metrics( - index=1, - metrics=[ - _gauge( - index=1, - data_points=[ - _number_data_point(13), - ], - ), - ], - ), - ], - ), - ] + # -- aggregation / temporality (unchanged pure logic) --------------------- + + def test_aggregation_temporality(self): + otlp_metric_exporter = OTLPMetricExporter() + + for ( + temporality + ) in otlp_metric_exporter._preferred_temporality.values(): + self.assertEqual(temporality, AggregationTemporality.CUMULATIVE) + + with patch.dict( + environ, + {OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "CUMULATIVE"}, + ): + otlp_metric_exporter = OTLPMetricExporter() + + for ( + temporality + ) in otlp_metric_exporter._preferred_temporality.values(): + self.assertEqual( + temporality, AggregationTemporality.CUMULATIVE + ) + + with patch.dict( + environ, {OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "ABC"} + ): + with self.assertLogs(level=WARNING): + otlp_metric_exporter = OTLPMetricExporter() + + for ( + temporality + ) in otlp_metric_exporter._preferred_temporality.values(): + self.assertEqual( + temporality, AggregationTemporality.CUMULATIVE + ) + + with patch.dict( + environ, + {OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "DELTA"}, + ): + otlp_metric_exporter = OTLPMetricExporter() + + self.assertEqual( + otlp_metric_exporter._preferred_temporality[Counter], + AggregationTemporality.DELTA, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[UpDownCounter], + AggregationTemporality.CUMULATIVE, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[Histogram], + AggregationTemporality.DELTA, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[ObservableCounter], + AggregationTemporality.DELTA, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[ + ObservableUpDownCounter + ], + AggregationTemporality.CUMULATIVE, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[ObservableGauge], + AggregationTemporality.CUMULATIVE, + ) + + with patch.dict( + environ, + {OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "LOWMEMORY"}, + ): + otlp_metric_exporter = OTLPMetricExporter() + + self.assertEqual( + otlp_metric_exporter._preferred_temporality[Counter], + AggregationTemporality.DELTA, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[UpDownCounter], + AggregationTemporality.CUMULATIVE, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[Histogram], + AggregationTemporality.DELTA, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[ObservableCounter], + AggregationTemporality.CUMULATIVE, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[ + ObservableUpDownCounter + ], + AggregationTemporality.CUMULATIVE, + ) + self.assertEqual( + otlp_metric_exporter._preferred_temporality[ObservableGauge], + AggregationTemporality.CUMULATIVE, + ) + + def test_exponential_explicit_bucket_histogram(self): + self.assertIsInstance( + OTLPMetricExporter()._preferred_aggregation[Histogram], + ExplicitBucketHistogramAggregation, + ) + + with patch.dict( + environ, + { + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION: "base2_exponential_bucket_histogram" + }, + ): + self.assertIsInstance( + OTLPMetricExporter()._preferred_aggregation[Histogram], + ExponentialBucketHistogramAggregation, + ) + + with patch.dict( + environ, + {OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION: "abc"}, + ): + with self.assertLogs(level=WARNING) as log: + self.assertIsInstance( + OTLPMetricExporter()._preferred_aggregation[Histogram], + ExplicitBucketHistogramAggregation, + ) + self.assertIn( + ( + "Invalid value for OTEL_EXPORTER_OTLP_METRICS_DEFAULT_" + "HISTOGRAM_AGGREGATION: ABC, using explicit bucket " + "histogram aggregation" ), - ], - split_metrics_data, + log.output[0], + ) + + with patch.dict( + environ, + { + OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION: "explicit_bucket_histogram" + }, + ): + self.assertIsInstance( + OTLPMetricExporter()._preferred_aggregation[Histogram], + ExplicitBucketHistogramAggregation, + ) + + def test_preferred_aggregation_override(self): + histogram_aggregation = ExplicitBucketHistogramAggregation( + boundaries=[0.05, 0.1, 0.5, 1, 5, 10], ) - def test_split_metrics_data_nb_data_points_equal_batch_size(self): + exporter = OTLPMetricExporter( + preferred_aggregation={ + Histogram: histogram_aggregation, + }, + ) + + self.assertEqual( + exporter._preferred_aggregation[Histogram], histogram_aggregation + ) + + # -- misc ----------------------------------------------------------------- + + @mocketize + def test_2xx_status_code(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + self.assertEqual( + OTLPMetricExporter(endpoint=_TEST_ENDPOINT).export( + self.metrics["sum_int"] + ), + MetricExportResult.SUCCESS, + ) + + @mocketize + def test_exporter_metrics_disabled_after_set_meter_provider(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPMetricExporter(endpoint=_TEST_ENDPOINT) + exporter.set_meter_provider(self.meter_provider) + + self.assertEqual( + exporter.export(self.metrics["sum_int"]), + MetricExportResult.SUCCESS, + ) + + self.assertIsNone(self.metric_reader.get_metrics_data()) + + def test_export_retryable_status_codes(self): + for status_code in (429, 502, 503, 504): + with self.subTest(status_code=status_code), Mocketizer(): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=status_code), + Response(status=200), + ) + exporter = OTLPMetricExporter( + endpoint=_TEST_ENDPOINT, timeout=30.0 + ) + shutdown_event = Mock(spec=threading.Event) + shutdown_event.is_set.return_value = False + exporter._client._shutdown_event = shutdown_event + + with _mock_clock(shutdown_event): + result = exporter.export(self.metrics["sum_int"]) + + self.assertEqual(result, MetricExportResult.SUCCESS) + self.assertEqual(len(Mocket.request_list()), 2) + + def test_export_non_retryable_status_codes(self): + for status_code in (400, 401, 403, 404, 408, 500, 501): + with self.subTest(status_code=status_code), Mocketizer(): + Entry.single_register( + Entry.POST, _TEST_ENDPOINT, status=status_code + ) + exporter = OTLPMetricExporter(endpoint=_TEST_ENDPOINT) + + result = exporter.export(self.metrics["sum_int"]) + + self.assertEqual(result, MetricExportResult.FAILURE) + self.assertEqual(len(Mocket.request_list()), 1) + + @mocketize + def test_shutdown_interrupts_retry_backoff(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + *[Response(status=503)] * 6, + ) + exporter = OTLPMetricExporter(endpoint=_TEST_ENDPOINT, timeout=1.5) + thread = threading.Thread( + target=exporter.export, args=(self.metrics["sum_int"],) + ) + before = time.time() + thread.start() + time.sleep(0.05) + exporter.shutdown() + thread.join() + after = time.time() + + self.assertLess(after - before, 0.5) + + # -- split_metrics_data / get_split_resource_metrics_pb2 (unchanged) ------ + + def test_split_metrics_data_many_data_points(self): metrics_data = ExportMetricsServiceRequest( resource_metrics=[ _resource_metrics( @@ -551,12 +720,10 @@ def test_split_metrics_data_nb_data_points_equal_batch_size(self): ), ] ) - split_metrics_data: list[ExportMetricsServiceRequest] = list( - # pylint: disable=protected-access _split_metrics_data( metrics_data=metrics_data, - max_export_batch_size=3, + max_export_batch_size=2, ) ) @@ -575,6 +742,25 @@ def test_split_metrics_data_nb_data_points_equal_batch_size(self): data_points=[ _number_data_point(11), _number_data_point(12), + ], + ), + ], + ), + ], + ), + ] + ), + ExportMetricsServiceRequest( + resource_metrics=[ + _resource_metrics( + index=1, + scope_metrics=[ + _scope_metrics( + index=1, + metrics=[ + _gauge( + index=1, + data_points=[ _number_data_point(13), ], ), @@ -588,8 +774,7 @@ def test_split_metrics_data_nb_data_points_equal_batch_size(self): split_metrics_data, ) - def test_split_metrics_data_many_resources_scopes_metrics(self): - # GIVEN + def test_split_metrics_data_nb_data_points_equal_batch_size(self): metrics_data = ExportMetricsServiceRequest( resource_metrics=[ _resource_metrics( @@ -602,22 +787,7 @@ def test_split_metrics_data_many_resources_scopes_metrics(self): index=1, data_points=[ _number_data_point(11), - ], - ), - _gauge( - index=2, - data_points=[ _number_data_point(12), - ], - ), - ], - ), - _scope_metrics( - index=2, - metrics=[ - _gauge( - index=3, - data_points=[ _number_data_point(13), ], ), @@ -625,30 +795,13 @@ def test_split_metrics_data_many_resources_scopes_metrics(self): ), ], ), - _resource_metrics( - index=2, - scope_metrics=[ - _scope_metrics( - index=3, - metrics=[ - _gauge( - index=4, - data_points=[ - _number_data_point(14), - ], - ), - ], - ), - ], - ), ] ) split_metrics_data: list[ExportMetricsServiceRequest] = list( - # pylint: disable=protected-access _split_metrics_data( metrics_data=metrics_data, - max_export_batch_size=2, + max_export_batch_size=3, ) ) @@ -666,7 +819,97 @@ def test_split_metrics_data_many_resources_scopes_metrics(self): index=1, data_points=[ _number_data_point(11), - ], + _number_data_point(12), + _number_data_point(13), + ], + ), + ], + ), + ], + ), + ] + ), + ], + split_metrics_data, + ) + + def test_split_metrics_data_many_resources_scopes_metrics(self): + metrics_data = ExportMetricsServiceRequest( + resource_metrics=[ + _resource_metrics( + index=1, + scope_metrics=[ + _scope_metrics( + index=1, + metrics=[ + _gauge( + index=1, + data_points=[ + _number_data_point(11), + ], + ), + _gauge( + index=2, + data_points=[ + _number_data_point(12), + ], + ), + ], + ), + _scope_metrics( + index=2, + metrics=[ + _gauge( + index=3, + data_points=[ + _number_data_point(13), + ], + ), + ], + ), + ], + ), + _resource_metrics( + index=2, + scope_metrics=[ + _scope_metrics( + index=3, + metrics=[ + _gauge( + index=4, + data_points=[ + _number_data_point(14), + ], + ), + ], + ), + ], + ), + ] + ) + + split_metrics_data: list[ExportMetricsServiceRequest] = list( + _split_metrics_data( + metrics_data=metrics_data, + max_export_batch_size=2, + ) + ) + + self.assertEqual( + [ + ExportMetricsServiceRequest( + resource_metrics=[ + _resource_metrics( + index=1, + scope_metrics=[ + _scope_metrics( + index=1, + metrics=[ + _gauge( + index=1, + data_points=[ + _number_data_point(11), + ], ), _gauge( index=2, @@ -915,560 +1158,6 @@ def test_get_split_resource_metrics_pb2_unsupported_metric_type(self): log.output[0], ) - @staticmethod - def _create_metrics_data_multiple_data_points( - num_data_points: int, - ) -> MetricsData: - """Helper to create MetricsData with specified number of data points for testing batch splitting.""" - metrics = [] - for idx in range(num_data_points): - metrics.append(_generate_sum(f"sum_int_{idx}", 33)) - - return MetricsData( - resource_metrics=[ - ResourceMetrics( - resource=Resource( - attributes={"a": 1, "b": False}, - schema_url="resource_schema_url", - ), - scope_metrics=[ - ScopeMetrics( - scope=SDKInstrumentationScope( - name="first_name", - version="first_version", - schema_url="insrumentation_scope_schema_url", - ), - metrics=metrics, - schema_url="instrumentation_scope_schema_url", - ) - ], - schema_url="resource_schema_url", - ) - ] - ) - - @patch.object(Session, "post") - def test_export_max_export_batch_size_single_batch_integration( - self, mock_post - ): - resp = Response() - resp.status_code = 200 - mock_post.return_value = resp - - # 2 data points, batch size of 3: fits in one batch - metrics_data = ( - TestOTLPMetricExporter._create_metrics_data_multiple_data_points(2) - ) - exporter = OTLPMetricExporter(max_export_batch_size=3) - result = exporter.export(metrics_data) - - self.assertEqual(result, MetricExportResult.SUCCESS) - self.assertEqual(mock_post.call_count, 1) - mock_post.assert_called_once() - - call_args = mock_post.call_args - self.assertEqual(call_args.kwargs["url"], exporter._endpoint) - self.assertIsInstance(call_args.kwargs["data"], bytes) - self.assertEqual( - call_args.kwargs["verify"], exporter._certificate_file - ) - batch_data = call_args.kwargs["data"] - request = ExportMetricsServiceRequest() - request.ParseFromString(batch_data) - self.assertEqual(len(request.resource_metrics), 1) - metrics = request.resource_metrics[0].scope_metrics[0].metrics - self.assertEqual(len(metrics), 2) - metric_names = {metric.name for metric in metrics} - self.assertEqual(metric_names, {"sum_int_0", "sum_int_1"}) - - @patch.object(Session, "post") - def test_export_max_export_batch_size_multiple_batches_integration( - self, mock_post - ): - resp = Response() - resp.status_code = 200 - mock_post.return_value = resp - - # 3 data points, batch size of 2: requires 2 batches - metrics_data = ( - TestOTLPMetricExporter._create_metrics_data_multiple_data_points(3) - ) - exporter = OTLPMetricExporter(max_export_batch_size=2) - result = exporter.export(metrics_data) - - self.assertEqual(result, MetricExportResult.SUCCESS) - self.assertEqual(mock_post.call_count, 2) - - for call_args in mock_post.call_args_list: - self.assertEqual(call_args.kwargs["url"], exporter._endpoint) - self.assertIsInstance(call_args.kwargs["data"], bytes) - self.assertEqual( - call_args.kwargs["verify"], exporter._certificate_file - ) - self.assertEqual(len(mock_post.call_args_list), 2) - - # First batch should contain sum_int_0 and sum_int_1 - first_batch_data = mock_post.call_args_list[0].kwargs["data"] - first_request = ExportMetricsServiceRequest() - first_request.ParseFromString(first_batch_data) - self.assertEqual(len(first_request.resource_metrics), 1) - first_metrics = ( - first_request.resource_metrics[0].scope_metrics[0].metrics - ) - self.assertEqual(len(first_metrics), 2) - first_metric_names = {metric.name for metric in first_metrics} - self.assertEqual(first_metric_names, {"sum_int_0", "sum_int_1"}) - - # Second batch should contain sum_int_2 - second_batch_data = mock_post.call_args_list[1].kwargs["data"] - second_request = ExportMetricsServiceRequest() - second_request.ParseFromString(second_batch_data) - self.assertEqual(len(second_request.resource_metrics), 1) - second_metrics = ( - second_request.resource_metrics[0].scope_metrics[0].metrics - ) - self.assertEqual(len(second_metrics), 1) - self.assertEqual(second_metrics[0].name, "sum_int_2") - - @patch.object(Session, "post") - def test_export_max_export_batch_size_retry_scenarios_integration( - self, mock_post - ): - # Setup HTTP responses: first request succeeds, second fails non-retryable - success_resp = Response() - success_resp.status_code = 200 - failure_resp = Response() - failure_resp.status_code = 400 - failure_resp.reason = "Bad Request" - mock_post.side_effect = [success_resp, failure_resp] - - # 3 data points, batch size of 2: requires 2 batches - metrics_data = ( - TestOTLPMetricExporter._create_metrics_data_multiple_data_points(3) - ) - exporter = OTLPMetricExporter(max_export_batch_size=2) - - # Export should fail when second batch fails - result = exporter.export(metrics_data) - self.assertEqual(result, MetricExportResult.FAILURE) - self.assertEqual(mock_post.call_count, 2) - - # Verify the content of successful first batch - first_batch_data = mock_post.call_args_list[0].kwargs["data"] - first_request = ExportMetricsServiceRequest() - first_request.ParseFromString(first_batch_data) - self.assertEqual(len(first_request.resource_metrics), 1) - first_metrics = ( - first_request.resource_metrics[0].scope_metrics[0].metrics - ) - self.assertEqual(len(first_metrics), 2) - first_metric_names = {metric.name for metric in first_metrics} - self.assertEqual(first_metric_names, {"sum_int_0", "sum_int_1"}) - - @patch.object(Session, "post") - def test_export_max_export_batch_size_retryable_failure_integration( - self, mock_post - ): - success_resp = Response() - success_resp.status_code = 200 - retryable_failure_resp = Response() - retryable_failure_resp.status_code = 503 - retryable_failure_resp.reason = "Service Unavailable" - mock_post.side_effect = [ - success_resp, - retryable_failure_resp, - success_resp, - ] - - # 3 data points, batch size of 2: requires 2 batches - metrics_data = ( - TestOTLPMetricExporter._create_metrics_data_multiple_data_points(3) - ) - exporter = OTLPMetricExporter(max_export_batch_size=2, timeout=2.0) - - # Export should eventually succeed after retry - result = exporter.export(metrics_data) - self.assertEqual(result, MetricExportResult.SUCCESS) - self.assertEqual( - mock_post.call_count, 3 - ) # First batch + retry of second batch - - first_batch_data = mock_post.call_args_list[0].kwargs["data"] - first_request = ExportMetricsServiceRequest() - first_request.ParseFromString(first_batch_data) - self.assertEqual(len(first_request.resource_metrics), 1) - first_metrics = ( - first_request.resource_metrics[0].scope_metrics[0].metrics - ) - self.assertEqual(len(first_metrics), 2) - first_metric_names = {metric.name for metric in first_metrics} - self.assertEqual(first_metric_names, {"sum_int_0", "sum_int_1"}) - # Second batch (retry) should contain sum_int_2 - second_batch_data = mock_post.call_args_list[2].kwargs["data"] - second_request = ExportMetricsServiceRequest() - second_request.ParseFromString(second_batch_data) - self.assertEqual(len(second_request.resource_metrics), 1) - second_metrics = ( - second_request.resource_metrics[0].scope_metrics[0].metrics - ) - self.assertEqual(len(second_metrics), 1) - self.assertEqual(second_metrics[0].name, "sum_int_2") - - def test_aggregation_temporality(self): - otlp_metric_exporter = OTLPMetricExporter() - - for ( - temporality - ) in otlp_metric_exporter._preferred_temporality.values(): - self.assertEqual(temporality, AggregationTemporality.CUMULATIVE) - - with patch.dict( - environ, - {OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "CUMULATIVE"}, - ): - otlp_metric_exporter = OTLPMetricExporter() - - for ( - temporality - ) in otlp_metric_exporter._preferred_temporality.values(): - self.assertEqual( - temporality, AggregationTemporality.CUMULATIVE - ) - - with patch.dict( - environ, {OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "ABC"} - ): - with self.assertLogs(level=WARNING): - otlp_metric_exporter = OTLPMetricExporter() - - for ( - temporality - ) in otlp_metric_exporter._preferred_temporality.values(): - self.assertEqual( - temporality, AggregationTemporality.CUMULATIVE - ) - - with patch.dict( - environ, - {OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "DELTA"}, - ): - otlp_metric_exporter = OTLPMetricExporter() - - self.assertEqual( - otlp_metric_exporter._preferred_temporality[Counter], - AggregationTemporality.DELTA, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[UpDownCounter], - AggregationTemporality.CUMULATIVE, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[Histogram], - AggregationTemporality.DELTA, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[ObservableCounter], - AggregationTemporality.DELTA, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[ - ObservableUpDownCounter - ], - AggregationTemporality.CUMULATIVE, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[ObservableGauge], - AggregationTemporality.CUMULATIVE, - ) - - with patch.dict( - environ, - {OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE: "LOWMEMORY"}, - ): - otlp_metric_exporter = OTLPMetricExporter() - - self.assertEqual( - otlp_metric_exporter._preferred_temporality[Counter], - AggregationTemporality.DELTA, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[UpDownCounter], - AggregationTemporality.CUMULATIVE, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[Histogram], - AggregationTemporality.DELTA, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[ObservableCounter], - AggregationTemporality.CUMULATIVE, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[ - ObservableUpDownCounter - ], - AggregationTemporality.CUMULATIVE, - ) - self.assertEqual( - otlp_metric_exporter._preferred_temporality[ObservableGauge], - AggregationTemporality.CUMULATIVE, - ) - - def test_exponential_explicit_bucket_histogram(self): - self.assertIsInstance( - OTLPMetricExporter()._preferred_aggregation[Histogram], - ExplicitBucketHistogramAggregation, - ) - - with patch.dict( - environ, - { - OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION: "base2_exponential_bucket_histogram" - }, - ): - self.assertIsInstance( - OTLPMetricExporter()._preferred_aggregation[Histogram], - ExponentialBucketHistogramAggregation, - ) - - with patch.dict( - environ, - {OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION: "abc"}, - ): - with self.assertLogs(level=WARNING) as log: - self.assertIsInstance( - OTLPMetricExporter()._preferred_aggregation[Histogram], - ExplicitBucketHistogramAggregation, - ) - self.assertIn( - ( - "Invalid value for OTEL_EXPORTER_OTLP_METRICS_DEFAULT_" - "HISTOGRAM_AGGREGATION: abc, using explicit bucket " - "histogram aggregation" - ), - log.output[0], - ) - - with patch.dict( - environ, - { - OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION: "explicit_bucket_histogram" - }, - ): - self.assertIsInstance( - OTLPMetricExporter()._preferred_aggregation[Histogram], - ExplicitBucketHistogramAggregation, - ) - - @patch.object(OTLPMetricExporter, "_export", return_value=Mock(ok=True)) - def test_2xx_status_code(self, mock_otlp_metric_exporter): - """ - Test that any HTTP 2XX code returns a successful result - """ - - self.assertEqual( - OTLPMetricExporter().export(MagicMock()), - MetricExportResult.SUCCESS, - ) - - @patch.dict("os.environ", {}, clear=True) - @patch.object(OTLPMetricExporter, "_export", return_value=Mock(ok=True)) - def test_exporter_metrics_disabled_after_set_meter_provider( - self, _mock_export - ): - exporter = OTLPMetricExporter() - exporter.set_meter_provider(self.meter_provider) - - self.assertEqual( - exporter.export(self.metrics["sum_int"]), - MetricExportResult.SUCCESS, - ) - - self.assertIsNone(self.metric_reader.get_metrics_data()) - - def test_preferred_aggregation_override(self): - histogram_aggregation = ExplicitBucketHistogramAggregation( - boundaries=[0.05, 0.1, 0.5, 1, 5, 10], - ) - - exporter = OTLPMetricExporter( - preferred_aggregation={ - Histogram: histogram_aggregation, - }, - ) - - self.assertEqual( - exporter._preferred_aggregation[Histogram], histogram_aggregation - ) - - @patch.dict( - "os.environ", {OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true"} - ) - @patch.object(Session, "post") - def test_retry_timeout(self, mock_post): - exporter = OTLPMetricExporter( - timeout=1.5, meter_provider=self.meter_provider - ) - - resp = Response() - resp.status_code = 503 - resp.reason = "UNAVAILABLE" - mock_post.return_value = resp - with self.assertLogs(level=WARNING) as warning: - before = time.time() - self.assertEqual( - exporter.export(self.metrics["sum_int"]), - MetricExportResult.FAILURE, - ) - after = time.time() - - # First call at time 0, second at time 1, then an early return before the second backoff sleep b/c it would exceed timeout. - self.assertEqual(mock_post.call_count, 2) - # There's a +/-20% jitter on each backoff. - self.assertTrue(0.75 < after - before < 1.25) - self.assertIn( - "Transient error UNAVAILABLE encountered while exporting metrics batch, retrying in", - warning.records[0].message, - ) - - metrics_data = self.metric_reader.get_metrics_data() - scope_metrics = metrics_data.resource_metrics[0].scope_metrics[0] - self.assertEqual(scope_metrics.scope.name, "opentelemetry-sdk") - metrics = sorted(scope_metrics.metrics, key=lambda m: m.name) - self.assertEqual(len(metrics), 3) - self.assertEqual( - metrics[0].name, "otel.sdk.exporter.metric_data_point.exported" - ) - self.assert_standard_metric_attrs( - metrics[0].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[0].data.data_points[0].attributes - ) - self.assertNotIn( - "http.response.status_code", - metrics[0].data.data_points[0].attributes, - ) - self.assertEqual( - metrics[1].name, "otel.sdk.exporter.metric_data_point.inflight" - ) - self.assert_standard_metric_attrs( - metrics[1].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[1].data.data_points[0].attributes - ) - self.assertNotIn( - "http.response.status_code", - metrics[1].data.data_points[0].attributes, - ) - self.assertEqual( - metrics[2].name, "otel.sdk.exporter.operation.duration" - ) - self.assert_standard_metric_attrs( - metrics[2].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[2].data.data_points[0].attributes - ) - self.assertEqual( - metrics[2] - .data.data_points[0] - .attributes["http.response.status_code"], - 503, - ) - - @patch.object(Session, "post") - def test_export_no_collector_available_retryable(self, mock_post): - exporter = OTLPMetricExporter(timeout=1.5) - msg = "Server not available." - mock_post.side_effect = ConnectionError(msg) - with self.assertLogs(level=WARNING) as warning: - self.assertEqual( - exporter.export(self.metrics["sum_int"]), - MetricExportResult.FAILURE, - ) - # Check for greater 2 because the request is on each retry - # done twice at the moment. - self.assertGreater(mock_post.call_count, 2) - self.assertIn( - f"Transient error {msg} encountered while exporting metrics batch, retrying in", - warning.records[0].message, - ) - - @patch.object(Session, "post") - def test_export_no_collector_available(self, mock_post): - exporter = OTLPMetricExporter(timeout=1.5) - - mock_post.side_effect = requests.exceptions.RequestException() - with self.assertLogs(level=WARNING) as warning: - self.assertEqual( - exporter.export(self.metrics["sum_int"]), - MetricExportResult.FAILURE, - ) - self.assertEqual(mock_post.call_count, 1) - self.assertIn( - "Failed to export metrics batch code", - warning.records[0].message, - ) - - @patch.object(Session, "post") - def test_timeout_set_correctly(self, mock_post): - resp = Response() - resp.status_code = 200 - - def export_side_effect(*args, **kwargs): - # Timeout should be set to something slightly less than 400 milliseconds depending on how much time has passed. - self.assertAlmostEqual(0.4, kwargs["timeout"], 2) - return resp - - mock_post.side_effect = export_side_effect - exporter = OTLPMetricExporter(timeout=0.4) - exporter.export(self.metrics["sum_int"]) - - @patch.object(Session, "post") - def test_shutdown_interrupts_retry_backoff(self, mock_post): - exporter = OTLPMetricExporter(timeout=1.5) - - resp = Response() - resp.status_code = 503 - resp.reason = "UNAVAILABLE" - mock_post.return_value = resp - thread = threading.Thread( - target=exporter.export, args=(self.metrics["sum_int"],) - ) - with self.assertLogs(level=WARNING) as warning: - before = time.time() - thread.start() - # Wait for the first attempt to fail, then enter a 1 second backoff. - time.sleep(0.05) - # Should cause export to wake up and return. - exporter.shutdown() - thread.join() - after = time.time() - self.assertIn( - "Transient error UNAVAILABLE encountered while exporting metrics batch, retrying in", - warning.records[0].message, - ) - self.assertIn( - "Shutdown in progress, aborting retry.", - warning.records[1].message, - ) - - assert after - before < 0.2 - - def assert_standard_metric_attrs(self, attributes): - self.assertEqual( - attributes["otel.component.type"], "otlp_http_metric_exporter" - ) - self.assertTrue( - attributes["otel.component.name"].startswith( - "otlp_http_metric_exporter/" - ) - ) - self.assertEqual(attributes["server.address"], "localhost") - self.assertEqual(attributes["server.port"], 4318) - def _resource_metrics( index: int, scope_metrics: list[pb2.ScopeMetrics] diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_internal.py b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_internal.py new file mode 100644 index 00000000000..16eff2c1d13 --- /dev/null +++ b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_internal.py @@ -0,0 +1,510 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=protected-access + +import os +import unittest +from logging import WARNING +from unittest.mock import patch + +import requests + +from opentelemetry.exporter.http.transport._requests import ( + RequestsHTTPTransport, +) +from opentelemetry.exporter.http.transport._urllib3 import ( + Urllib3HTTPTransport, +) +from opentelemetry.exporter.otlp.common._http import ( + Compression as CommonCompression, +) +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http._internal import ( + _DEFAULT_TIMEOUT, + _build_transport, + _normalize_compression, + _resolve_compression, + _resolve_endpoint, + _resolve_headers, + _resolve_session, + _resolve_timeout, +) +from opentelemetry.exporter.otlp.proto.http.version import __version__ +from opentelemetry.sdk.environment_variables import ( + OTEL_EXPORTER_OTLP_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_CLIENT_KEY, + OTEL_EXPORTER_OTLP_COMPRESSION, + OTEL_EXPORTER_OTLP_ENDPOINT, + OTEL_EXPORTER_OTLP_HEADERS, + OTEL_EXPORTER_OTLP_TIMEOUT, + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + OTEL_EXPORTER_OTLP_TRACES_HEADERS, + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT, +) + +_USER_AGENT = "OTel-OTLP-Exporter-Python/" + __version__ +_BASE_HEADERS = { + "content-type": "application/x-protobuf", + "user-agent": _USER_AGENT, +} +_CREDENTIAL_ENV_VAR = ( + "OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER" +) + + +class TestResolveInternal(unittest.TestCase): + def test_resolve_endpoint(self): + cases = [ + # per-signal wins over base and is returned verbatim (no path added) + ( + "per_signal_verbatim", + { + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://per-signal:4318/v1/traces", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://base:4318", + }, + "v1/traces", + "http://per-signal:4318/v1/traces", + ), + ( + "base_no_trailing_slash", + {OTEL_EXPORTER_OTLP_ENDPOINT: "http://base:4318"}, + "v1/traces", + "http://base:4318/v1/traces", + ), + ( + "base_trailing_slash_normalized", + {OTEL_EXPORTER_OTLP_ENDPOINT: "http://base:4318/"}, + "v1/metrics", + "http://base:4318/v1/metrics", + ), + ( + "empty_per_signal_falls_back", + { + OTEL_EXPORTER_OTLP_ENDPOINT: "http://base:4318", + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "", + }, + "v1/traces", + "http://base:4318/v1/traces", + ), + ( + "empty_base_falls_back_to_default", + {OTEL_EXPORTER_OTLP_ENDPOINT: ""}, + "v1/traces", + "http://localhost:4318/v1/traces", + ), + ( + "default_traces", + {}, + "v1/traces", + "http://localhost:4318/v1/traces", + ), + ( + "default_logs", + {}, + "v1/logs", + "http://localhost:4318/v1/logs", + ), + ] + for label, env, default_path, expected in cases: + with self.subTest(label), patch.dict(os.environ, env, clear=True): + self.assertEqual( + _resolve_endpoint( + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, default_path + ), + expected, + ) + + def test_resolve_headers(self): + cases = [ + ("defaults_only", {}, None, _BASE_HEADERS), + ( + "general_env_merged", + {OTEL_EXPORTER_OTLP_HEADERS: "k1=v1,k2=v2"}, + None, + {**_BASE_HEADERS, "k1": "v1", "k2": "v2"}, + ), + # per-signal var is used instead of (not merged with) the general one + ( + "per_signal_overrides_general", + { + OTEL_EXPORTER_OTLP_HEADERS: "api-key=general,shared=g", + OTEL_EXPORTER_OTLP_TRACES_HEADERS: "api-key=per-signal", + }, + None, + {**_BASE_HEADERS, "api-key": "per-signal"}, + ), + # empty per-signal var falls back to (inherits) the general one + ( + "empty_per_signal_falls_back_to_general", + { + OTEL_EXPORTER_OTLP_HEADERS: "api-key=general", + OTEL_EXPORTER_OTLP_TRACES_HEADERS: "", + }, + None, + {**_BASE_HEADERS, "api-key": "general"}, + ), + # explicit arg wins over env and base, by exact key + ( + "explicit_arg_overrides", + {OTEL_EXPORTER_OTLP_HEADERS: "api-key=from-env"}, + {"api-key": "explicit", "Content-Type": "text/plain"}, + { + "content-type": "text/plain", + "user-agent": _USER_AGENT, + "api-key": "explicit", + }, + ), + # env override of a default header must replace it + ( + "env_overrides_default_header_case_insensitively", + {OTEL_EXPORTER_OTLP_HEADERS: "user-agent=custom-agent"}, + None, + { + "content-type": "application/x-protobuf", + "user-agent": "custom-agent", + }, + ), + ( + "explicit_arg_overrides_default_header_case_insensitively", + {}, + {"User-Agent": "explicit-agent"}, + { + "content-type": "application/x-protobuf", + "user-agent": "explicit-agent", + }, + ), + ] + for label, env, headers_arg, expected in cases: + with self.subTest(label), patch.dict(os.environ, env, clear=True): + self.assertEqual( + _resolve_headers( + headers_arg, OTEL_EXPORTER_OTLP_TRACES_HEADERS + ), + expected, + ) + + def test_resolve_timeout(self): + cases = [ + ( + "per_signal_wins", + { + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "5", + OTEL_EXPORTER_OTLP_TIMEOUT: "7", + }, + 5.0, + False, + ), + ( + "falls_back_to_general", + {OTEL_EXPORTER_OTLP_TIMEOUT: "7"}, + 7.0, + False, + ), + ( + "fractional", + {OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "2.5"}, + 2.5, + False, + ), + ("default", {}, float(_DEFAULT_TIMEOUT), False), + ( + "empty_per_signal_falls_back_to_general", + { + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "", + OTEL_EXPORTER_OTLP_TIMEOUT: "7", + }, + 7.0, + False, + ), + ( + "empty_falls_back_to_default", + {OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: ""}, + float(_DEFAULT_TIMEOUT), + False, + ), + ( + "invalid_value_logs_and_defaults", + {OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "not-a-number"}, + float(_DEFAULT_TIMEOUT), + True, + ), + ] + for label, env, expected, errors in cases: + with self.subTest(label), patch.dict(os.environ, env, clear=True): + if errors: + with self.assertLogs(level=WARNING): + result = _resolve_timeout( + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT + ) + else: + result = _resolve_timeout( + OTEL_EXPORTER_OTLP_TRACES_TIMEOUT + ) + self.assertEqual(result, expected) + self.assertIsInstance(result, float) + + def test_resolve_compression(self): + cases = [ + ( + "gzip", + {OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: "gzip"}, + CommonCompression.GZIP, + False, + ), + ( + "deflate", + {OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: "deflate"}, + CommonCompression.DEFLATE, + False, + ), + ( + "none", + {OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: "none"}, + CommonCompression.NONE, + False, + ), + ( + "case_and_whitespace", + {OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: " GzIp "}, + CommonCompression.GZIP, + False, + ), + ( + "falls_back_to_general", + {OTEL_EXPORTER_OTLP_COMPRESSION: "deflate"}, + CommonCompression.DEFLATE, + False, + ), + ( + "per_signal_wins", + { + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: "gzip", + OTEL_EXPORTER_OTLP_COMPRESSION: "deflate", + }, + CommonCompression.GZIP, + False, + ), + ( + "empty_per_signal_falls_back_to_general", + { + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: "", + OTEL_EXPORTER_OTLP_COMPRESSION: "gzip", + }, + CommonCompression.GZIP, + False, + ), + ("default", {}, CommonCompression.NONE, False), + ( + "invalid_warns", + {OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: "bogus"}, + CommonCompression.NONE, + True, + ), + ] + for label, env, expected, warns in cases: + with self.subTest(label), patch.dict(os.environ, env, clear=True): + if warns: + with self.assertLogs(level=WARNING): + result = _resolve_compression( + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION + ) + else: + result = _resolve_compression( + OTEL_EXPORTER_OTLP_TRACES_COMPRESSION + ) + self.assertEqual(result, expected) + + def test_normalize_compression(self): + cases = [ + ("none_passthrough", None, None), + ( + "legacy_no_compression", + Compression.NoCompression, + CommonCompression.NONE, + ), + ("legacy_deflate", Compression.Deflate, CommonCompression.DEFLATE), + ("legacy_gzip", Compression.Gzip, CommonCompression.GZIP), + ( + "common_passthrough", + CommonCompression.GZIP, + CommonCompression.GZIP, + ), + ] + for label, given, expected in cases: + with self.subTest(label): + self.assertEqual(_normalize_compression(given), expected) + + def test_resolve_session(self): + explicit = requests.Session() + with patch.dict(os.environ, {}, clear=True): + self.assertIs( + _resolve_session(explicit, _CREDENTIAL_ENV_VAR), explicit + ) + self.assertIsNone(_resolve_session(None, _CREDENTIAL_ENV_VAR)) + + +class TestBuildTransport(unittest.TestCase): + def test_default_transport_is_urllib3(self): + with patch.dict(os.environ, {}, clear=True): + result = _build_transport( + None, + None, + None, + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + session=None, + ) + self.assertIsInstance(result, Urllib3HTTPTransport) + + def test_session_forces_requests_transport(self): + session = requests.Session() + with patch.dict(os.environ, {}, clear=True): + result = _build_transport( + None, + None, + None, + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + session=session, + ) + self.assertIsInstance(result, RequestsHTTPTransport) + # pylint: disable-next=protected-access + self.assertIs(result._session, session) + + def test_build_transport_verify_and_cert(self): + cases = [ + ( + "explicit_args_win", + { + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE: "env-cert.pem", + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY: "env-key.pem", + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE: "env-cert2.pem", + }, + "arg-cert.pem", + "arg-key.pem", + "arg-cert2.pem", + "arg-cert.pem", + ("arg-cert2.pem", "arg-key.pem"), + ), + ( + "per_signal_env_wins_over_general", + { + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE: "per-signal-cert.pem", + OTEL_EXPORTER_OTLP_CERTIFICATE: "general-cert.pem", + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY: "per-signal-key.pem", + OTEL_EXPORTER_OTLP_CLIENT_KEY: "general-key.pem", + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE: "per-signal-cert2.pem", + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: "general-cert2.pem", + }, + None, + None, + None, + "per-signal-cert.pem", + ("per-signal-cert2.pem", "per-signal-key.pem"), + ), + ( + "falls_back_to_general_env", + { + OTEL_EXPORTER_OTLP_CERTIFICATE: "general-cert.pem", + OTEL_EXPORTER_OTLP_CLIENT_KEY: "general-key.pem", + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: "general-cert2.pem", + }, + None, + None, + None, + "general-cert.pem", + ("general-cert2.pem", "general-key.pem"), + ), + ( + "empty_certificate_at_every_level_falls_back_to_default_true", + { + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE: "", + OTEL_EXPORTER_OTLP_CERTIFICATE: "", + }, + None, + None, + None, + True, + None, + ), + ( + "defaults_verify_true_no_cert", + {}, + None, + None, + None, + True, + None, + ), + ( + "only_cert_file_not_tupled", + { + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE: "cert2.pem", + }, + None, + None, + None, + True, + "cert2.pem", + ), + ] + for ( + label, + env, + certificate_file, + client_key_file, + client_certificate_file, + expected_verify, + expected_cert, + ) in cases: + with self.subTest(label), patch.dict(os.environ, env, clear=True): + with patch( + "opentelemetry.exporter.otlp.proto.http._internal.Urllib3HTTPTransport" + ) as mock_transport: + result = _build_transport( + certificate_file, + client_key_file, + client_certificate_file, + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + session=None, + ) + mock_transport.assert_called_once_with( + verify=expected_verify, cert=expected_cert + ) + self.assertIs(result, mock_transport.return_value) + + def test_build_transport_passes_verify_and_cert_to_requests(self): + session = requests.Session() + with ( + patch.dict( + os.environ, + {OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE: "cert.pem"}, + clear=True, + ), + patch( + "opentelemetry.exporter.otlp.proto.http._internal.RequestsHTTPTransport" + ) as mock_transport, + ): + result = _build_transport( + None, + None, + None, + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + session=session, + ) + mock_transport.assert_called_once_with( + verify="cert.pem", cert=None, session=session + ) + self.assertIs(result, mock_transport.return_value) diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_log_exporter.py b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_log_exporter.py index d7f3592e288..baa52638234 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_log_exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_log_exporter.py @@ -1,30 +1,39 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 -# pylint: disable=protected-access +# pylint: disable=protected-access,too-many-public-methods +import gzip +import os import threading import time import unittest -from logging import WARNING -from unittest.mock import MagicMock, Mock, patch +import zlib +from unittest.mock import Mock, patch import requests -from google.protobuf.json_format import MessageToDict -from requests import Session -from requests.exceptions import ConnectionError -from requests.models import Response +import urllib3.exceptions +from mocket import Mocket, Mocketizer, mocketize +from mocket.mocks.mockhttp import Entry, Response from opentelemetry._logs import LogRecord, SeverityNumber +from opentelemetry.exporter.http.transport._requests import ( + RequestsHTTPTransport, +) +from opentelemetry.exporter.http.transport._urllib3 import ( + Urllib3HTTPTransport, +) +from opentelemetry.exporter.otlp.common._http import ( + Compression as CommonCompression, +) +from opentelemetry.exporter.otlp.proto.common._log_encoder import encode_logs from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http._internal import _build_transport from opentelemetry.exporter.otlp.proto.http._log_exporter import ( - DEFAULT_COMPRESSION, DEFAULT_ENDPOINT, DEFAULT_LOGS_EXPORT_PATH, - DEFAULT_TIMEOUT, OTLPLogExporter, ) -from opentelemetry.exporter.otlp.proto.http.version import __version__ from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ( ExportLogsServiceRequest, ) @@ -32,20 +41,13 @@ from opentelemetry.sdk._logs.export import LogRecordExportResult from opentelemetry.sdk.environment_variables import ( _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER, - OTEL_EXPORTER_OTLP_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE, OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY, - OTEL_EXPORTER_OTLP_LOGS_COMPRESSION, OTEL_EXPORTER_OTLP_LOGS_ENDPOINT, OTEL_EXPORTER_OTLP_LOGS_HEADERS, - OTEL_EXPORTER_OTLP_LOGS_TIMEOUT, - OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, ) from opentelemetry.sdk.metrics import MeterProvider @@ -53,617 +55,500 @@ from opentelemetry.sdk.resources import Resource as SDKResource from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.test.mock_test_classes import IterEntryPoint -from opentelemetry.trace import ( - NonRecordingSpan, - SpanContext, - TraceFlags, - set_span_in_context, -) -ENV_ENDPOINT = "http://localhost.env:8080/" -ENV_CERTIFICATE = "/etc/base.crt" -ENV_CLIENT_CERTIFICATE = "/etc/client-cert.pem" -ENV_CLIENT_KEY = "/etc/client-key.pem" -ENV_HEADERS = "envHeader1=val1,envHeader2=val2,User-agent=Overridden" -ENV_TIMEOUT = "30" +from . import _mock_clock + +_TEST_ENDPOINT = "http://localhost:4318/v1/logs" +_LOGGER_NAME = "opentelemetry.exporter.otlp.proto.http._log_exporter" + + +def _decode_body(body: bytes) -> ExportLogsServiceRequest: + return ExportLogsServiceRequest.FromString(body) + + +def _make_log_record() -> ReadWriteLogRecord: + return ReadWriteLogRecord( + LogRecord( + timestamp=1644650195189786182, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="a log message", + attributes={"a": 1, "b": "c"}, + ), + resource=SDKResource({"first_resource": "value"}), + instrumentation_scope=InstrumentationScope("name", "version"), + ) class TestOTLPHTTPLogExporter(unittest.TestCase): def setUp(self): + env_patcher = patch.dict(os.environ, {}, clear=True) + env_patcher.start() + self.addCleanup(env_patcher.stop) + self.metric_reader = InMemoryMetricReader() self.meter_provider = MeterProvider( metric_readers=[self.metric_reader] ) - def test_constructor_default(self): - exporter = OTLPLogExporter() + @staticmethod + def _mocked_shutdown_event() -> Mock: + shutdown_event = Mock(spec=threading.Event) + shutdown_event.is_set.return_value = False + return shutdown_event + def assert_standard_metric_attrs(self, attributes): self.assertEqual( - exporter._endpoint, DEFAULT_ENDPOINT + DEFAULT_LOGS_EXPORT_PATH - ) - self.assertEqual(exporter._certificate_file, True) - self.assertEqual(exporter._client_certificate_file, None) - self.assertEqual(exporter._client_key_file, None) - self.assertEqual(exporter._timeout, DEFAULT_TIMEOUT) - self.assertIs(exporter._compression, DEFAULT_COMPRESSION) - self.assertEqual(exporter._headers, {}) - self.assertIsInstance(exporter._session, requests.Session) - self.assertIn("User-Agent", exporter._session.headers) - self.assertEqual( - exporter._session.headers.get("Content-Type"), - "application/x-protobuf", + attributes["otel.component.type"], "otlp_http_log_exporter" ) - self.assertEqual( - exporter._session.headers.get("User-Agent"), - "OTel-OTLP-Exporter-Python/" + __version__, + self.assertTrue( + attributes["otel.component.name"].startswith( + "otlp_http_log_exporter/" + ) ) + self.assertEqual(attributes["server.address"], "localhost") + self.assertEqual(attributes["server.port"], 4318) - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_CERTIFICATE: ENV_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: ENV_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY: ENV_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION: Compression.Gzip.value, - OTEL_EXPORTER_OTLP_ENDPOINT: ENV_ENDPOINT, - OTEL_EXPORTER_OTLP_HEADERS: ENV_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT: ENV_TIMEOUT, - OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE: "logs/certificate.env", - OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE: "logs/client-cert.pem", - OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY: "logs/client-key.pem", - OTEL_EXPORTER_OTLP_LOGS_COMPRESSION: Compression.Deflate.value, - OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: "https://logs.endpoint.env", - OTEL_EXPORTER_OTLP_LOGS_HEADERS: "logsEnv1=val1,logsEnv2=val2,logsEnv3===val3==,User-agent=LogsUserAgent", - OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "40", - _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER: "credential_provider", - }, - ) - @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") - def test_exporter_logs_env_take_priority(self, mock_entry_points): - credential = Session() - - def f(): - return credential + # -- construction / transport selection -------------------------------- - mock_entry_points.configure_mock( - return_value=[IterEntryPoint("custom_credential", f)] - ) + def test_constructor_default_uses_urllib3_transport(self): exporter = OTLPLogExporter() - self.assertEqual(exporter._endpoint, "https://logs.endpoint.env") - self.assertEqual(exporter._certificate_file, "logs/certificate.env") self.assertEqual( - exporter._client_certificate_file, "logs/client-cert.pem" + exporter._endpoint, DEFAULT_ENDPOINT + DEFAULT_LOGS_EXPORT_PATH ) - self.assertEqual(exporter._client_key_file, "logs/client-key.pem") - self.assertEqual(exporter._timeout, 40) - self.assertIs(exporter._compression, Compression.Deflate) - self.assertEqual( - exporter._headers, + self.assertIs(exporter._compression, CommonCompression.NONE) + self.assertIsNone(exporter._session) + self.assertIsInstance( + exporter._client._transport, Urllib3HTTPTransport + ) + + def test_explicit_session_uses_requests_transport(self): + session = requests.Session() + exporter = OTLPLogExporter(session=session) + + self.assertIs(exporter._session, session) + self.assertIsInstance( + exporter._client._transport, RequestsHTTPTransport + ) + self.assertIs(exporter._client._transport._session, session) + + @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") + def test_credential_provider_uses_requests_transport( + self, mock_entry_point + ): + credential = requests.Session() + mock_entry_point.configure_mock( + return_value=[ + IterEntryPoint("custom_credential", lambda: credential) + ] + ) + with patch.dict( + os.environ, { - "logsenv1": "val1", - "logsenv2": "val2", - "logsenv3": "==val3==", - "user-agent": "LogsUserAgent", + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER: "custom_credential", }, - ) + ): + exporter = OTLPLogExporter() + self.assertIs(exporter._session, credential) - self.assertIsInstance(exporter._session, requests.Session) - self.assertEqual( - exporter._session.headers.get("User-Agent"), - "LogsUserAgent", - ) - self.assertEqual( - exporter._session.headers.get("Content-Type"), - "application/x-protobuf", + self.assertIsInstance( + exporter._client._transport, RequestsHTTPTransport ) - @patch.dict( - "os.environ", - { - _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER: "provider_without_entry_point", - }, - ) @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") def test_exception_raised_when_entrypoint_returns_wrong_type( - self, mock_entry_points + self, mock_entry_point ): - def f(): - return 1 - - mock_entry_points.configure_mock( - return_value=[IterEntryPoint("custom_credential", f)] - ) - with self.assertRaises(RuntimeError): + mock_entry_point.configure_mock( + return_value=[IterEntryPoint("bad_credential", lambda: 1)] + ) + with ( + patch.dict( + os.environ, + { + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER: "bad_credential", + }, + ), + self.assertRaises(RuntimeError), + ): OTLPLogExporter() - @patch.dict( - "os.environ", - { - _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER: "provider_without_entry_point", - }, - ) - def test_exception_raised_when_entrypoint_does_not_exist(self): - with self.assertRaises(RuntimeError): + @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") + def test_exception_raised_when_entrypoint_does_not_exist( + self, mock_entry_point + ): + mock_entry_point.configure_mock(return_value=[]) + with ( + patch.dict( + os.environ, + { + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_LOGS_CREDENTIAL_PROVIDER: "missing", + }, + ), + self.assertRaises(RuntimeError), + ): OTLPLogExporter() - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_CERTIFICATE: ENV_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: ENV_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY: ENV_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION: Compression.Gzip.value, - OTEL_EXPORTER_OTLP_ENDPOINT: ENV_ENDPOINT, - OTEL_EXPORTER_OTLP_HEADERS: ENV_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT: ENV_TIMEOUT, - }, - ) - def test_exporter_constructor_take_priority(self): - sess = MagicMock() - exporter = OTLPLogExporter( - endpoint="endpoint.local:69/logs", - certificate_file="/hello.crt", - client_key_file="/client-key.pem", - client_certificate_file="/client-cert.pem", - headers={"testHeader1": "value1", "testHeader2": "value2"}, - timeout=70, - compression=Compression.NoCompression, - session=sess(), + def test_compression_dual_enum_acceptance(self): + for compression in (Compression.Gzip, CommonCompression.GZIP): + with self.subTest(compression=compression): + exporter = OTLPLogExporter(compression=compression) + self.assertIs(exporter._compression, CommonCompression.GZIP) + + # -- export / wire format ------------------------------------------------ + + @mocketize + def test_export_single_log(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT) + logs = [_make_log_record()] + transport = exporter._client._transport + + with patch.object( + transport, "request", wraps=transport.request + ) as mock_request: + result = exporter.export(logs) + + self.assertEqual(result, LogRecordExportResult.SUCCESS) + request = Mocket.last_request() + self.assertEqual(request.method, "POST") + self.assertEqual(request.path, "/v1/logs") + sent_data = mock_request.call_args.kwargs["data"] + self.assertEqual(_decode_body(sent_data), encode_logs(logs)) + + @mocketize + def test_default_endpoint_and_headers(self): + Entry.single_register( + Entry.POST, "http://localhost:4318/v1/logs", status=200 ) - - self.assertEqual(exporter._endpoint, "endpoint.local:69/logs") - self.assertEqual(exporter._certificate_file, "/hello.crt") - self.assertEqual(exporter._client_certificate_file, "/client-cert.pem") - self.assertEqual(exporter._client_key_file, "/client-key.pem") - self.assertEqual(exporter._timeout, 70) - self.assertIs(exporter._compression, Compression.NoCompression) - self.assertEqual( - exporter._headers, - {"testHeader1": "value1", "testHeader2": "value2"}, - ) - self.assertTrue(sess.called) - - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_CERTIFICATE: ENV_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: ENV_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY: ENV_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION: Compression.Gzip.value, - OTEL_EXPORTER_OTLP_ENDPOINT: ENV_ENDPOINT, - OTEL_EXPORTER_OTLP_HEADERS: ENV_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT: ENV_TIMEOUT, - }, - ) - def test_exporter_env(self): exporter = OTLPLogExporter() - self.assertEqual( - exporter._endpoint, ENV_ENDPOINT + DEFAULT_LOGS_EXPORT_PATH - ) - self.assertEqual(exporter._certificate_file, ENV_CERTIFICATE) - self.assertEqual( - exporter._client_certificate_file, ENV_CLIENT_CERTIFICATE - ) - self.assertEqual(exporter._client_key_file, ENV_CLIENT_KEY) - self.assertEqual(exporter._timeout, int(ENV_TIMEOUT)) - self.assertIs(exporter._compression, Compression.Gzip) - self.assertEqual( - exporter._headers, - { - "envheader1": "val1", - "envheader2": "val2", - "user-agent": "Overridden", - }, - ) - self.assertIsInstance(exporter._session, requests.Session) + result = exporter.export([_make_log_record()]) - @staticmethod - def export_log_and_deserialize(log): - with patch("requests.Session.post") as mock_post: - exporter = OTLPLogExporter() - exporter.export([log]) - request_body = mock_post.call_args[1]["data"] - request = ExportLogsServiceRequest() - request.ParseFromString(request_body) - request_dict = MessageToDict(request) - log_records = ( - request_dict.get("resourceLogs")[0] - .get("scopeLogs")[0] - .get("logRecords") - ) - return log_records - - def test_exported_log_without_trace_id(self): - ctx = set_span_in_context( - NonRecordingSpan( - SpanContext( - 0, - 1312458408527513292, - False, - TraceFlags(0x01), - ) - ) - ) - log = ReadWriteLogRecord( - LogRecord( - timestamp=1644650195189786182, - context=ctx, - severity_text="WARN", - severity_number=SeverityNumber.WARN, - body="Invalid trace id check", - attributes={"a": 1, "b": "c"}, - ), - resource=SDKResource({"first_resource": "value"}), - instrumentation_scope=InstrumentationScope("name", "version"), - ) - log_records = TestOTLPHTTPLogExporter.export_log_and_deserialize(log) - if log_records: - log_record = log_records[0] - self.assertIn("spanId", log_record) - self.assertNotIn( - "traceId", - log_record, - "trace_id should not be present in the log record", - ) - else: - self.fail("No log records found") - - def test_exported_log_without_span_id(self): - ctx = set_span_in_context( - NonRecordingSpan( - SpanContext( - 89564621134313219400156819398935297696, - 0, - False, - TraceFlags(0x01), - ) - ) + self.assertEqual(result, LogRecordExportResult.SUCCESS) + headers = Mocket.last_request().headers + self.assertEqual(headers["content-type"], "application/x-protobuf") + self.assertTrue( + headers["user-agent"].startswith("OTel-OTLP-Exporter-Python/") ) - log = ReadWriteLogRecord( - LogRecord( - timestamp=1644650195189786360, - context=ctx, - severity_text="WARN", - severity_number=SeverityNumber.WARN, - body="Invalid span id check", - attributes={"a": 1, "b": "c"}, + def test_custom_endpoint(self): + url = "http://custom.example:9999/v1/logs" + cases = ( + ("constructor", {}, {"endpoint": url}), + ( + "generic_env", + {OTEL_EXPORTER_OTLP_ENDPOINT: "http://custom.example:9999"}, + {}, + ), + ("per_signal_env", {OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: url}, {}), + ) + for label, env, kwargs in cases: + with ( + self.subTest(label), + patch.dict(os.environ, env, clear=True), + Mocketizer(), + ): + Entry.single_register(Entry.POST, url, status=200) + exporter = OTLPLogExporter(**kwargs) + + result = exporter.export([_make_log_record()]) + + self.assertEqual(result, LogRecordExportResult.SUCCESS) + + def test_custom_headers(self): + cases = ( + ("constructor", {}, {"headers": {"x-api-key": "secret"}}), + ( + "generic_env", + {OTEL_EXPORTER_OTLP_HEADERS: "x-api-key=secret"}, + {}, + ), + ( + "per_signal_env", + {OTEL_EXPORTER_OTLP_LOGS_HEADERS: "x-api-key=secret"}, + {}, ), - resource=SDKResource({"first_resource": "value"}), - instrumentation_scope=InstrumentationScope("name", "version"), ) - log_records = TestOTLPHTTPLogExporter.export_log_and_deserialize(log) - if log_records: - log_record = log_records[0] - self.assertIn("traceId", log_record) - self.assertNotIn( - "spanId", - log_record, - "spanId should not be present in the log record", + for label, env, kwargs in cases: + with ( + self.subTest(label), + patch.dict(os.environ, env, clear=True), + Mocketizer(), + ): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT, **kwargs) + + exporter.export([_make_log_record()]) + + headers = Mocket.last_request().headers + self.assertEqual(headers["x-api-key"], "secret") + + @mocketize + def test_custom_transport(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + custom_transport = Urllib3HTTPTransport() + + with patch( + "opentelemetry.exporter.otlp.proto.http._log_exporter._build_transport" + ) as mock_build_transport: + exporter = OTLPLogExporter( + endpoint=_TEST_ENDPOINT, _transport=custom_transport ) - else: - self.fail("No log records found") - @staticmethod - def _get_sdk_log_data() -> list[ReadWriteLogRecord]: - ctx_log1 = set_span_in_context( - NonRecordingSpan( - SpanContext( - 89564621134313219400156819398935297684, - 1312458408527513268, - False, - TraceFlags(0x01), - ) + mock_build_transport.assert_not_called() + self.assertIs(exporter._client._transport, custom_transport) + + result = exporter.export([_make_log_record()]) + self.assertEqual(result, LogRecordExportResult.SUCCESS) + + @mocketize + def test_certificate_args(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + + with patch( + "opentelemetry.exporter.otlp.proto.http._log_exporter._build_transport", + wraps=_build_transport, + ) as mock_build_transport: + exporter = OTLPLogExporter( + endpoint=_TEST_ENDPOINT, + certificate_file="ca.pem", + client_key_file="client-key.pem", + client_certificate_file="client-cert.pem", ) - ) - log1 = ReadWriteLogRecord( - LogRecord( - timestamp=1644650195189786880, - context=ctx_log1, - severity_text="WARN", - severity_number=SeverityNumber.WARN, - body="Do not go gentle into that good night. Rage, rage against the dying of the light", - attributes={"a": 1, "b": "c"}, - ), - resource=SDKResource({"first_resource": "value"}), - instrumentation_scope=InstrumentationScope( - "first_name", "first_version" - ), - ) - ctx_log2 = set_span_in_context( - NonRecordingSpan( - SpanContext( - 0, - 0, - False, + mock_build_transport.assert_called_once_with( + "ca.pem", + "client-key.pem", + "client-cert.pem", + OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE, + OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY, + OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE, + session=None, + ) + + result = exporter.export([_make_log_record()]) + self.assertEqual(result, LogRecordExportResult.SUCCESS) + + def test_compression_options(self): + cases = ( + (Compression.NoCompression, None, lambda data: data), + (Compression.Gzip, "gzip", gzip.decompress), + (Compression.Deflate, "deflate", zlib.decompress), + ) + for compression, expected_encoding, decompress in cases: + with self.subTest(compression=compression), Mocketizer(): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPLogExporter( + endpoint=_TEST_ENDPOINT, compression=compression ) - ) - ) - log2 = ReadWriteLogRecord( - LogRecord( - timestamp=1644650249738562048, - context=ctx_log2, - severity_text="WARN", - severity_number=SeverityNumber.WARN, - body="Cooper, this is no time for caution!", - attributes={}, - ), - resource=SDKResource({"second_resource": "CASE"}), - instrumentation_scope=InstrumentationScope( - "second_name", "second_version" - ), - ) - ctx_log3 = set_span_in_context( - NonRecordingSpan( - SpanContext( - 271615924622795969659406376515024083555, - 4242561578944770265, - False, - TraceFlags(0x01), + transport = exporter._client._transport + logs = [_make_log_record()] + + with patch.object( + transport, "request", wraps=transport.request + ) as mock_request: + result = exporter.export(logs) + + self.assertEqual(result, LogRecordExportResult.SUCCESS) + sent_headers = mock_request.call_args.kwargs["headers"] + if expected_encoding is None: + self.assertNotIn("Content-Encoding", sent_headers) + else: + self.assertEqual( + sent_headers["Content-Encoding"], expected_encoding + ) + sent_data = mock_request.call_args.kwargs["data"] + decompressed = decompress(sent_data) + self.assertEqual(_decode_body(decompressed), encode_logs(logs)) + + # -- retry / backoff ------------------------------------------------------ + + def test_export_retryable_status_codes(self): + for status_code in (429, 502, 503, 504): + with self.subTest(status_code=status_code), Mocketizer(): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=status_code), + Response(status=200), ) - ) - ) - log3 = ReadWriteLogRecord( - LogRecord( - timestamp=1644650427658989056, - context=ctx_log3, - severity_text="DEBUG", - severity_number=SeverityNumber.DEBUG, - body="To our galaxy", - attributes={"a": 1, "b": "c"}, - ), - resource=SDKResource({"second_resource": "CASE"}), - instrumentation_scope=None, - ) - ctx_log4 = set_span_in_context( - NonRecordingSpan( - SpanContext( - 212592107417388365804938480559624925555, - 6077757853989569223, - False, - TraceFlags(0x01), + exporter = OTLPLogExporter( + endpoint=_TEST_ENDPOINT, timeout=30.0 ) - ) - ) - log4 = ReadWriteLogRecord( - LogRecord( - timestamp=1644650584292683008, - context=ctx_log4, - severity_text="INFO", - severity_number=SeverityNumber.INFO, - body="Love is the one thing that transcends time and space", - attributes={"filename": "model.py", "func_name": "run_method"}, - ), - resource=SDKResource({"first_resource": "value"}), - instrumentation_scope=InstrumentationScope( - "another_name", "another_version" - ), - ) + shutdown_event = self._mocked_shutdown_event() + exporter._client._shutdown_event = shutdown_event - return [log1, log2, log3, log4] + with _mock_clock(shutdown_event): + result = exporter.export([_make_log_record()]) - @patch.object(OTLPLogExporter, "_export", return_value=Mock(ok=True)) - def test_2xx_status_code(self, mock_otlp_metric_exporter): - """ - Test that any HTTP 2XX code returns a successful result - """ + self.assertEqual(result, LogRecordExportResult.SUCCESS) + self.assertEqual(len(Mocket.request_list()), 2) - self.assertEqual( - OTLPLogExporter().export(MagicMock()), - LogRecordExportResult.SUCCESS, - ) + def test_export_non_retryable_status_codes(self): + for status_code in (400, 401, 403, 404, 408, 500, 501): + with self.subTest(status_code=status_code), Mocketizer(): + Entry.single_register( + Entry.POST, _TEST_ENDPOINT, status=status_code + ) + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT) - @patch.dict( - "os.environ", {OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: " true "} - ) - @patch.object(Session, "post") - def test_retry_timeout(self, mock_post): - exporter = OTLPLogExporter( - timeout=1.5, meter_provider=self.meter_provider - ) + result = exporter.export([_make_log_record()]) - resp = Response() - resp.status_code = 503 - resp.reason = "UNAVAILABLE" - mock_post.return_value = resp - with self.assertLogs(level=WARNING) as warning: - before = time.time() - # Set timeout to 1.5 seconds - self.assertEqual( - exporter.export(self._get_sdk_log_data()), - LogRecordExportResult.FAILURE, - ) - after = time.time() - # First call at time 0, second at time 1, then an early return before the second backoff sleep b/c it would exceed timeout. - self.assertEqual(mock_post.call_count, 2) - # There's a +/-20% jitter on each backoff. - self.assertTrue(0.75 < after - before < 1.25) - self.assertIn( - "Transient error UNAVAILABLE encountered while exporting logs batch, retrying in", - warning.records[0].message, - ) + self.assertEqual(result, LogRecordExportResult.FAILURE) + self.assertEqual(len(Mocket.request_list()), 1) - metrics_data = self.metric_reader.get_metrics_data() - scope_metrics = metrics_data.resource_metrics[0].scope_metrics[0] - self.assertEqual(scope_metrics.scope.name, "opentelemetry-sdk") - metrics = sorted(scope_metrics.metrics, key=lambda m: m.name) - self.assertEqual(len(metrics), 3) - self.assertEqual(metrics[0].name, "otel.sdk.exporter.log.exported") - self.assert_standard_metric_attrs( - metrics[0].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[0].data.data_points[0].attributes - ) - self.assertNotIn( - "http.response.status_code", - metrics[0].data.data_points[0].attributes, + @mocketize + def test_export_connection_error(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + urllib3.exceptions.ProtocolError("simulated reset"), + Response(status=200), ) - self.assertEqual(metrics[1].name, "otel.sdk.exporter.log.inflight") - self.assert_standard_metric_attrs( - metrics[1].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[1].data.data_points[0].attributes - ) - self.assertNotIn( - "http.response.status_code", - metrics[1].data.data_points[0].attributes, - ) - self.assertEqual( - metrics[2].name, "otel.sdk.exporter.operation.duration" + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT, timeout=5.0) + + result = exporter.export([_make_log_record()]) + + self.assertEqual(result, LogRecordExportResult.SUCCESS) + self.assertEqual(len(Mocket.request_list()), 2) + + @mocketize + def test_shutdown_interrupts_retry_backoff(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + *[Response(status=503)] * 6, ) - self.assert_standard_metric_attrs( - metrics[2].data.data_points[0].attributes + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT, timeout=1.5) + thread = threading.Thread( + target=exporter.export, args=([_make_log_record()],) ) - self.assertNotIn( - "error.type", metrics[2].data.data_points[0].attributes + before = time.time() + thread.start() + time.sleep(0.05) + exporter.shutdown() + thread.join() + after = time.time() + + self.assertLess(after - before, 0.5) + + # -- self-observability metrics ------------------------------------------- + + @mocketize + def test_exporter_metrics_disabled_by_default(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPLogExporter( + endpoint=_TEST_ENDPOINT, meter_provider=self.meter_provider ) + self.assertEqual( - metrics[2] - .data.data_points[0] - .attributes["http.response.status_code"], - 503, + exporter.export([_make_log_record()]), + LogRecordExportResult.SUCCESS, ) - - @patch.object(Session, "post") - def test_export_no_collector_available_retryable(self, mock_post): - exporter = OTLPLogExporter(timeout=1.5) - msg = "Server not available." - mock_post.side_effect = ConnectionError(msg) - with self.assertLogs(level=WARNING) as warning: - self.assertEqual( - exporter.export(self._get_sdk_log_data()), - LogRecordExportResult.FAILURE, - ) - # Check for greater 2 because the request is on each retry - # done twice at the moment. - self.assertGreater(mock_post.call_count, 2) - self.assertIn( - f"Transient error {msg} encountered while exporting logs batch, retrying in", - warning.records[0].message, - ) + self.assertIsNone(self.metric_reader.get_metrics_data()) @patch.dict( "os.environ", {OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true"} ) - @patch.object(Session, "post") - def test_export_no_collector_available(self, mock_post): - exporter = OTLPLogExporter( - timeout=1.5, meter_provider=self.meter_provider - ) - - mock_post.side_effect = requests.exceptions.RequestException() - with self.assertLogs(level=WARNING) as warning: - self.assertEqual( - exporter.export(self._get_sdk_log_data()), - LogRecordExportResult.FAILURE, + def test_retry_timeout_records_metrics(self): + with Mocketizer(): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=503), + Response(status=503), ) - self.assertEqual(mock_post.call_count, 1) - self.assertIn( - "Failed to export logs batch code", - warning.records[0].message, + exporter = OTLPLogExporter( + endpoint=_TEST_ENDPOINT, + timeout=1.5, + meter_provider=self.meter_provider, ) + shutdown_event = self._mocked_shutdown_event() + exporter._client._shutdown_event = shutdown_event + + with _mock_clock(shutdown_event): + result = exporter.export([_make_log_record()]) + + self.assertEqual(result, LogRecordExportResult.FAILURE) metrics_data = self.metric_reader.get_metrics_data() scope_metrics = metrics_data.resource_metrics[0].scope_metrics[0] - self.assertEqual(scope_metrics.scope.name, "opentelemetry-sdk") metrics = sorted(scope_metrics.metrics, key=lambda m: m.name) self.assertEqual(len(metrics), 3) - self.assertEqual(metrics[0].name, "otel.sdk.exporter.log.exported") - self.assert_standard_metric_attrs( - metrics[0].data.data_points[0].attributes - ) - self.assertEqual( - metrics[0].data.data_points[0].attributes["error.type"], - "RequestException", - ) - self.assertNotIn( - "http.response.status_code", - metrics[0].data.data_points[0].attributes, + names = [m.name for m in metrics] + self.assertIn("otel.sdk.exporter.log.exported", names) + self.assertIn("otel.sdk.exporter.log.inflight", names) + self.assertIn("otel.sdk.exporter.operation.duration", names) + duration_metric = next( + m + for m in metrics + if m.name == "otel.sdk.exporter.operation.duration" ) - self.assertEqual(metrics[1].name, "otel.sdk.exporter.log.inflight") self.assert_standard_metric_attrs( - metrics[1].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[1].data.data_points[0].attributes - ) - self.assertNotIn( - "http.response.status_code", - metrics[1].data.data_points[0].attributes, - ) - self.assertEqual( - metrics[2].name, "otel.sdk.exporter.operation.duration" - ) - self.assert_standard_metric_attrs( - metrics[2].data.data_points[0].attributes + duration_metric.data.data_points[0].attributes ) self.assertEqual( - metrics[2].data.data_points[0].attributes["error.type"], - "RequestException", - ) - self.assertNotIn( - "http.response.status_code", - metrics[2].data.data_points[0].attributes, + duration_metric.data.data_points[0].attributes[ + "http.response.status_code" + ], + 503, ) - @patch.object(Session, "post") - def test_timeout_set_correctly(self, mock_post): - resp = Response() - resp.status_code = 200 + # -- misc ----------------------------------------------------------------- - def export_side_effect(*args, **kwargs): - # Timeout should be set to something slightly less than 400 milliseconds depending on how much time has passed. - self.assertAlmostEqual(0.4, kwargs["timeout"], 2) - return resp + @mocketize + def test_export_after_shutdown(self): + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT) + exporter.shutdown() - mock_post.side_effect = export_side_effect - exporter = OTLPLogExporter(timeout=0.4) - exporter.export(self._get_sdk_log_data()) + with self.assertLogs(_LOGGER_NAME, level="WARNING"): + result = exporter.export([_make_log_record()]) - @patch.object(Session, "post") - def test_shutdown_interrupts_retry_backoff(self, mock_post): - exporter = OTLPLogExporter(timeout=1.5) + self.assertEqual(result, LogRecordExportResult.FAILURE) + self.assertIsNone(Mocket.last_request()) - resp = Response() - resp.status_code = 503 - resp.reason = "UNAVAILABLE" - mock_post.return_value = resp - thread = threading.Thread( - target=exporter.export, args=(self._get_sdk_log_data(),) - ) - with self.assertLogs(level=WARNING) as warning: - before = time.time() - thread.start() - # Wait for the first attempt to fail, then enter a 1 second backoff. - time.sleep(0.05) - # Should cause export to wake up and return. + def test_shutdown_idempotent(self): + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT) + exporter.shutdown() + + with self.assertLogs(_LOGGER_NAME, level="WARNING"): exporter.shutdown() - thread.join() - after = time.time() - self.assertIn( - "Transient error UNAVAILABLE encountered while exporting logs batch, retrying in", - warning.records[0].message, - ) - self.assertIn( - "Shutdown in progress, aborting retry.", - warning.records[1].message, - ) - assert after - before < 0.2 + def test_shutdown_closes_transport(self): + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT) - def assert_standard_metric_attrs(self, attributes): - self.assertEqual( - attributes["otel.component.type"], "otlp_http_log_exporter" - ) - self.assertTrue( - attributes["otel.component.name"].startswith( - "otlp_http_log_exporter/" - ) - ) - self.assertEqual(attributes["server.address"], "localhost") - self.assertEqual(attributes["server.port"], 4318) + with patch.object(exporter._client._transport, "close") as mock_close: + exporter.shutdown() + + mock_close.assert_called_once() + + @mocketize + def test_force_flush(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT) + + self.assertTrue(exporter.force_flush()) + exporter.export([_make_log_record()]) + self.assertTrue(exporter.force_flush()) + + @mocketize + def test_export_encoding_failure(self): + exporter = OTLPLogExporter(endpoint=_TEST_ENDPOINT) + + with ( + patch( + "opentelemetry.exporter.otlp.proto.http._log_exporter.encode_logs", + side_effect=ValueError("boom"), + ), + self.assertLogs(_LOGGER_NAME, level="ERROR"), + ): + result = exporter.export([_make_log_record()]) + + self.assertEqual(result, LogRecordExportResult.FAILURE) + self.assertIsNone(Mocket.last_request()) + + +if __name__ == "__main__": + unittest.main() diff --git a/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_span_exporter.py b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_span_exporter.py index 1580e5a1802..93434e5fb15 100644 --- a/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_span_exporter.py +++ b/exporter/opentelemetry-exporter-otlp-proto-http/tests/test_proto_span_exporter.py @@ -1,326 +1,583 @@ # Copyright The OpenTelemetry Authors # SPDX-License-Identifier: Apache-2.0 +# pylint: disable=protected-access,too-many-public-methods + +import gzip +import os import threading import time import unittest -from logging import WARNING -from unittest.mock import MagicMock, Mock, patch +import zlib +from datetime import datetime, timezone +from email.utils import format_datetime +from unittest.mock import Mock, patch import requests -from requests import Session -from requests.exceptions import ConnectionError -from requests.models import Response +import urllib3.exceptions +from mocket import Mocket, Mocketizer, mocketize +from mocket.mocks.mockhttp import Entry, Response +from opentelemetry.exporter.http.transport._requests import ( + RequestsHTTPTransport, +) +from opentelemetry.exporter.http.transport._urllib3 import ( + Urllib3HTTPTransport, +) +from opentelemetry.exporter.otlp.common._http import ( + Compression as CommonCompression, +) +from opentelemetry.exporter.otlp.proto.common.trace_encoder import ( + encode_spans, +) from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.exporter.otlp.proto.http._internal import _build_transport from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( - DEFAULT_COMPRESSION, DEFAULT_ENDPOINT, - DEFAULT_TIMEOUT, DEFAULT_TRACES_EXPORT_PATH, OTLPSpanExporter, ) -from opentelemetry.exporter.otlp.proto.http.version import __version__ +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, +) from opentelemetry.sdk.environment_variables import ( _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER, - OTEL_EXPORTER_OTLP_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_ENDPOINT, OTEL_EXPORTER_OTLP_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT, OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, - OTEL_EXPORTER_OTLP_TRACES_COMPRESSION, OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, OTEL_EXPORTER_OTLP_TRACES_HEADERS, - OTEL_EXPORTER_OTLP_TRACES_TIMEOUT, OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, ) from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader -from opentelemetry.sdk.trace import _Span -from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExportResult, +) +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) from opentelemetry.test.mock_test_classes import IterEntryPoint -OS_ENV_ENDPOINT = "os.env.base" -OS_ENV_CERTIFICATE = "os/env/base.crt" -OS_ENV_CLIENT_CERTIFICATE = "os/env/client-cert.pem" -OS_ENV_CLIENT_KEY = "os/env/client-key.pem" -OS_ENV_HEADERS = "envHeader1=val1,envHeader2=val2,User-agent=Overridden" -OS_ENV_TIMEOUT = "30" -BASIC_SPAN = _Span( - "abc", - context=Mock( - **{ - "trace_state": {"a": "b", "c": "d"}, - "span_id": 10217189687419569865, - "trace_id": 67545097771067222548457157018666467027, - } - ), -) +from . import _mock_clock + +_TEST_ENDPOINT = "http://localhost:4318/v1/traces" +_LOGGER_NAME = "opentelemetry.exporter.otlp.proto.http.trace_exporter" + + +def _decode_body(body: bytes) -> ExportTraceServiceRequest: + return ExportTraceServiceRequest.FromString(body) -# pylint: disable=protected-access class TestOTLPSpanExporter(unittest.TestCase): def setUp(self): + env_patcher = patch.dict(os.environ, {}, clear=True) + env_patcher.start() + self.addCleanup(env_patcher.stop) + + self._in_memory = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(self._in_memory)) + self._tracer = provider.get_tracer(__name__) + self.metric_reader = InMemoryMetricReader() self.meter_provider = MeterProvider( metric_readers=[self.metric_reader] ) - def test_constructor_default(self): - exporter = OTLPSpanExporter() + def _finished_spans(self): + return list(self._in_memory.get_finished_spans()) + def _make_span(self, name: str = "test-span"): + with self._tracer.start_as_current_span(name): + pass + return self._finished_spans() + + @staticmethod + def _mocked_shutdown_event() -> Mock: + shutdown_event = Mock(spec=threading.Event) + shutdown_event.is_set.return_value = False + return shutdown_event + + def assert_standard_metric_attrs(self, attributes): self.assertEqual( - exporter._endpoint, DEFAULT_ENDPOINT + DEFAULT_TRACES_EXPORT_PATH - ) - self.assertEqual(exporter._certificate_file, True) - self.assertEqual(exporter._client_certificate_file, None) - self.assertEqual(exporter._client_key_file, None) - self.assertEqual(exporter._timeout, DEFAULT_TIMEOUT) - self.assertIs(exporter._compression, DEFAULT_COMPRESSION) - self.assertEqual(exporter._headers, {}) - self.assertIsInstance(exporter._session, requests.Session) - self.assertIn("User-Agent", exporter._session.headers) - self.assertEqual( - exporter._session.headers.get("Content-Type"), - "application/x-protobuf", + attributes["otel.component.type"], "otlp_http_span_exporter" ) - self.assertEqual( - exporter._session.headers.get("User-Agent"), - "OTel-OTLP-Exporter-Python/" + __version__, + self.assertTrue( + attributes["otel.component.name"].startswith( + "otlp_http_span_exporter/" + ) ) + self.assertEqual(attributes["server.address"], "localhost") + self.assertEqual(attributes["server.port"], 4318) - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_CERTIFICATE: OS_ENV_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: OS_ENV_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY: OS_ENV_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION: Compression.Gzip.value, - OTEL_EXPORTER_OTLP_ENDPOINT: OS_ENV_ENDPOINT, - OTEL_EXPORTER_OTLP_HEADERS: OS_ENV_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT: OS_ENV_TIMEOUT, - OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE: "traces/certificate.env", - OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE: "traces/client-cert.pem", - OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY: "traces/client-key.pem", - OTEL_EXPORTER_OTLP_TRACES_COMPRESSION: Compression.Deflate.value, - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "https://traces.endpoint.env", - OTEL_EXPORTER_OTLP_TRACES_HEADERS: "tracesEnv1=val1,tracesEnv2=val2,traceEnv3===val3==,User-agent=TraceUserAgent", - OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "40", - _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER: "credential_provider", - }, - ) - @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") - def test_exporter_traces_env_take_priority(self, mock_entry_point): - credential = Session() - - def f(): - return credential + # -- construction / transport selection -------------------------------- - mock_entry_point.configure_mock( - return_value=[IterEntryPoint("custom_credential", f)] - ) + def test_constructor_default_uses_urllib3_transport(self): exporter = OTLPSpanExporter() - self.assertEqual(exporter._endpoint, "https://traces.endpoint.env") - self.assertEqual(exporter._certificate_file, "traces/certificate.env") - self.assertEqual( - exporter._client_certificate_file, "traces/client-cert.pem" - ) - self.assertEqual(exporter._client_key_file, "traces/client-key.pem") - self.assertEqual(exporter._timeout, 40) - self.assertIs(exporter._compression, Compression.Deflate) - self.assertEqual( - exporter._headers, - { - "tracesenv1": "val1", - "tracesenv2": "val2", - "traceenv3": "==val3==", - "user-agent": "TraceUserAgent", - }, - ) - self.assertIs(exporter._session, credential) - self.assertIsInstance(exporter._session, requests.Session) self.assertEqual( - exporter._session.headers.get("Content-Type"), - "application/x-protobuf", + exporter._endpoint, DEFAULT_ENDPOINT + DEFAULT_TRACES_EXPORT_PATH ) - self.assertEqual( - exporter._session.headers.get("User-Agent"), - "TraceUserAgent", + self.assertIs(exporter._compression, CommonCompression.NONE) + self.assertIsNone(exporter._session) + self.assertIsInstance( + exporter._client._transport, Urllib3HTTPTransport ) - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_CERTIFICATE: OS_ENV_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: OS_ENV_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY: OS_ENV_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION: Compression.Gzip.value, - OTEL_EXPORTER_OTLP_ENDPOINT: OS_ENV_ENDPOINT, - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "https://traces.endpoint.env", - OTEL_EXPORTER_OTLP_HEADERS: OS_ENV_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT: OS_ENV_TIMEOUT, - }, - ) - def test_exporter_constructor_take_priority(self): - exporter = OTLPSpanExporter( - endpoint="example.com/1234", - certificate_file="path/to/service.crt", - client_key_file="path/to/client-key.pem", - client_certificate_file="path/to/client-cert.pem", - headers={"testHeader1": "value1", "testHeader2": "value2"}, - timeout=20, - compression=Compression.NoCompression, - session=requests.Session(), - ) + def test_explicit_session_uses_requests_transport(self): + session = requests.Session() + exporter = OTLPSpanExporter(session=session) - self.assertEqual(exporter._endpoint, "example.com/1234") - self.assertEqual(exporter._certificate_file, "path/to/service.crt") - self.assertEqual( - exporter._client_certificate_file, "path/to/client-cert.pem" + self.assertIs(exporter._session, session) + self.assertIsInstance( + exporter._client._transport, RequestsHTTPTransport ) - self.assertEqual(exporter._client_key_file, "path/to/client-key.pem") - self.assertEqual(exporter._timeout, 20) - self.assertIs(exporter._compression, Compression.NoCompression) - self.assertEqual( - exporter._headers, - {"testHeader1": "value1", "testHeader2": "value2"}, - ) - self.assertIsInstance(exporter._session, requests.Session) - - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_CERTIFICATE: OS_ENV_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: OS_ENV_CLIENT_CERTIFICATE, - OTEL_EXPORTER_OTLP_CLIENT_KEY: OS_ENV_CLIENT_KEY, - OTEL_EXPORTER_OTLP_COMPRESSION: Compression.Gzip.value, - OTEL_EXPORTER_OTLP_HEADERS: OS_ENV_HEADERS, - OTEL_EXPORTER_OTLP_TIMEOUT: OS_ENV_TIMEOUT, - }, - ) - def test_exporter_env(self): - exporter = OTLPSpanExporter() + self.assertIs(exporter._client._transport._session, session) - self.assertEqual(exporter._certificate_file, OS_ENV_CERTIFICATE) - self.assertEqual( - exporter._client_certificate_file, OS_ENV_CLIENT_CERTIFICATE + @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") + def test_credential_provider_uses_requests_transport( + self, mock_entry_point + ): + credential = requests.Session() + mock_entry_point.configure_mock( + return_value=[ + IterEntryPoint("custom_credential", lambda: credential) + ] ) - self.assertEqual(exporter._client_key_file, OS_ENV_CLIENT_KEY) - self.assertEqual(exporter._timeout, int(OS_ENV_TIMEOUT)) - self.assertIs(exporter._compression, Compression.Gzip) - self.assertEqual( - exporter._headers, + with patch.dict( + os.environ, { - "envheader1": "val1", - "envheader2": "val2", - "user-agent": "Overridden", + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER: "custom_credential", }, + ): + exporter = OTLPSpanExporter() + + self.assertIs(exporter._session, credential) + self.assertIsInstance( + exporter._client._transport, RequestsHTTPTransport ) + self.assertIs(exporter._client._transport._session, credential) - @patch.dict( - "os.environ", - {OTEL_EXPORTER_OTLP_ENDPOINT: OS_ENV_ENDPOINT}, - ) - def test_exporter_env_endpoint_without_slash(self): - exporter = OTLPSpanExporter() + @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") + def test_exception_raised_when_entrypoint_returns_wrong_type( + self, mock_entry_point + ): + mock_entry_point.configure_mock( + return_value=[IterEntryPoint("bad_credential", lambda: 1)] + ) + with ( + patch.dict( + os.environ, + { + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER: "bad_credential", + }, + ), + self.assertRaises(RuntimeError), + ): + OTLPSpanExporter() - self.assertEqual( - exporter._endpoint, - OS_ENV_ENDPOINT + f"/{DEFAULT_TRACES_EXPORT_PATH}", + @patch("opentelemetry.exporter.otlp.proto.http._common.entry_points") + def test_exception_raised_when_entrypoint_does_not_exist( + self, mock_entry_point + ): + mock_entry_point.configure_mock(return_value=[]) + with ( + patch.dict( + os.environ, + { + _OTEL_PYTHON_EXPORTER_OTLP_HTTP_TRACES_CREDENTIAL_PROVIDER: "missing", + }, + ), + self.assertRaises(RuntimeError), + ): + OTLPSpanExporter() + + def test_compression_dual_enum_acceptance(self): + for compression in (Compression.Gzip, CommonCompression.GZIP): + with self.subTest(compression=compression): + exporter = OTLPSpanExporter(compression=compression) + self.assertIs(exporter._compression, CommonCompression.GZIP) + + # -- export / wire format ------------------------------------------------ + + @mocketize + def test_export_single_span(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT) + spans = self._make_span("my-span") + transport = exporter._client._transport + + with patch.object( + transport, "request", wraps=transport.request + ) as mock_request: + result = exporter.export(spans) + + self.assertEqual(result, SpanExportResult.SUCCESS) + request = Mocket.last_request() + self.assertEqual(request.method, "POST") + self.assertEqual(request.path, "/v1/traces") + sent_data = mock_request.call_args.kwargs["data"] + self.assertEqual(_decode_body(sent_data), encode_spans(spans)) + + @mocketize + def test_export_spans_different_resources(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT) + transport = exporter._client._transport + spans = [] + for name, host in (("from-a", "a"), ("from-b", "b")): + in_memory = InMemorySpanExporter() + provider = TracerProvider(resource=Resource({"host": host})) + provider.add_span_processor(SimpleSpanProcessor(in_memory)) + tracer = provider.get_tracer(__name__) + with tracer.start_as_current_span(name): + pass + spans.extend(in_memory.get_finished_spans()) + + with patch.object( + transport, "request", wraps=transport.request + ) as mock_request: + result = exporter.export(spans) + + self.assertEqual(result, SpanExportResult.SUCCESS) + sent_data = mock_request.call_args.kwargs["data"] + body = _decode_body(sent_data) + self.assertEqual(body, encode_spans(spans)) + self.assertEqual(len(body.resource_spans), 2) + + @mocketize + def test_default_endpoint_and_headers(self): + Entry.single_register( + Entry.POST, "http://localhost:4318/v1/traces", status=200 ) - - @patch.dict( - "os.environ", - {OTEL_EXPORTER_OTLP_ENDPOINT: OS_ENV_ENDPOINT + "/"}, - ) - def test_exporter_env_endpoint_with_slash(self): exporter = OTLPSpanExporter() - self.assertEqual( - exporter._endpoint, - OS_ENV_ENDPOINT + f"/{DEFAULT_TRACES_EXPORT_PATH}", - ) + result = exporter.export(self._make_span()) - @patch.dict( - "os.environ", - { - OTEL_EXPORTER_OTLP_HEADERS: "envHeader1=val1,envHeader2=val2,missingValue" - }, - ) - def test_headers_parse_from_env(self): - with self.assertLogs(level="WARNING") as cm: - _ = OTLPSpanExporter() - - self.assertEqual( - cm.records[0].message, - ( - "Header format invalid! Header values in environment " - "variables must be URL encoded per the OpenTelemetry " - "Protocol Exporter specification or a comma separated " - "list of name=value occurrences: missingValue" - ), + self.assertEqual(result, SpanExportResult.SUCCESS) + headers = Mocket.last_request().headers + self.assertEqual(headers["content-type"], "application/x-protobuf") + self.assertTrue( + headers["user-agent"].startswith("OTel-OTLP-Exporter-Python/") + ) + + def test_custom_endpoint(self): + url = "http://custom.example:9999/v1/traces" + cases = ( + ("constructor", {}, {"endpoint": url}), + ( + "generic_env", + {OTEL_EXPORTER_OTLP_ENDPOINT: "http://custom.example:9999"}, + {}, + ), + ("per_signal_env", {OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: url}, {}), + ) + for label, env, kwargs in cases: + with ( + self.subTest(label), + patch.dict(os.environ, env, clear=True), + Mocketizer(), + ): + Entry.single_register(Entry.POST, url, status=200) + exporter = OTLPSpanExporter(**kwargs) + self._in_memory.clear() + + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.SUCCESS) + + def test_custom_headers(self): + cases = ( + ("constructor", {}, {"headers": {"x-api-key": "secret"}}), + ( + "generic_env", + {OTEL_EXPORTER_OTLP_HEADERS: "x-api-key=secret"}, + {}, + ), + ( + "per_signal_env", + {OTEL_EXPORTER_OTLP_TRACES_HEADERS: "x-api-key=secret"}, + {}, + ), + ) + for label, env, kwargs in cases: + with ( + self.subTest(label), + patch.dict(os.environ, env, clear=True), + Mocketizer(), + ): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT, **kwargs) + self._in_memory.clear() + + exporter.export(self._make_span()) + + headers = Mocket.last_request().headers + self.assertEqual(headers["x-api-key"], "secret") + + @mocketize + def test_custom_transport(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + custom_transport = Urllib3HTTPTransport() + + with patch( + "opentelemetry.exporter.otlp.proto.http.trace_exporter._build_transport" + ) as mock_build_transport: + exporter = OTLPSpanExporter( + endpoint=_TEST_ENDPOINT, _transport=custom_transport ) - @patch.object(OTLPSpanExporter, "_export", return_value=Mock(ok=True)) - def test_2xx_status_code(self, mock_otlp_metric_exporter): - """ - Test that any HTTP 2XX code returns a successful result - """ + mock_build_transport.assert_not_called() + self.assertIs(exporter._client._transport, custom_transport) + + result = exporter.export(self._make_span()) + self.assertEqual(result, SpanExportResult.SUCCESS) + + @mocketize + def test_certificate_args(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + + with patch( + "opentelemetry.exporter.otlp.proto.http.trace_exporter._build_transport", + wraps=_build_transport, + ) as mock_build_transport: + exporter = OTLPSpanExporter( + endpoint=_TEST_ENDPOINT, + certificate_file="ca.pem", + client_key_file="client-key.pem", + client_certificate_file="client-cert.pem", + ) - self.assertEqual( - OTLPSpanExporter().export(MagicMock()), SpanExportResult.SUCCESS + mock_build_transport.assert_called_once_with( + "ca.pem", + "client-key.pem", + "client-cert.pem", + OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY, + OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE, + session=None, + ) + + result = exporter.export(self._make_span()) + self.assertEqual(result, SpanExportResult.SUCCESS) + + def test_compression_options(self): + cases = ( + (Compression.NoCompression, None, lambda data: data), + (Compression.Gzip, "gzip", gzip.decompress), + (Compression.Deflate, "deflate", zlib.decompress), + ) + for compression, expected_encoding, decompress in cases: + with self.subTest(compression=compression), Mocketizer(): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPSpanExporter( + endpoint=_TEST_ENDPOINT, compression=compression + ) + transport = exporter._client._transport + self._in_memory.clear() + spans = self._make_span() + + with patch.object( + transport, "request", wraps=transport.request + ) as mock_request: + result = exporter.export(spans) + + self.assertEqual(result, SpanExportResult.SUCCESS) + sent_headers = mock_request.call_args.kwargs["headers"] + if expected_encoding is None: + self.assertNotIn("Content-Encoding", sent_headers) + else: + self.assertEqual( + sent_headers["Content-Encoding"], expected_encoding + ) + sent_data = mock_request.call_args.kwargs["data"] + decompressed = decompress(sent_data) + self.assertEqual( + _decode_body(decompressed), encode_spans(spans) + ) + + # -- retry / backoff ------------------------------------------------------ + + def test_export_retryable_status_codes(self): + for status_code in (429, 502, 503, 504): + with self.subTest(status_code=status_code), Mocketizer(): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=status_code), + Response(status=200), + ) + exporter = OTLPSpanExporter( + endpoint=_TEST_ENDPOINT, timeout=30.0 + ) + shutdown_event = self._mocked_shutdown_event() + exporter._client._shutdown_event = shutdown_event + self._in_memory.clear() + + with _mock_clock(shutdown_event): + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.SUCCESS) + self.assertEqual(len(Mocket.request_list()), 2) + + def test_export_non_retryable_status_codes(self): + for status_code in (400, 401, 403, 404, 408, 500, 501): + with self.subTest(status_code=status_code), Mocketizer(): + Entry.single_register( + Entry.POST, _TEST_ENDPOINT, status=status_code + ) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT) + self._in_memory.clear() + + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.FAILURE) + self.assertEqual(len(Mocket.request_list()), 1) + + @mocketize + def test_export_max_retries(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + *[Response(status=503)] * 6, + ) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT, timeout=1000.0) + shutdown_event = self._mocked_shutdown_event() + exporter._client._shutdown_event = shutdown_event + + with _mock_clock(shutdown_event): + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.FAILURE) + self.assertEqual(len(Mocket.request_list()), 6) + + @mocketize + def test_export_retry_after_header(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=429, headers={"Retry-After": "5"}), + Response(status=200), + ) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT, timeout=60.0) + shutdown_event = self._mocked_shutdown_event() + shutdown_event.wait.return_value = False + exporter._client._shutdown_event = shutdown_event + + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.SUCCESS) + self.assertEqual(len(Mocket.request_list()), 2) + shutdown_event.wait.assert_called_once_with(5.0) + + @mocketize + def test_export_retry_after_header_http_date(self): + base = 1_700_000_000.0 + retry_at = format_datetime( + datetime.fromtimestamp(base + 30, timezone.utc), usegmt=True + ) + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=503, headers={"Retry-After": retry_at}), + Response(status=200), + ) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT, timeout=120.0) + shutdown_event = self._mocked_shutdown_event() + shutdown_event.wait.return_value = False + exporter._client._shutdown_event = shutdown_event + + with patch( + "opentelemetry.exporter.otlp.common._http.time.time", + return_value=base, + ): + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.SUCCESS) + self.assertEqual(len(Mocket.request_list()), 2) + shutdown_event.wait.assert_called_once_with(30.0) + + @mocketize + def test_export_connection_error(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + urllib3.exceptions.ProtocolError("simulated reset"), + Response(status=200), + ) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT, timeout=5.0) + + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.SUCCESS) + self.assertEqual(len(Mocket.request_list()), 2) + + @mocketize + def test_shutdown_interrupts_retry_backoff(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + *[Response(status=503)] * 6, + ) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT, timeout=1.5) + thread = threading.Thread( + target=exporter.export, args=(self._make_span(),) + ) + before = time.time() + thread.start() + time.sleep(0.05) + exporter.shutdown() + thread.join() + after = time.time() + + self.assertLess(after - before, 0.5) + + # -- self-observability metrics ------------------------------------------- + + @mocketize + def test_exporter_metrics_disabled_by_default(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPSpanExporter( + endpoint=_TEST_ENDPOINT, meter_provider=self.meter_provider ) - @patch.dict("os.environ", {}, clear=True) - @patch.object(OTLPSpanExporter, "_export", return_value=Mock(ok=True)) - def test_exporter_metrics_disabled_by_default(self, _mock_export): - exporter = OTLPSpanExporter(meter_provider=self.meter_provider) - self.assertEqual( - exporter.export([BASIC_SPAN]), SpanExportResult.SUCCESS + exporter.export(self._make_span()), SpanExportResult.SUCCESS ) - self.assertIsNone(self.metric_reader.get_metrics_data()) @patch.dict( "os.environ", {OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: " true "} ) - @patch.object(Session, "post") - def test_retry_timeout(self, mock_post): - exporter = OTLPSpanExporter( - timeout=1.5, meter_provider=self.meter_provider - ) - - resp = Response() - resp.status_code = 503 - resp.reason = "UNAVAILABLE" - mock_post.return_value = resp - with self.assertLogs(level=WARNING) as warning: - before = time.time() - # Set timeout to 1.5 seconds - self.assertEqual( - exporter.export([BASIC_SPAN]), - SpanExportResult.FAILURE, + def test_retry_timeout_records_metrics(self): + with Mocketizer(): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + Response(status=503), + Response(status=503), ) - after = time.time() - # First call at time 0, second at time 1, then an early return before the second backoff sleep b/c it would exceed timeout. - self.assertEqual(mock_post.call_count, 2) - # There's a +/-20% jitter on each backoff. - self.assertTrue(0.75 < after - before < 1.25) - self.assertIn( - "Transient error UNAVAILABLE encountered while exporting span batch, retrying in", - warning.records[0].message, + exporter = OTLPSpanExporter( + endpoint=_TEST_ENDPOINT, + timeout=1.5, + meter_provider=self.meter_provider, ) + shutdown_event = self._mocked_shutdown_event() + exporter._client._shutdown_event = shutdown_event + + with _mock_clock(shutdown_event): + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.FAILURE) metrics_data = self.metric_reader.get_metrics_data() scope_metrics = metrics_data.resource_metrics[0].scope_metrics[0] - self.assertEqual(scope_metrics.scope.name, "opentelemetry-sdk") metrics = sorted(scope_metrics.metrics, key=lambda m: m.name) self.assertEqual(len(metrics), 3) self.assertEqual( @@ -329,9 +586,6 @@ def test_retry_timeout(self, mock_post): self.assert_standard_metric_attrs( metrics[0].data.data_points[0].attributes ) - self.assertNotIn( - "error.type", metrics[0].data.data_points[0].attributes - ) self.assertEqual( metrics[0] .data.data_points[0] @@ -339,161 +593,92 @@ def test_retry_timeout(self, mock_post): 503, ) self.assertEqual(metrics[1].name, "otel.sdk.exporter.span.exported") - self.assert_standard_metric_attrs( - metrics[1].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[1].data.data_points[0].attributes - ) - self.assertNotIn( - "http.response.status_code", - metrics[1].data.data_points[0].attributes, - ) self.assertEqual(metrics[2].name, "otel.sdk.exporter.span.inflight") - self.assert_standard_metric_attrs( - metrics[2].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[2].data.data_points[0].attributes - ) - self.assertNotIn( - "http.response.status_code", - metrics[2].data.data_points[0].attributes, - ) - - @patch.object(Session, "post") - def test_export_no_collector_available_retryable(self, mock_post): - exporter = OTLPSpanExporter(timeout=1.5) - msg = "Server not available." - mock_post.side_effect = ConnectionError(msg) - with self.assertLogs(level=WARNING) as warning: - self.assertEqual( - exporter.export([BASIC_SPAN]), - SpanExportResult.FAILURE, - ) - # Check for greater 2 because the request is on each retry - # done twice at the moment. - self.assertGreater(mock_post.call_count, 2) - self.assertIn( - f"Transient error {msg} encountered while exporting span batch, retrying in", - warning.records[0].message, - ) @patch.dict( "os.environ", {OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED: "true"} ) - @patch.object(Session, "post") - def test_export_no_collector_available(self, mock_post): + @mocketize + def test_export_connection_error_records_metrics(self): + Entry.register( + Entry.POST, + _TEST_ENDPOINT, + *([urllib3.exceptions.ProtocolError("boom")] * 6), + ) exporter = OTLPSpanExporter( - timeout=1.5, meter_provider=self.meter_provider + endpoint=_TEST_ENDPOINT, + timeout=1000.0, + meter_provider=self.meter_provider, ) + shutdown_event = self._mocked_shutdown_event() + exporter._client._shutdown_event = shutdown_event - mock_post.side_effect = requests.exceptions.RequestException() - with self.assertLogs(level=WARNING) as warning: - self.assertEqual( - exporter.export([BASIC_SPAN]), - SpanExportResult.FAILURE, - ) - self.assertEqual(mock_post.call_count, 1) - self.assertIn( - "Failed to export span batch code", - warning.records[0].message, - ) + with _mock_clock(shutdown_event): + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.FAILURE) metrics_data = self.metric_reader.get_metrics_data() scope_metrics = metrics_data.resource_metrics[0].scope_metrics[0] - self.assertEqual(scope_metrics.scope.name, "opentelemetry-sdk") metrics = sorted(scope_metrics.metrics, key=lambda m: m.name) - self.assertEqual(len(metrics), 3) self.assertEqual( metrics[0].name, "otel.sdk.exporter.operation.duration" ) - self.assert_standard_metric_attrs( - metrics[0].data.data_points[0].attributes - ) - self.assertEqual( - metrics[0].data.data_points[0].attributes["error.type"], - "RequestException", - ) + self.assertIn("error.type", metrics[0].data.data_points[0].attributes) self.assertNotIn( "http.response.status_code", metrics[0].data.data_points[0].attributes, ) - self.assertEqual(metrics[1].name, "otel.sdk.exporter.span.exported") - self.assert_standard_metric_attrs( - metrics[1].data.data_points[0].attributes - ) - self.assertEqual( - metrics[1].data.data_points[0].attributes["error.type"], - "RequestException", - ) - self.assertNotIn( - "http.response.status_code", - metrics[1].data.data_points[0].attributes, - ) - self.assertEqual(metrics[2].name, "otel.sdk.exporter.span.inflight") - self.assert_standard_metric_attrs( - metrics[2].data.data_points[0].attributes - ) - self.assertNotIn( - "error.type", metrics[2].data.data_points[0].attributes - ) - self.assertNotIn( - "http.response.status_code", - metrics[2].data.data_points[0].attributes, - ) - @patch.object(Session, "post") - def test_timeout_set_correctly(self, mock_post): - resp = Response() - resp.status_code = 200 - - def export_side_effect(*args, **kwargs): - # Timeout should be set to something slightly less than 400 milliseconds depending on how much time has passed. - self.assertAlmostEqual(0.4, kwargs["timeout"], 2) - return resp - - mock_post.side_effect = export_side_effect - exporter = OTLPSpanExporter(timeout=0.4) - exporter.export([BASIC_SPAN]) - - @patch.object(Session, "post") - def test_shutdown_interrupts_retry_backoff(self, mock_post): - exporter = OTLPSpanExporter(timeout=1.5) - - resp = Response() - resp.status_code = 503 - resp.reason = "UNAVAILABLE" - mock_post.return_value = resp - thread = threading.Thread(target=exporter.export, args=([BASIC_SPAN],)) - with self.assertLogs(level=WARNING) as warning: - before = time.time() - thread.start() - # Wait for the first attempt to fail, then enter a 1 second backoff. - time.sleep(0.05) - # Should cause export to wake up and return. + # -- misc ----------------------------------------------------------------- + + @mocketize + def test_export_after_shutdown(self): + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT) + exporter.shutdown() + + with self.assertLogs(_LOGGER_NAME, level="WARNING"): + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.FAILURE) + self.assertIsNone(Mocket.last_request()) + + def test_shutdown_idempotent(self): + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT) + exporter.shutdown() + + with self.assertLogs(_LOGGER_NAME, level="WARNING"): exporter.shutdown() - thread.join() - after = time.time() - self.assertIn( - "Transient error UNAVAILABLE encountered while exporting span batch, retrying in", - warning.records[0].message, - ) - self.assertIn( - "Shutdown in progress, aborting retry.", - warning.records[1].message, - ) - assert after - before < 0.2 + def test_shutdown_closes_transport(self): + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT) - def assert_standard_metric_attrs(self, attributes): - self.assertEqual( - attributes["otel.component.type"], "otlp_http_span_exporter" - ) - self.assertTrue( - attributes["otel.component.name"].startswith( - "otlp_http_span_exporter/" - ) - ) - self.assertEqual(attributes["server.address"], "localhost") - self.assertEqual(attributes["server.port"], 4318) + with patch.object(exporter._client._transport, "close") as mock_close: + exporter.shutdown() + + mock_close.assert_called_once() + + @mocketize + def test_force_flush(self): + Entry.single_register(Entry.POST, _TEST_ENDPOINT, status=200) + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT) + + self.assertTrue(exporter.force_flush()) + exporter.export(self._make_span()) + self.assertTrue(exporter.force_flush()) + + @mocketize + def test_export_encoding_failure(self): + exporter = OTLPSpanExporter(endpoint=_TEST_ENDPOINT) + + with ( + patch( + "opentelemetry.exporter.otlp.proto.http.trace_exporter.encode_spans", + side_effect=ValueError("boom"), + ), + self.assertLogs(_LOGGER_NAME, level="ERROR"), + ): + result = exporter.export(self._make_span()) + + self.assertEqual(result, SpanExportResult.FAILURE) + self.assertIsNone(Mocket.last_request()) diff --git a/pyproject.toml b/pyproject.toml index a273dff32da..1e1cc24af24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "opentelemetry-exporter-otlp-proto-grpc", "opentelemetry-exporter-otlp-proto-http", "opentelemetry-exporter-otlp-proto-common", + "opentelemetry-exporter-otlp-common", "opentelemetry-exporter-otlp-json-common", "opentelemetry-exporter-zipkin-json", "opentelemetry-exporter-prometheus", @@ -39,6 +40,7 @@ opentelemetry-exporter-http-transport = { workspace = true } opentelemetry-exporter-otlp-proto-grpc = { workspace = true } opentelemetry-exporter-otlp-proto-http = { workspace = true } opentelemetry-exporter-otlp-proto-common = { workspace = true } +opentelemetry-exporter-otlp-common = { workspace = true } opentelemetry-exporter-otlp-json-common = { workspace = true } opentelemetry-exporter-zipkin-json = { workspace = true } opentelemetry-exporter-prometheus = {workspace = true } diff --git a/tox.ini b/tox.ini index 1acd90d9764..b106c0daa74 100644 --- a/tox.ini +++ b/tox.ini @@ -365,6 +365,8 @@ deps = ; OTLP packages otlpexporter: -e {toxinidir}/opentelemetry-proto otlpexporter: -e {toxinidir}/exporter/opentelemetry-exporter-otlp-proto-common + otlpexporter: -e {toxinidir}/exporter/opentelemetry-exporter-otlp-common + otlpexporter: -e {toxinidir}/exporter/opentelemetry-exporter-http-transport otlpexporter: -e {toxinidir}/exporter/opentelemetry-exporter-otlp-proto-grpc otlpexporter: -e {toxinidir}/exporter/opentelemetry-exporter-otlp-proto-http otlpexporter: -e {toxinidir}/exporter/opentelemetry-exporter-otlp @@ -432,6 +434,7 @@ deps = -e {toxinidir}/opentelemetry-sdk[file-configuration] -e {toxinidir}/tests/opentelemetry-test-utils -e {toxinidir}/exporter/opentelemetry-exporter-http-transport + -e {toxinidir}/exporter/opentelemetry-exporter-otlp-common -e {toxinidir}/exporter/opentelemetry-exporter-otlp-proto-common -e {toxinidir}/exporter/opentelemetry-exporter-otlp-json-common -e {toxinidir}/exporter/opentelemetry-exporter-otlp diff --git a/uv.lock b/uv.lock index 55fc9ceef21..067f7e840d2 100644 --- a/uv.lock +++ b/uv.lock @@ -975,6 +975,8 @@ source = { editable = "exporter/opentelemetry-exporter-otlp-proto-http" } dependencies = [ { name = "googleapis-common-protos" }, { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-http-transport", extra = ["requests", "urllib3"] }, + { name = "opentelemetry-exporter-otlp-common" }, { name = "opentelemetry-exporter-otlp-proto-common" }, { name = "opentelemetry-proto" }, { name = "opentelemetry-sdk" }, @@ -992,6 +994,8 @@ requires-dist = [ { name = "googleapis-common-protos", specifier = "~=1.52" }, { name = "opentelemetry-api", editable = "opentelemetry-api" }, { name = "opentelemetry-exporter-credential-provider-gcp", marker = "extra == 'gcp-auth'", specifier = ">=0.59b0" }, + { name = "opentelemetry-exporter-http-transport", extras = ["requests", "urllib3"], editable = "exporter/opentelemetry-exporter-http-transport" }, + { name = "opentelemetry-exporter-otlp-common", editable = "exporter/opentelemetry-exporter-otlp-common" }, { name = "opentelemetry-exporter-otlp-proto-common", editable = "exporter/opentelemetry-exporter-otlp-proto-common" }, { name = "opentelemetry-proto", editable = "opentelemetry-proto" }, { name = "opentelemetry-sdk", editable = "opentelemetry-sdk" }, @@ -1078,6 +1082,7 @@ dependencies = [ { name = "opentelemetry-api" }, { name = "opentelemetry-codegen-json" }, { name = "opentelemetry-exporter-http-transport" }, + { name = "opentelemetry-exporter-otlp-common" }, { name = "opentelemetry-exporter-otlp-json-common" }, { name = "opentelemetry-exporter-otlp-json-file" }, { name = "opentelemetry-exporter-otlp-proto-common" }, @@ -1108,6 +1113,7 @@ requires-dist = [ { name = "opentelemetry-api", editable = "opentelemetry-api" }, { name = "opentelemetry-codegen-json", editable = "codegen/opentelemetry-codegen-json" }, { name = "opentelemetry-exporter-http-transport", editable = "exporter/opentelemetry-exporter-http-transport" }, + { name = "opentelemetry-exporter-otlp-common", editable = "exporter/opentelemetry-exporter-otlp-common" }, { name = "opentelemetry-exporter-otlp-json-common", editable = "exporter/opentelemetry-exporter-otlp-json-common" }, { name = "opentelemetry-exporter-otlp-json-file", editable = "exporter/opentelemetry-exporter-otlp-json-file" }, { name = "opentelemetry-exporter-otlp-proto-common", editable = "exporter/opentelemetry-exporter-otlp-proto-common" },