From ee1bc367a9f312c655a6b3d5ba104b30000623ec Mon Sep 17 00:00:00 2001 From: Hector Hernandez <39923391+hectorhdzg@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:20:33 -0700 Subject: [PATCH] Handle ServiceResponseError as retryable to prevent telemetry loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ServiceResponseError (e.g. read timeout) was not caught by the exporter's _transmit method. Since ServiceResponseError and ServiceRequestError are sibling classes (both extend AzureError directly), a read timeout fell through to the generic 'except Exception' handler, which marked the batch as FAILED_NOT_RETRYABLE — permanently dropping the telemetry instead of persisting it to local storage for retry. This change adds ServiceResponseError to the import and catch chain, treating it as FAILED_RETRYABLE so that envelopes are written to disk and retried on the next successful export cycle, consistent with the existing ServiceRequestError handling and the azure-core docstring which states these errors 'can be retried for idempotent or safe operations'. --- .../opentelemetry/exporter/export/_base.py | 18 ++++++++++++- .../tests/test_base_customer_sdkstats.py | 12 ++++++++- .../tests/test_base_exporter.py | 26 ++++++++++++++++++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py index f122c5812683..8247f6cf9ad2 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py @@ -12,7 +12,7 @@ from urllib.parse import urlparse import psutil -from azure.core.exceptions import HttpResponseError, ServiceRequestError +from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError from azure.core.pipeline.policies import ( ContentDecodePolicy, HttpLoggingPolicy, @@ -594,6 +594,22 @@ def _transmit(self, envelopes: List[TelemetryItem], _skip_rate_limit: bool = Fal exc_type = request_error.__class__.__name__ # type: ignore _update_requests_map(_REQ_EXCEPTION_NAME[1], value=exc_type) result = ExportResult.FAILED_RETRYABLE + except ServiceResponseError as response_error: + # The request was sent but the client failed to receive a response + # (e.g. read timeout). It is unknown whether the server processed + # the request, so treat as retryable to avoid data loss. + logger.warning("Retrying due to server response error: %s.", response_error.message) + + # Track retry items in customer sdkstats for client-side exceptions + if self._should_collect_customer_sdkstats(): + track_retry_items(envelopes, response_error) + + if self._should_collect_stats(): + exc_type = response_error.exc_type + if exc_type is None or exc_type is type(None): # pylint: disable=unidiomatic-typecheck + exc_type = response_error.__class__.__name__ # type: ignore + _update_requests_map(_REQ_EXCEPTION_NAME[1], value=exc_type) + result = ExportResult.FAILED_RETRYABLE except Exception as ex: logger.exception( # pylint: disable=do-not-log-exceptions-if-not-debug, do-not-use-logging-exception "Envelopes could not be exported and are not retryable: %s.", ex diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py index 47bc5bac3b96..986561992a6b 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py @@ -6,7 +6,7 @@ from unittest import mock from datetime import datetime -from azure.core.exceptions import HttpResponseError, ServiceRequestError +from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError from azure.monitor.opentelemetry.exporter.export._base import ( BaseExporter, ExportResult, @@ -215,6 +215,16 @@ def test_transmit_service_request_error_customer_sdkstats_track_retry_items(self track_retry_mock.assert_called_once() self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + @mock.patch("azure.monitor.opentelemetry.exporter.export._base.track_retry_items") + def test_transmit_service_response_error_customer_sdkstats_track_retry_items(self, track_retry_mock): + """Test that _track_retry_items is called on ServiceResponseError (e.g. read timeout)""" + exporter = self._create_exporter_with_customer_sdkstats_enabled() + with mock.patch.object(AzureMonitorClient, "track", side_effect=ServiceResponseError("Read timed out")): + result = exporter._transmit(self._envelopes_to_export) + + track_retry_mock.assert_called_once() + self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + @mock.patch("azure.monitor.opentelemetry.exporter.export._base.track_dropped_items") def test_transmit_general_exception_customer_sdkstats_track_dropped_items(self, track_dropped_mock): """Test that _track_dropped_items is called on general exceptions""" diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py index 63b558035c24..59483c43166b 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py @@ -7,7 +7,7 @@ from unittest import mock from datetime import datetime, timedelta, timezone -from azure.core.exceptions import HttpResponseError, ServiceRequestError +from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError from azure.core.pipeline.transport import HttpResponse from azure.monitor.opentelemetry.exporter.export._base import ( _get_auth_policy, @@ -861,6 +861,11 @@ def test_transmit_request_error(self): result = self._base._transmit(self._envelopes_to_export) self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + def test_transmit_response_error(self): + with mock.patch.object(AzureMonitorClient, "track", throw(ServiceResponseError, message="Read timed out")): + result = self._base._transmit(self._envelopes_to_export) + self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + def test_transmission_200(self): with mock.patch.object(AzureMonitorClient, "track") as post: post.return_value = TrackResponse( @@ -1177,6 +1182,25 @@ def test_transmit_request_error_statsbeat(self, stats_mock): self.assertEqual(_REQUESTS_MAP["count"], 1) self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + @mock.patch.dict( + os.environ, + { + "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "false", + "APPLICATIONINSIGHTS_SDKSTATS_DISABLED": "false", + }, + ) + @mock.patch("azure.monitor.opentelemetry.exporter.statsbeat._statsbeat.collect_statsbeat_metrics") + def test_transmit_response_error_statsbeat(self, stats_mock): + exporter = BaseExporter(disable_offline_storage=True) + with mock.patch.object(AzureMonitorClient, "track", throw(ServiceResponseError, message="Read timed out")): + result = exporter._transmit(self._envelopes_to_export) + stats_mock.assert_called_once() + self.assertEqual(len(_REQUESTS_MAP), 3) + self.assertEqual(_REQUESTS_MAP[_REQ_EXCEPTION_NAME[1]]["ServiceResponseError"], 1) + self.assertIsNotNone(_REQUESTS_MAP[_REQ_DURATION_NAME[1]]) + self.assertEqual(_REQUESTS_MAP["count"], 1) + self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + def test_transmit_request_exception(self): with mock.patch.object(AzureMonitorClient, "track", throw(Exception)): result = self._base._transmit(self._envelopes_to_export)