diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py index 55a0faa617fe..eaf2bbd30b70 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py @@ -12,7 +12,7 @@ from urllib.parse import urlparse import psutil -from azure.core.exceptions import HttpResponseError, ServiceRequestError +from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError from azure.core.pipeline.policies import ( ContentDecodePolicy, HttpLoggingPolicy, @@ -594,6 +594,22 @@ def _transmit(self, envelopes: List[TelemetryItem], _skip_rate_limit: bool = Fal exc_type = request_error.__class__.__name__ # type: ignore _update_requests_map(_REQ_EXCEPTION_NAME[1], value=exc_type) result = ExportResult.FAILED_RETRYABLE + except ServiceResponseError as response_error: + # The request was sent but the client failed to receive a response + # (e.g. read timeout). It is unknown whether the server processed + # the request, so treat as retryable to avoid data loss. + logger.warning("Retrying due to server response error: %s.", response_error.message) + + # Track retry items in customer sdkstats for client-side exceptions + if self._should_collect_customer_sdkstats(): + track_retry_items(envelopes, response_error) + + if self._should_collect_stats(): + exc_type = response_error.exc_type + if exc_type is None or exc_type is type(None): # pylint: disable=unidiomatic-typecheck + exc_type = response_error.__class__.__name__ # type: ignore + _update_requests_map(_REQ_EXCEPTION_NAME[1], value=exc_type) + result = ExportResult.FAILED_RETRYABLE except Exception as ex: logger.exception( # pylint: disable=do-not-log-exceptions-if-not-debug, do-not-use-logging-exception "Envelopes could not be exported and are not retryable: %s.", ex diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py index 47bc5bac3b96..986561992a6b 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py @@ -6,7 +6,7 @@ from unittest import mock from datetime import datetime -from azure.core.exceptions import HttpResponseError, ServiceRequestError +from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError from azure.monitor.opentelemetry.exporter.export._base import ( BaseExporter, ExportResult, @@ -215,6 +215,16 @@ def test_transmit_service_request_error_customer_sdkstats_track_retry_items(self track_retry_mock.assert_called_once() self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + @mock.patch("azure.monitor.opentelemetry.exporter.export._base.track_retry_items") + def test_transmit_service_response_error_customer_sdkstats_track_retry_items(self, track_retry_mock): + """Test that _track_retry_items is called on ServiceResponseError (e.g. read timeout)""" + exporter = self._create_exporter_with_customer_sdkstats_enabled() + with mock.patch.object(AzureMonitorClient, "track", side_effect=ServiceResponseError("Read timed out")): + result = exporter._transmit(self._envelopes_to_export) + + track_retry_mock.assert_called_once() + self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + @mock.patch("azure.monitor.opentelemetry.exporter.export._base.track_dropped_items") def test_transmit_general_exception_customer_sdkstats_track_dropped_items(self, track_dropped_mock): """Test that _track_dropped_items is called on general exceptions""" diff --git a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py index 68ee212df77d..1f2499e1d4be 100644 --- a/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py +++ b/sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py @@ -7,7 +7,7 @@ from unittest import mock from datetime import datetime, timedelta, timezone -from azure.core.exceptions import HttpResponseError, ServiceRequestError +from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError from azure.core.pipeline.transport import HttpResponse from azure.monitor.opentelemetry.exporter.export._base import ( _get_auth_policy, @@ -861,6 +861,11 @@ def test_transmit_request_error(self): result = self._base._transmit(self._envelopes_to_export) self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + def test_transmit_response_error(self): + with mock.patch.object(AzureMonitorClient, "track", throw(ServiceResponseError, message="Read timed out")): + result = self._base._transmit(self._envelopes_to_export) + self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + def test_transmission_200(self): with mock.patch.object(AzureMonitorClient, "track") as post: post.return_value = TrackResponse( @@ -1180,6 +1185,25 @@ def test_transmit_request_error_statsbeat(self, stats_mock): self.assertEqual(_REQUESTS_MAP["count"], 1) self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + @mock.patch.dict( + os.environ, + { + "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "false", + "APPLICATIONINSIGHTS_SDKSTATS_DISABLED": "false", + }, + ) + @mock.patch("azure.monitor.opentelemetry.exporter.statsbeat._statsbeat.collect_statsbeat_metrics") + def test_transmit_response_error_statsbeat(self, stats_mock): + exporter = BaseExporter(disable_offline_storage=True) + with mock.patch.object(AzureMonitorClient, "track", throw(ServiceResponseError, message="Read timed out")): + result = exporter._transmit(self._envelopes_to_export) + stats_mock.assert_called_once() + self.assertEqual(len(_REQUESTS_MAP), 3) + self.assertEqual(_REQUESTS_MAP[_REQ_EXCEPTION_NAME[1]]["ServiceResponseError"], 1) + self.assertIsNotNone(_REQUESTS_MAP[_REQ_DURATION_NAME[1]]) + self.assertEqual(_REQUESTS_MAP["count"], 1) + self.assertEqual(result, ExportResult.FAILED_RETRYABLE) + def test_transmit_request_exception(self): with mock.patch.object(AzureMonitorClient, "track", throw(Exception)): result = self._base._transmit(self._envelopes_to_export)