[monitor opentelemetry exporter] Handle ServiceResponseError as retryable to prevent telemetry loss

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py

@@ -12,7 +12,7 @@
 from urllib.parse import urlparse
 import psutil
 
-from azure.core.exceptions import HttpResponseError, ServiceRequestError
+from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError
 from azure.core.pipeline.policies import (
     ContentDecodePolicy,
     HttpLoggingPolicy,
@@ -594,6 +594,22 @@ def _transmit(self, envelopes: List[TelemetryItem], _skip_rate_limit: bool = Fal
                         exc_type = request_error.__class__.__name__  # type: ignore
                     _update_requests_map(_REQ_EXCEPTION_NAME[1], value=exc_type)
                 result = ExportResult.FAILED_RETRYABLE
+            except ServiceResponseError as response_error:
+                # The request was sent but the client failed to receive a response
+                # (e.g. read timeout). It is unknown whether the server processed
+                # the request, so treat as retryable to avoid data loss.
+                logger.warning("Retrying due to server response error: %s.", response_error.message)
+
+                # Track retry items in customer sdkstats for client-side exceptions
+                if self._should_collect_customer_sdkstats():
+                    track_retry_items(envelopes, response_error)
+
+                if self._should_collect_stats():
+                    exc_type = response_error.exc_type
+                    if exc_type is None or exc_type is type(None):  # pylint: disable=unidiomatic-typecheck
+                        exc_type = response_error.__class__.__name__  # type: ignore
+                    _update_requests_map(_REQ_EXCEPTION_NAME[1], value=exc_type)
+                result = ExportResult.FAILED_RETRYABLE
             except Exception as ex:
                 logger.exception(  # pylint: disable=do-not-log-exceptions-if-not-debug, do-not-use-logging-exception
                     "Envelopes could not be exported and are not retryable: %s.", ex

sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_customer_sdkstats.py

@@ -6,7 +6,7 @@
 from unittest import mock
 from datetime import datetime
 
-from azure.core.exceptions import HttpResponseError, ServiceRequestError
+from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError
 from azure.monitor.opentelemetry.exporter.export._base import (
     BaseExporter,
     ExportResult,
@@ -215,6 +215,16 @@ def test_transmit_service_request_error_customer_sdkstats_track_retry_items(self
         track_retry_mock.assert_called_once()
         self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
 
+    @mock.patch("azure.monitor.opentelemetry.exporter.export._base.track_retry_items")
+    def test_transmit_service_response_error_customer_sdkstats_track_retry_items(self, track_retry_mock):
+        """Test that _track_retry_items is called on ServiceResponseError (e.g. read timeout)"""
+        exporter = self._create_exporter_with_customer_sdkstats_enabled()
+        with mock.patch.object(AzureMonitorClient, "track", side_effect=ServiceResponseError("Read timed out")):
+            result = exporter._transmit(self._envelopes_to_export)
+
+        track_retry_mock.assert_called_once()
+        self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
+
     @mock.patch("azure.monitor.opentelemetry.exporter.export._base.track_dropped_items")
     def test_transmit_general_exception_customer_sdkstats_track_dropped_items(self, track_dropped_mock):
         """Test that _track_dropped_items is called on general exceptions"""

sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py

@@ -7,7 +7,7 @@
 from unittest import mock
 from datetime import datetime, timedelta, timezone
 
-from azure.core.exceptions import HttpResponseError, ServiceRequestError
+from azure.core.exceptions import HttpResponseError, ServiceRequestError, ServiceResponseError
 from azure.core.pipeline.transport import HttpResponse
 from azure.monitor.opentelemetry.exporter.export._base import (
     _get_auth_policy,
@@ -861,6 +861,11 @@ def test_transmit_request_error(self):
             result = self._base._transmit(self._envelopes_to_export)
         self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
 
+    def test_transmit_response_error(self):
+        with mock.patch.object(AzureMonitorClient, "track", throw(ServiceResponseError, message="Read timed out")):
+            result = self._base._transmit(self._envelopes_to_export)
+        self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
+
     def test_transmission_200(self):
         with mock.patch.object(AzureMonitorClient, "track") as post:
             post.return_value = TrackResponse(
@@ -1177,6 +1182,25 @@ def test_transmit_request_error_statsbeat(self, stats_mock):
         self.assertEqual(_REQUESTS_MAP["count"], 1)
         self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
 
+    @mock.patch.dict(
+        os.environ,
+        {
+            "APPLICATIONINSIGHTS_STATSBEAT_DISABLED_ALL": "false",
+            "APPLICATIONINSIGHTS_SDKSTATS_DISABLED": "false",
+        },
+    )
+    @mock.patch("azure.monitor.opentelemetry.exporter.statsbeat._statsbeat.collect_statsbeat_metrics")
+    def test_transmit_response_error_statsbeat(self, stats_mock):
+        exporter = BaseExporter(disable_offline_storage=True)
+        with mock.patch.object(AzureMonitorClient, "track", throw(ServiceResponseError, message="Read timed out")):
+            result = exporter._transmit(self._envelopes_to_export)
+        stats_mock.assert_called_once()
+        self.assertEqual(len(_REQUESTS_MAP), 3)
+        self.assertEqual(_REQUESTS_MAP[_REQ_EXCEPTION_NAME[1]]["ServiceResponseError"], 1)
+        self.assertIsNotNone(_REQUESTS_MAP[_REQ_DURATION_NAME[1]])
+        self.assertEqual(_REQUESTS_MAP["count"], 1)
+        self.assertEqual(result, ExportResult.FAILED_RETRYABLE)
+
     def test_transmit_request_exception(self):
         with mock.patch.object(AzureMonitorClient, "track", throw(Exception)):
             result = self._base._transmit(self._envelopes_to_export)