diff --git a/.gitignore b/.gitignore index c5b6d2e..5765e8f 100644 --- a/.gitignore +++ b/.gitignore @@ -402,4 +402,7 @@ ASALocalRun/ .mfractor/ # Local History for Visual Studio -.localhistory/ \ No newline at end of file +.localhistory/ + +# e2e-cli build output +e2e-cli/build/ \ No newline at end of file diff --git a/Analytics-CSharp/Segment/Analytics/Configuration.cs b/Analytics-CSharp/Segment/Analytics/Configuration.cs index 79de436..59d489c 100644 --- a/Analytics-CSharp/Segment/Analytics/Configuration.cs +++ b/Analytics-CSharp/Segment/Analytics/Configuration.cs @@ -47,6 +47,12 @@ private set public IEventPipelineProvider EventPipelineProvider { get; } + public int MaxRetries { get; } + + public TimeSpan MaxTotalBackoffDuration { get; } + + public TimeSpan MaxRateLimitDuration { get; } + /// /// Configuration that analytics can use /// @@ -73,6 +79,9 @@ private set /// defaults to DefaultHTTPClientProvider /// /// set custom flush policies to tell analytics when and how to flush. If a value is given, it overwrites flushAt and flushInterval + /// maximum number of backoff retries per batch upload, defaults to 10 + /// wall-clock cap on total backoff time, defaults to 12 hours + /// wall-clock cap on 429 Retry-After retries, defaults to 12 hours public Configuration(string writeKey, int flushAt = 20, int flushInterval = 30, @@ -85,7 +94,10 @@ public Configuration(string writeKey, IStorageProvider storageProvider = default, IHTTPClientProvider httpClientProvider = default, IList flushPolicies = default, - IEventPipelineProvider eventPipelineProvider = default) + IEventPipelineProvider eventPipelineProvider = default, + int maxRetries = 10, + TimeSpan? maxTotalBackoffDuration = null, + TimeSpan? maxRateLimitDuration = null) { WriteKey = writeKey; FlushAt = flushAt; @@ -102,6 +114,9 @@ public Configuration(string writeKey, FlushPolicies.Add(new CountFlushPolicy(flushAt)); FlushPolicies.Add(new FrequencyFlushPolicy(flushInterval * 1000L)); EventPipelineProvider = eventPipelineProvider ?? new EventPipelineProvider(); + MaxRetries = maxRetries; + MaxTotalBackoffDuration = maxTotalBackoffDuration ?? TimeSpan.FromHours(12); + MaxRateLimitDuration = maxRateLimitDuration ?? TimeSpan.FromHours(12); } public Configuration(string writeKey, diff --git a/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs b/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs index 9586be0..6e985b9 100644 --- a/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs +++ b/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs @@ -1,3 +1,6 @@ +using System; +using System.Collections.Generic; +using System.Globalization; using Segment.Analytics.Utilities; using Segment.Serialization; using Segment.Sovran; @@ -77,11 +80,72 @@ public override void Update(Settings settings, UpdateType type) base.Update(settings, type); JsonObject segmentInfo = settings.Integrations?.GetJsonObject(Key); + string apiHost = segmentInfo?.GetString(ApiHost); if (apiHost != null && _pipeline != null) - { _pipeline.ApiHost = apiHost; + + JsonObject httpConfig = segmentInfo?.GetJsonObject("httpConfig"); + EventPipeline concretePipeline = _pipeline as EventPipeline; + if (httpConfig != null && concretePipeline?._httpClient != null) + ApplyHttpConfig(concretePipeline._httpClient, httpConfig); + } + + private static void ApplyHttpConfig(HTTPClient client, JsonObject httpConfig) + { + JsonObject backoff = httpConfig.GetJsonObject("backoffConfig"); + if (backoff != null) + { + string enabledStr = backoff.GetString("enabled"); + if (enabledStr != null && bool.TryParse(enabledStr, out bool enabled)) + client.BackoffEnabled = enabled; + + string maxRetriesStr = backoff.GetString("maxRetryCount"); + if (maxRetriesStr != null && int.TryParse(maxRetriesStr, out int maxRetries)) + client.MaxRetries = maxRetries; + + string baseStr = backoff.GetString("baseBackoffInterval"); + if (baseStr != null && double.TryParse(baseStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double baseMs)) + client.BaseBackoffMs = baseMs; + + string capStr = backoff.GetString("maxBackoffInterval"); + if (capStr != null && double.TryParse(capStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double capMs)) + client.MaxBackoffMs = capMs; + + JsonObject overridesJson = backoff.GetJsonObject("statusCodeOverrides"); + if (overridesJson != null) + client.StatusCodeOverrides = ParseStatusCodeOverrides(overridesJson); + } + + JsonObject rateLimit = httpConfig.GetJsonObject("rateLimitConfig"); + if (rateLimit != null) + { + string enabledStr = rateLimit.GetString("enabled"); + if (enabledStr != null && bool.TryParse(enabledStr, out bool enabled)) + client.RateLimitEnabled = enabled; + + string maxRetriesStr = rateLimit.GetString("maxRetryCount"); + if (maxRetriesStr != null && int.TryParse(maxRetriesStr, out int maxRetries)) + client.MaxRateLimitRetries = maxRetries; + + string capStr = rateLimit.GetString("maxRetryInterval"); + if (capStr != null && int.TryParse(capStr, out int capSec)) + client.MaxRetryAfterCapSeconds = capSec; + } + } + + private static Dictionary ParseStatusCodeOverrides(JsonObject overridesJson) + { + var result = new Dictionary(); + foreach (string key in overridesJson.Keys) + { + if (!int.TryParse(key, out int code) || code < 100 || code > 599) + continue; + string val = overridesJson.GetString(key); + if (val == "retry" || val == "drop") + result[code] = val; } + return result; } public override void Reset() diff --git a/Analytics-CSharp/Segment/Analytics/Utilities/EventPipeline.cs b/Analytics-CSharp/Segment/Analytics/Utilities/EventPipeline.cs index 21fe7ce..a58668a 100644 --- a/Analytics-CSharp/Segment/Analytics/Utilities/EventPipeline.cs +++ b/Analytics-CSharp/Segment/Analytics/Utilities/EventPipeline.cs @@ -19,7 +19,7 @@ public class EventPipeline: IEventPipeline private Channel _uploadChannel; - private readonly HTTPClient _httpClient; + internal readonly HTTPClient _httpClient; private readonly IStorage _storage; @@ -49,6 +49,9 @@ public EventPipeline( _uploadChannel = new Channel(); _httpClient = analytics.Configuration.HttpClientProvider.CreateHTTPClient(apiKey, apiHost: apiHost); _httpClient.AnalyticsRef = analytics; + _httpClient.MaxRetries = analytics.Configuration.MaxRetries; + _httpClient.MaxTotalBackoffDuration = analytics.Configuration.MaxTotalBackoffDuration; + _httpClient.MaxRateLimitDuration = analytics.Configuration.MaxRateLimitDuration; _storage = analytics.Storage; Running = false; } diff --git a/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs b/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs index 552383b..a0dfe41 100644 --- a/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs +++ b/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; @@ -31,16 +33,34 @@ public abstract class HTTPClient public Analytics AnalyticsRef { - get - { - return _reference.TryGetTarget(out Analytics analytics) ? analytics : null; - } - set - { - _reference.SetTarget(value); - } + get => _reference.TryGetTarget(out Analytics analytics) ? analytics : null; + set => _reference.SetTarget(value); } + // --- Retry configuration (set from Configuration or httpConfig settings) --- + + public int MaxRetries { get; set; } = 10; + + public TimeSpan MaxTotalBackoffDuration { get; set; } = TimeSpan.FromHours(12); + + public TimeSpan MaxRateLimitDuration { get; set; } = TimeSpan.FromHours(12); + + public bool BackoffEnabled { get; set; } = true; + + public bool RateLimitEnabled { get; set; } = true; + + public int MaxRateLimitRetries { get; set; } = 10; + + public int MaxRetryAfterCapSeconds { get; set; } = 300; + + public double BaseBackoffMs { get; set; } = 500.0; + + public double MaxBackoffMs { get; set; } = 60_000.0; + + public Dictionary StatusCodeOverrides { get; set; } = new Dictionary(); + + // ------------------------------------------------------------------------- + public HTTPClient(string apiKey, string apiHost = null, string cdnHost = null) { _apiKey = apiKey; @@ -50,23 +70,8 @@ public HTTPClient(string apiKey, string apiHost = null, string cdnHost = null) /// /// Returns formatted url to Segment's server. - /// If you want to use your own server, override this method like the following - /// - /// public virtual string SegmentURL(string host, string path) - /// { - /// if (host is cdnHost) - /// { - /// return cdn url with your own path - /// } - /// else { // is apiHost - /// return api url with your own path - /// } - /// } - /// + /// Override to use a custom server. /// - /// cdnHost or apiHost - /// Path to segment's /settings endpoint or /b endpoints - /// Formatted url public virtual string SegmentURL(string host, string path) => "https://" + host + path; public virtual async Task Settings() @@ -78,17 +83,18 @@ public HTTPClient(string apiKey, string apiHost = null, string cdnHost = null) Response response = await DoGet(settingsURL); if (!response.IsSuccessStatusCode) { - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnexpectedHttpCode, message: "Error " + response.StatusCode + " getting from settings url"); + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnexpectedHttpCode, + message: "Error " + response.StatusCode + " getting from settings url"); } else { - string json = response.Content; - result = JsonUtility.FromJson(json); + result = JsonUtility.FromJson(response.Content); } } catch (Exception e) { - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnknown, e, "Unknown network error when getting from settings url"); + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnknown, e, + "Unknown network error when getting from settings url"); } return result; @@ -97,56 +103,143 @@ public HTTPClient(string apiKey, string apiHost = null, string cdnHost = null) public virtual async Task Upload(byte[] data) { string uploadURL = SegmentURL(_apiHost, "/b"); - try + + // Snapshot config at start of upload to avoid mid-loop mutation + int maxRetries = MaxRetries; + int maxRateLimitRetries = MaxRateLimitRetries; + int retryAfterCapSeconds = MaxRetryAfterCapSeconds; + TimeSpan maxTotalBackoff = MaxTotalBackoffDuration; + TimeSpan maxRateLimit = MaxRateLimitDuration; + bool backoffEnabled = BackoffEnabled; + bool rateLimitEnabled = RateLimitEnabled; + double backoffMs = BaseBackoffMs; + double backoffCapMs = MaxBackoffMs; + var overrides = StatusCodeOverrides; + + int totalAttempts = 0; + int backoffAttempts = 0; + int rateLimitAttempts = 0; + DateTime? firstFailureTime = null; + DateTime? rateLimitStartTime = null; + + while (true) { - Response response = await DoPost(uploadURL, data); + totalAttempts++; + Response response = null; + bool isNetworkError = false; - if (!response.IsSuccessStatusCode) + try { - Analytics.Logger.Log(LogLevel.Error, message: "Error " + response.StatusCode + " uploading to url"); + response = await DoPost(uploadURL, data, retryCount: totalAttempts - 1); + } + catch (Exception e) + { + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnknown, e, + "Unknown network error when uploading to url"); + isNetworkError = true; + } - switch (response.StatusCode) + if (!isNetworkError) + { + // 2xx and 3xx are success + if (IsSuccess(response.StatusCode)) + return true; + + Analytics.Logger.Log(LogLevel.Error, + message: "Error " + response.StatusCode + " uploading to url"); + + // 429 handling + if (response.StatusCode == 429) { - case var n when n >= 1 && n < 300: - return false; - case var n when n >= 300 && n < 400: - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnexpectedHttpCode, message: "Response code: " + n); - return false; - case 429: - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkServerLimited, message: "Response code: 429"); - return false; - case var n when n >= 400 && n < 500: - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkServerRejected, message: "Response code: " + n + ". Payloads were rejected by server. Marked for removal."); - return true; - default: - return false; + if (!rateLimitEnabled) + return true; // rate limiting disabled — discard immediately + + TimeSpan? retryAfter = ParseRetryAfter(response.RetryAfterHeader, retryAfterCapSeconds); + if (retryAfter.HasValue) + { + if (rateLimitStartTime == null) rateLimitStartTime = DateTime.UtcNow; + rateLimitAttempts++; + if (rateLimitAttempts > maxRateLimitRetries || + DateTime.UtcNow - rateLimitStartTime.Value > maxRateLimit) + { + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkServerLimited, + message: "Max rate limit duration exceeded"); + return true; + } + await Task.Delay(retryAfter.Value); + continue; + } + // No Retry-After — fall through to counted backoff + } + + string action = GetStatusCodeAction(response.StatusCode, overrides); + if (action != "retry" || !backoffEnabled) + { + if (action != "retry") + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkServerRejected, + message: "Response code: " + response.StatusCode + ". Non-retryable. Discarding batch."); + return true; // non-retryable or backoff disabled → discard } } - return true; - } - catch (Exception e) - { - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnknown, e, "Unknown network error when uploading to url"); + // Counted exponential backoff + if (firstFailureTime == null) firstFailureTime = DateTime.UtcNow; + if (DateTime.UtcNow - firstFailureTime.Value > maxTotalBackoff) + { + Analytics.Logger.Log(LogLevel.Error, message: "Max total backoff duration exceeded"); + return true; // discard — budget exhausted, no point retrying next cycle + } + + backoffAttempts++; + if (backoffAttempts > maxRetries) + { + Analytics.Logger.Log(LogLevel.Error, + message: $"Retries exhausted after {totalAttempts} attempts"); + return true; // discard — retry budget exhausted + } + + await Task.Delay(TimeSpan.FromMilliseconds(backoffMs)); + backoffMs = Math.Min(backoffMs * 2, backoffCapMs); } + } - return false; + // --- Status classification helpers --- + + private static bool IsSuccess(int statusCode) => statusCode >= 200 && statusCode < 400; + + private static readonly int[] s_retryableClientErrors = { 408, 410, 429, 460 }; + private static readonly int[] s_nonRetryableServerErrors = { 501, 505, 511 }; + + private static bool IsRetryable(int statusCode) + { + if (statusCode >= 500 && statusCode < 600) + return Array.IndexOf(s_nonRetryableServerErrors, statusCode) < 0; + return Array.IndexOf(s_retryableClientErrors, statusCode) >= 0; } - /// - /// Handle GET request - /// - /// URL where the GET request sent to - /// Awaitable response of the GET request + private static string GetStatusCodeAction(int statusCode, Dictionary overrides) + { + if (overrides != null && overrides.TryGetValue(statusCode, out string action)) + return action; + return IsRetryable(statusCode) ? "retry" : "drop"; + } + + private static TimeSpan? ParseRetryAfter(string headerValue, int capSeconds = 300) + { + if (string.IsNullOrWhiteSpace(headerValue)) return null; + if (!int.TryParse(headerValue.Trim(), out int seconds)) return null; + if (seconds < 0) return null; + int clamped = Math.Max(1, Math.Min(seconds, Math.Max(1, capSeconds))); + return TimeSpan.FromSeconds(clamped); + } + + // ----------------------------------------------------------------------- + + /// Handle GET request public abstract Task DoGet(string url); - /// - /// Handle POST request - /// - /// URL where the POST request sent to - /// data to upload - /// Awaitable response of the POST request - public abstract Task DoPost(string url, byte[] data); + /// Handle POST request + public abstract Task DoPost(string url, byte[] data, int retryCount = 0); /// /// A wrapper class for http response, so that the HTTPClient is @@ -154,29 +247,24 @@ public virtual async Task Upload(byte[] data) /// public class Response { - /// - /// Status code of the http request - /// public int StatusCode { get; set; } - /// - /// Response content of the http request - /// public string Content { get; set; } - /// - /// A convenient method to check if the http request is successful - /// + /// Value of the Retry-After response header, or null if absent. + public string RetryAfterHeader { get; set; } + + /// True for 2xx responses only (used by Settings()). public bool IsSuccessStatusCode => StatusCode >= 200 && StatusCode < 300; } } public class DefaultHTTPClient : HTTPClient { - private readonly HttpClient _httpClient; - public DefaultHTTPClient(string apiKey, string apiHost = null, string cdnHost = null) : base(apiKey, apiHost, cdnHost) + public DefaultHTTPClient(string apiKey, string apiHost = null, string cdnHost = null) + : base(apiKey, apiHost, cdnHost) { _httpClient = new HttpClient(new HttpClientHandler { @@ -197,11 +285,10 @@ public override async Task DoGet(string url) Content = await response.Content.ReadAsStringAsync() }; response.Dispose(); - return result; } - public override async Task DoPost(string url, byte[] data) + public override async Task DoPost(string url, byte[] data, int retryCount = 0) { using (MemoryStream ms = new MemoryStream()) { @@ -217,12 +304,21 @@ public override async Task DoPost(string url, byte[] data) var request = new HttpRequestMessage(HttpMethod.Post, url); request.Headers.Add("Connection", "close"); request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/plain")); + if (retryCount > 0) + request.Headers.Add("X-Retry-Count", retryCount.ToString()); request.Content = streamContent; HttpResponseMessage response = await _httpClient.SendAsync(request); - var result = new Response {StatusCode = (int)response.StatusCode}; - response.Dispose(); + string retryAfterHeader = null; + if (response.Headers.TryGetValues("Retry-After", out var values)) + retryAfterHeader = values.FirstOrDefault(); + var result = new Response + { + StatusCode = (int)response.StatusCode, + RetryAfterHeader = retryAfterHeader + }; + response.Dispose(); return result; } } diff --git a/Samples/UnitySample/UnityHTTPClient.cs b/Samples/UnitySample/UnityHTTPClient.cs index 60cb8e0..65da86f 100644 --- a/Samples/UnitySample/UnityHTTPClient.cs +++ b/Samples/UnitySample/UnityHTTPClient.cs @@ -45,7 +45,7 @@ IEnumerator GetRequest(NetworkRequest networkRequest) } } - public override async Task DoPost(string url, byte[] data) + public override async Task DoPost(string url, byte[] data, int retryCount = 0) { using (var request = new NetworkRequest {URL = url, Data = data, Action = PostRequest}) { diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 099502d..7d36e00 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -1,7 +1,7 @@ - net6.0;net46 + net6.0;net46;net10.0 false diff --git a/e2e-cli/Program.cs b/e2e-cli/Program.cs index c2f1c31..6265705 100644 --- a/e2e-cli/Program.cs +++ b/e2e-cli/Program.cs @@ -54,6 +54,7 @@ // config block (optional) int flushAt = 15; int flushInterval = 10; // seconds +int maxRetries = 10; if (root.TryGetProperty("config", out var configEl)) { if (configEl.TryGetProperty("flushAt", out var fa)) flushAt = fa.GetInt32(); @@ -63,8 +64,13 @@ int fiMs = fi.GetInt32(); flushInterval = Math.Max(1, fiMs / 1000); } + if (configEl.TryGetProperty("maxRetries", out var mr)) maxRetries = mr.GetInt32(); } +// ── Logger (captures retry-exhaustion errors) ───────────────────────────────── +var capturingLogger = new CapturingLogger(); +Analytics.Logger = capturingLogger; + // ── Error handler ──────────────────────────────────────────────────────────── var errors = new List(); var errorHandler = new CapturingErrorHandler(errors); @@ -105,13 +111,22 @@ storageProvider: new InMemoryStorageProvider(), apiHost: rawApiHost, cdnHost: rawCdnHost, - httpClientProvider: httpClientProvider + httpClientProvider: httpClientProvider, + maxRetries: maxRetries ); Console.Error.WriteLine($"[e2e-cli] Initialising analytics (writeKey={writeKey[..Math.Min(8, writeKey.Length)]}…, apiHost={apiHost ?? "default"})"); var analytics = new Analytics(configBuilder); +// If AUTO_SETTINGS is enabled, wait briefly for the settings fetch to complete +// so that httpConfig overrides (BackoffEnabled, MaxRateLimitRetries, etc.) are +// applied before the first upload starts. +bool autoSettings = string.Equals(Environment.GetEnvironmentVariable("AUTO_SETTINGS"), "true", + StringComparison.OrdinalIgnoreCase); +if (autoSettings) + Thread.Sleep(2000); + // ── Process sequences ──────────────────────────────────────────────────────── int totalEvents = 0; @@ -149,23 +164,16 @@ case "track": { - // Set userId state first if provided - if (userId != null && analytics.UserId() != userId) - analytics.Identify(userId); - string eventName = ev.TryGetProperty("event", out var enEl) ? enEl.GetString() ?? "Unknown" : "Unknown"; JsonObject? properties = GetJsonObject(ev, "properties"); - analytics.Track(eventName, properties); + analytics.Track(eventName, properties, userId != null ? e => { ((TrackEvent)e).UserId = userId; return e; } : null); break; } case "page": { - if (userId != null && analytics.UserId() != userId) - analytics.Identify(userId); - string title = ev.TryGetProperty("name", out var nameEl) ? nameEl.GetString() ?? "" : ""; @@ -173,15 +181,12 @@ ? catEl.GetString() ?? "" : ""; JsonObject? properties = GetJsonObject(ev, "properties"); - analytics.Page(title, properties, category); + analytics.Page(title, properties, category, userId != null ? e => { ((PageEvent)e).UserId = userId; return e; } : null); break; } case "screen": { - if (userId != null && analytics.UserId() != userId) - analytics.Identify(userId); - string title = ev.TryGetProperty("name", out var nameEl) ? nameEl.GetString() ?? "" : ""; @@ -189,38 +194,29 @@ ? catEl.GetString() ?? "" : ""; JsonObject? properties = GetJsonObject(ev, "properties"); - analytics.Screen(title, properties, category); + analytics.Screen(title, properties, category, userId != null ? e => { ((ScreenEvent)e).UserId = userId; return e; } : null); break; } case "alias": { - // For alias: previousId becomes the current userId state, newId is the alias target. - // The SDK Alias(newId) uses _userInfo._userId as previousId. string? previousId = ev.TryGetProperty("previousId", out var prevEl) ? prevEl.GetString() : null; string newId = userId ?? (ev.TryGetProperty("newId", out var newIdEl) ? newIdEl.GetString() ?? "" : ""); - - if (previousId != null && analytics.UserId() != previousId) - analytics.Identify(previousId); - - analytics.Alias(newId); + analytics.Alias(newId, previousId != null ? e => { ((AliasEvent)e).PreviousId = previousId; return e; } : null); break; } case "group": { - if (userId != null && analytics.UserId() != userId) - analytics.Identify(userId); - string groupId = ev.TryGetProperty("groupId", out var gidEl) ? gidEl.GetString() ?? "" : ""; JsonObject? traits = GetJsonObject(ev, "traits"); - analytics.Group(groupId, traits); + analytics.Group(groupId, traits, userId != null ? e => { ((GroupEvent)e).UserId = userId; return e; } : null); break; } @@ -237,11 +233,19 @@ Console.Error.WriteLine($"[e2e-cli] Flushing {totalEvents} event(s)…"); analytics.Flush(); -// Give the async pipeline time to upload -Thread.Sleep(5000); +// Wait for the async pipeline to finish flushing + retries. +// Cap at 25s so tests with a 30s timeout always get a result. +int waitMs = Math.Min(Math.Max(10_000, maxRetries * 2_000 + 5_000), 25_000); +Thread.Sleep(waitMs); // ── Output result ───────────────────────────────────────────────────────────── -bool success = errors.Count == 0; +// Combine SDK error handler errors (non-retryable drops) with captured logger errors +// (retry exhaustion, backoff budget exceeded). Either signals final failure. +var logErrors = ((CapturingLogger)Analytics.Logger).Errors; +var allErrors = new List(errors); +allErrors.AddRange(logErrors); + +bool success = allErrors.Count == 0; if (success) { Console.WriteLine($"{{\"success\":true,\"sentBatches\":1}}"); @@ -249,7 +253,7 @@ } else { - string combinedErrors = string.Join("; ", errors); + string combinedErrors = string.Join("; ", allErrors); Console.WriteLine($"{{\"success\":false,\"sentBatches\":0,\"error\":\"{Escape(combinedErrors)}\"}}"); Environment.Exit(1); } @@ -312,3 +316,25 @@ public void OnExceptionThrown(Exception e) _errors.Add(msg); } } + +// ── Logger that captures Error-level messages (retry exhaustion, etc.) ──────── + +class CapturingLogger : Segment.Analytics.Utilities.ISegmentLogger +{ + private readonly List _errors = new List(); + public IReadOnlyList Errors => _errors; + + public void Log(Segment.Analytics.Utilities.LogLevel logLevel, Exception exception = null, string message = null) + { + string text = message ?? exception?.Message ?? ""; + Console.Error.WriteLine($"[analytics][{logLevel}] {text}"); + // Only capture final-failure messages, not transient per-attempt errors. + // Transient errors look like "Error 500 uploading to url". + // Final failures are "Retries exhausted..." and "Max total backoff...". + if (logLevel == Segment.Analytics.Utilities.LogLevel.Error && !string.IsNullOrEmpty(text) + && (text.StartsWith("Retries exhausted") || text.StartsWith("Max total backoff"))) + { + _errors.Add(text); + } + } +} diff --git a/e2e-cli/e2e-config.json b/e2e-cli/e2e-config.json index eea1cca..91e0a5f 100644 --- a/e2e-cli/e2e-config.json +++ b/e2e-cli/e2e-config.json @@ -1,7 +1,9 @@ { "sdk": "csharp", - "test_suites": "basic", - "auto_settings": false, + "test_suites": "basic,retry,retry-settings", + "auto_settings": true, "patch": null, - "env": {} + "env": { + "HTTP_CONFIG_SETTINGS": "true" + } }