From bb0e9259c536e371c333d0a271edf81ea62b24e2 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Fri, 15 May 2026 18:57:32 -0400 Subject: [PATCH 1/4] Implement status-response retry improvements - HTTPClient: dual-path retry loop (429+Retry-After vs counted exponential backoff), configurable retry properties (MaxRetries, MaxTotalBackoffDuration, MaxRateLimitDuration, BackoffEnabled, RateLimitEnabled, BaseBackoffMs, MaxBackoffMs, StatusCodeOverrides), X-Retry-Count header on retries - Configuration: add MaxRetries, MaxTotalBackoffDuration, MaxRateLimitDuration constructor params - EventPipeline: wire config retry params into HTTPClient at construction; expose _httpClient as internal - SegmentDestination: parse httpConfig from CDN settings response and apply to HTTPClient at runtime - UnityHTTPClient: update DoPost signature to match new abstract base - Tests.csproj: add net10.0 target for local dev (net6.0 runtime no longer installed) --- .../Segment/Analytics/Configuration.cs | 17 +- .../Analytics/Plugins/SegmentDestination.cs | 66 ++++- .../Analytics/Utilities/EventPipeline.cs | 5 +- .../Segment/Analytics/Utilities/HTTPClient.cs | 247 ++++++++++++------ Samples/UnitySample/UnityHTTPClient.cs | 2 +- Tests/Tests.csproj | 2 +- 6 files changed, 255 insertions(+), 84 deletions(-) diff --git a/Analytics-CSharp/Segment/Analytics/Configuration.cs b/Analytics-CSharp/Segment/Analytics/Configuration.cs index 79de436..59d489c 100644 --- a/Analytics-CSharp/Segment/Analytics/Configuration.cs +++ b/Analytics-CSharp/Segment/Analytics/Configuration.cs @@ -47,6 +47,12 @@ private set public IEventPipelineProvider EventPipelineProvider { get; } + public int MaxRetries { get; } + + public TimeSpan MaxTotalBackoffDuration { get; } + + public TimeSpan MaxRateLimitDuration { get; } + /// /// Configuration that analytics can use /// @@ -73,6 +79,9 @@ private set /// defaults to DefaultHTTPClientProvider /// /// set custom flush policies to tell analytics when and how to flush. If a value is given, it overwrites flushAt and flushInterval + /// maximum number of backoff retries per batch upload, defaults to 10 + /// wall-clock cap on total backoff time, defaults to 12 hours + /// wall-clock cap on 429 Retry-After retries, defaults to 12 hours public Configuration(string writeKey, int flushAt = 20, int flushInterval = 30, @@ -85,7 +94,10 @@ public Configuration(string writeKey, IStorageProvider storageProvider = default, IHTTPClientProvider httpClientProvider = default, IList flushPolicies = default, - IEventPipelineProvider eventPipelineProvider = default) + IEventPipelineProvider eventPipelineProvider = default, + int maxRetries = 10, + TimeSpan? maxTotalBackoffDuration = null, + TimeSpan? maxRateLimitDuration = null) { WriteKey = writeKey; FlushAt = flushAt; @@ -102,6 +114,9 @@ public Configuration(string writeKey, FlushPolicies.Add(new CountFlushPolicy(flushAt)); FlushPolicies.Add(new FrequencyFlushPolicy(flushInterval * 1000L)); EventPipelineProvider = eventPipelineProvider ?? new EventPipelineProvider(); + MaxRetries = maxRetries; + MaxTotalBackoffDuration = maxTotalBackoffDuration ?? TimeSpan.FromHours(12); + MaxRateLimitDuration = maxRateLimitDuration ?? TimeSpan.FromHours(12); } public Configuration(string writeKey, diff --git a/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs b/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs index 9586be0..58061b0 100644 --- a/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs +++ b/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs @@ -1,3 +1,5 @@ +using System; +using System.Collections.Generic; using Segment.Analytics.Utilities; using Segment.Serialization; using Segment.Sovran; @@ -77,11 +79,73 @@ public override void Update(Settings settings, UpdateType type) base.Update(settings, type); JsonObject segmentInfo = settings.Integrations?.GetJsonObject(Key); + string apiHost = segmentInfo?.GetString(ApiHost); if (apiHost != null && _pipeline != null) - { _pipeline.ApiHost = apiHost; + + JsonObject httpConfig = segmentInfo?.GetJsonObject("httpConfig"); + EventPipeline concretePipeline = _pipeline as EventPipeline; + if (httpConfig != null && concretePipeline?._httpClient != null) + ApplyHttpConfig(concretePipeline._httpClient, httpConfig); + } + + private static void ApplyHttpConfig(HTTPClient client, JsonObject httpConfig) + { + JsonObject backoff = httpConfig.GetJsonObject("backoffConfig"); + if (backoff != null) + { + string enabledStr = backoff.GetString("enabled"); + if (enabledStr != null && bool.TryParse(enabledStr, out bool enabled)) + client.BackoffEnabled = enabled; + + string maxRetriesStr = backoff.GetString("maxRetryCount"); + if (maxRetriesStr != null && int.TryParse(maxRetriesStr, out int maxRetries)) + client.MaxRetries = maxRetries; + + string baseStr = backoff.GetString("baseBackoffInterval"); + if (baseStr != null && double.TryParse(baseStr, out double baseMs)) + client.BaseBackoffMs = baseMs; + + string capStr = backoff.GetString("maxBackoffInterval"); + if (capStr != null && double.TryParse(capStr, out double capMs)) + client.MaxBackoffMs = capMs; + + JsonObject overridesJson = backoff.GetJsonObject("statusCodeOverrides"); + if (overridesJson != null) + client.StatusCodeOverrides = ParseStatusCodeOverrides(overridesJson); + } + + JsonObject rateLimit = httpConfig.GetJsonObject("rateLimitConfig"); + if (rateLimit != null) + { + string enabledStr = rateLimit.GetString("enabled"); + if (enabledStr != null && bool.TryParse(enabledStr, out bool enabled)) + client.RateLimitEnabled = enabled; + + string maxRetriesStr = rateLimit.GetString("maxRetryCount"); + if (maxRetriesStr != null && int.TryParse(maxRetriesStr, out int maxRetries)) + client.MaxRateLimitRetries = maxRetries; + + string capStr = rateLimit.GetString("maxRetryInterval"); + if (capStr != null && int.TryParse(capStr, out int capSec)) + client.MaxRateLimitDuration = TimeSpan.FromSeconds(capSec); + } + } + + private static Dictionary ParseStatusCodeOverrides(JsonObject overridesJson) + { + var result = new Dictionary(); + // JsonObject iteration — enumerate known entries via string keys + // We use a conservative approach: try common status codes + foreach (int code in new[] { 200, 201, 204, 301, 302, 400, 401, 403, 404, 408, 410, 413, + 422, 429, 460, 499, 500, 501, 502, 503, 504, 505, 508, 511 }) + { + string val = overridesJson.GetString(code.ToString()); + if (val == "retry" || val == "drop") + result[code] = val; } + return result; } public override void Reset() diff --git a/Analytics-CSharp/Segment/Analytics/Utilities/EventPipeline.cs b/Analytics-CSharp/Segment/Analytics/Utilities/EventPipeline.cs index 21fe7ce..a58668a 100644 --- a/Analytics-CSharp/Segment/Analytics/Utilities/EventPipeline.cs +++ b/Analytics-CSharp/Segment/Analytics/Utilities/EventPipeline.cs @@ -19,7 +19,7 @@ public class EventPipeline: IEventPipeline private Channel _uploadChannel; - private readonly HTTPClient _httpClient; + internal readonly HTTPClient _httpClient; private readonly IStorage _storage; @@ -49,6 +49,9 @@ public EventPipeline( _uploadChannel = new Channel(); _httpClient = analytics.Configuration.HttpClientProvider.CreateHTTPClient(apiKey, apiHost: apiHost); _httpClient.AnalyticsRef = analytics; + _httpClient.MaxRetries = analytics.Configuration.MaxRetries; + _httpClient.MaxTotalBackoffDuration = analytics.Configuration.MaxTotalBackoffDuration; + _httpClient.MaxRateLimitDuration = analytics.Configuration.MaxRateLimitDuration; _storage = analytics.Storage; Running = false; } diff --git a/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs b/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs index 552383b..e73103e 100644 --- a/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs +++ b/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; @@ -31,16 +33,32 @@ public abstract class HTTPClient public Analytics AnalyticsRef { - get - { - return _reference.TryGetTarget(out Analytics analytics) ? analytics : null; - } - set - { - _reference.SetTarget(value); - } + get => _reference.TryGetTarget(out Analytics analytics) ? analytics : null; + set => _reference.SetTarget(value); } + // --- Retry configuration (set from Configuration or httpConfig settings) --- + + public int MaxRetries { get; set; } = 10; + + public TimeSpan MaxTotalBackoffDuration { get; set; } = TimeSpan.FromHours(12); + + public TimeSpan MaxRateLimitDuration { get; set; } = TimeSpan.FromHours(12); + + public bool BackoffEnabled { get; set; } = true; + + public bool RateLimitEnabled { get; set; } = true; + + public int MaxRateLimitRetries { get; set; } = 10; + + public double BaseBackoffMs { get; set; } = 500.0; + + public double MaxBackoffMs { get; set; } = 60_000.0; + + public Dictionary StatusCodeOverrides { get; set; } = new Dictionary(); + + // ------------------------------------------------------------------------- + public HTTPClient(string apiKey, string apiHost = null, string cdnHost = null) { _apiKey = apiKey; @@ -50,23 +68,8 @@ public HTTPClient(string apiKey, string apiHost = null, string cdnHost = null) /// /// Returns formatted url to Segment's server. - /// If you want to use your own server, override this method like the following - /// - /// public virtual string SegmentURL(string host, string path) - /// { - /// if (host is cdnHost) - /// { - /// return cdn url with your own path - /// } - /// else { // is apiHost - /// return api url with your own path - /// } - /// } - /// + /// Override to use a custom server. /// - /// cdnHost or apiHost - /// Path to segment's /settings endpoint or /b endpoints - /// Formatted url public virtual string SegmentURL(string host, string path) => "https://" + host + path; public virtual async Task Settings() @@ -78,17 +81,18 @@ public HTTPClient(string apiKey, string apiHost = null, string cdnHost = null) Response response = await DoGet(settingsURL); if (!response.IsSuccessStatusCode) { - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnexpectedHttpCode, message: "Error " + response.StatusCode + " getting from settings url"); + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnexpectedHttpCode, + message: "Error " + response.StatusCode + " getting from settings url"); } else { - string json = response.Content; - result = JsonUtility.FromJson(json); + result = JsonUtility.FromJson(response.Content); } } catch (Exception e) { - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnknown, e, "Unknown network error when getting from settings url"); + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnknown, e, + "Unknown network error when getting from settings url"); } return result; @@ -97,56 +101,138 @@ public HTTPClient(string apiKey, string apiHost = null, string cdnHost = null) public virtual async Task Upload(byte[] data) { string uploadURL = SegmentURL(_apiHost, "/b"); - try + + // Snapshot config at start of upload to avoid mid-loop mutation + int maxRetries = MaxRetries; + int maxRateLimitRetries = MaxRateLimitRetries; + TimeSpan maxTotalBackoff = MaxTotalBackoffDuration; + TimeSpan maxRateLimit = MaxRateLimitDuration; + bool backoffEnabled = BackoffEnabled; + bool rateLimitEnabled = RateLimitEnabled; + double backoffMs = BaseBackoffMs; + double backoffCapMs = MaxBackoffMs; + var overrides = StatusCodeOverrides; + + int totalAttempts = 0; + int backoffAttempts = 0; + int rateLimitAttempts = 0; + DateTime? firstFailureTime = null; + DateTime? rateLimitStartTime = null; + + while (true) { - Response response = await DoPost(uploadURL, data); + totalAttempts++; + Response response = null; + bool isNetworkError = false; - if (!response.IsSuccessStatusCode) + try + { + response = await DoPost(uploadURL, data, retryCount: totalAttempts - 1); + } + catch (Exception e) + { + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnknown, e, + "Unknown network error when uploading to url"); + isNetworkError = true; + } + + if (!isNetworkError) { - Analytics.Logger.Log(LogLevel.Error, message: "Error " + response.StatusCode + " uploading to url"); + // 2xx and 3xx are success + if (IsSuccess(response.StatusCode)) + return true; + + Analytics.Logger.Log(LogLevel.Error, + message: "Error " + response.StatusCode + " uploading to url"); - switch (response.StatusCode) + // 429 handling + if (response.StatusCode == 429 && rateLimitEnabled) { - case var n when n >= 1 && n < 300: - return false; - case var n when n >= 300 && n < 400: - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnexpectedHttpCode, message: "Response code: " + n); - return false; - case 429: - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkServerLimited, message: "Response code: 429"); - return false; - case var n when n >= 400 && n < 500: - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkServerRejected, message: "Response code: " + n + ". Payloads were rejected by server. Marked for removal."); - return true; - default: - return false; + TimeSpan? retryAfter = ParseRetryAfter(response.RetryAfterHeader); + if (retryAfter.HasValue) + { + if (rateLimitStartTime == null) rateLimitStartTime = DateTime.UtcNow; + rateLimitAttempts++; + if (rateLimitAttempts > maxRateLimitRetries || + DateTime.UtcNow - rateLimitStartTime.Value > maxRateLimit) + { + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkServerLimited, + message: "Max rate limit duration exceeded"); + return true; + } + await Task.Delay(retryAfter.Value); + continue; + } + // No Retry-After — fall through to counted backoff + } + + string action = GetStatusCodeAction(response.StatusCode, overrides); + if (action != "retry" || !backoffEnabled) + { + if (action != "retry") + AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkServerRejected, + message: "Response code: " + response.StatusCode + ". Non-retryable. Discarding batch."); + return action != "retry"; // non-retryable → discard (true); backoff disabled → discard (true) } } - return true; - } - catch (Exception e) - { - AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkUnknown, e, "Unknown network error when uploading to url"); + // Counted exponential backoff + if (firstFailureTime == null) firstFailureTime = DateTime.UtcNow; + if (DateTime.UtcNow - firstFailureTime.Value > maxTotalBackoff) + { + Analytics.Logger.Log(LogLevel.Error, message: "Max total backoff duration exceeded"); + return false; + } + + backoffAttempts++; + if (backoffAttempts > maxRetries) + { + Analytics.Logger.Log(LogLevel.Error, + message: $"Retries exhausted after {totalAttempts} attempts"); + return false; + } + + await Task.Delay(TimeSpan.FromMilliseconds(backoffMs)); + backoffMs = Math.Min(backoffMs * 2, backoffCapMs); } + } - return false; + // --- Status classification helpers --- + + private static bool IsSuccess(int statusCode) => statusCode >= 200 && statusCode < 400; + + private static readonly int[] s_retryableClientErrors = { 408, 410, 429, 460 }; + private static readonly int[] s_nonRetryableServerErrors = { 501, 505, 511 }; + + private static bool IsRetryable(int statusCode) + { + if (statusCode >= 500 && statusCode < 600) + return Array.IndexOf(s_nonRetryableServerErrors, statusCode) < 0; + return Array.IndexOf(s_retryableClientErrors, statusCode) >= 0; } - /// - /// Handle GET request - /// - /// URL where the GET request sent to - /// Awaitable response of the GET request + private static string GetStatusCodeAction(int statusCode, Dictionary overrides) + { + if (overrides != null && overrides.TryGetValue(statusCode, out string action)) + return action; + return IsRetryable(statusCode) ? "retry" : "drop"; + } + + private static TimeSpan? ParseRetryAfter(string headerValue, int capSeconds = 300) + { + if (string.IsNullOrWhiteSpace(headerValue)) return null; + if (!int.TryParse(headerValue.Trim(), out int seconds)) return null; + if (seconds <= 0) return null; + return TimeSpan.FromSeconds(Math.Min(seconds, capSeconds)); + } + + // ----------------------------------------------------------------------- + + /// Handle GET request public abstract Task DoGet(string url); - /// - /// Handle POST request - /// - /// URL where the POST request sent to - /// data to upload - /// Awaitable response of the POST request - public abstract Task DoPost(string url, byte[] data); + /// Handle POST request + public abstract Task DoPost(string url, byte[] data, int retryCount = 0); /// /// A wrapper class for http response, so that the HTTPClient is @@ -154,29 +240,24 @@ public virtual async Task Upload(byte[] data) /// public class Response { - /// - /// Status code of the http request - /// public int StatusCode { get; set; } - /// - /// Response content of the http request - /// public string Content { get; set; } - /// - /// A convenient method to check if the http request is successful - /// + /// Value of the Retry-After response header, or null if absent. + public string RetryAfterHeader { get; set; } + + /// True for 2xx responses only (used by Settings()). public bool IsSuccessStatusCode => StatusCode >= 200 && StatusCode < 300; } } public class DefaultHTTPClient : HTTPClient { - private readonly HttpClient _httpClient; - public DefaultHTTPClient(string apiKey, string apiHost = null, string cdnHost = null) : base(apiKey, apiHost, cdnHost) + public DefaultHTTPClient(string apiKey, string apiHost = null, string cdnHost = null) + : base(apiKey, apiHost, cdnHost) { _httpClient = new HttpClient(new HttpClientHandler { @@ -197,11 +278,10 @@ public override async Task DoGet(string url) Content = await response.Content.ReadAsStringAsync() }; response.Dispose(); - return result; } - public override async Task DoPost(string url, byte[] data) + public override async Task DoPost(string url, byte[] data, int retryCount = 0) { using (MemoryStream ms = new MemoryStream()) { @@ -217,12 +297,21 @@ public override async Task DoPost(string url, byte[] data) var request = new HttpRequestMessage(HttpMethod.Post, url); request.Headers.Add("Connection", "close"); request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/plain")); + if (retryCount > 0) + request.Headers.Add("X-Retry-Count", retryCount.ToString()); request.Content = streamContent; HttpResponseMessage response = await _httpClient.SendAsync(request); - var result = new Response {StatusCode = (int)response.StatusCode}; - response.Dispose(); + string retryAfterHeader = null; + if (response.Headers.TryGetValues("Retry-After", out var values)) + retryAfterHeader = values.FirstOrDefault(); + var result = new Response + { + StatusCode = (int)response.StatusCode, + RetryAfterHeader = retryAfterHeader + }; + response.Dispose(); return result; } } diff --git a/Samples/UnitySample/UnityHTTPClient.cs b/Samples/UnitySample/UnityHTTPClient.cs index 60cb8e0..65da86f 100644 --- a/Samples/UnitySample/UnityHTTPClient.cs +++ b/Samples/UnitySample/UnityHTTPClient.cs @@ -45,7 +45,7 @@ IEnumerator GetRequest(NetworkRequest networkRequest) } } - public override async Task DoPost(string url, byte[] data) + public override async Task DoPost(string url, byte[] data, int retryCount = 0) { using (var request = new NetworkRequest {URL = url, Data = data, Action = PostRequest}) { diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 099502d..7d36e00 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -1,7 +1,7 @@ - net6.0;net46 + net6.0;net46;net10.0 false From 3611efad27a64219d2b6927140c1aaa10ea6e302 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Tue, 19 May 2026 13:33:39 -0400 Subject: [PATCH 2/4] Implement dual-path retry with CDN-driven httpConfig support HTTPClient.cs: - Rewrite Upload() with dual-path retry: 429+Retry-After uses uncounted rate-limit path; other retryable errors use counted exponential backoff - Add configurable properties: MaxRetries, MaxRateLimitRetries, BackoffEnabled, RateLimitEnabled, BaseBackoffMs, MaxBackoffMs, MaxRetryAfterCapSeconds, StatusCodeOverrides - Return true (discard) on all terminal failure paths so EventPipeline removes the batch file instead of re-uploading indefinitely - Accept Retry-After: 0 as valid (sleep 0ms, still counts as rate-limit) SegmentDestination.cs: - Read httpConfig from CDN settings and apply backoffConfig/rateLimitConfig - Use CultureInfo.InvariantCulture for double parsing - Map maxRetryInterval to MaxRetryAfterCapSeconds (per-request cap) e2e-cli: - Add CapturingLogger that only captures final-failure messages - Use enrichment callbacks for userId instead of pre-identify - Add AUTO_SETTINGS 2s wait for settings to arrive before first upload - Enable retry-settings test suite --- .../Analytics/Plugins/SegmentDestination.cs | 7 +- .../Segment/Analytics/Utilities/HTTPClient.cs | 18 ++-- e2e-cli/Program.cs | 84 ++++++++++++------- e2e-cli/e2e-config.json | 8 +- 4 files changed, 76 insertions(+), 41 deletions(-) diff --git a/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs b/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs index 58061b0..319c787 100644 --- a/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs +++ b/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using Segment.Analytics.Utilities; using Segment.Serialization; using Segment.Sovran; @@ -104,11 +105,11 @@ private static void ApplyHttpConfig(HTTPClient client, JsonObject httpConfig) client.MaxRetries = maxRetries; string baseStr = backoff.GetString("baseBackoffInterval"); - if (baseStr != null && double.TryParse(baseStr, out double baseMs)) + if (baseStr != null && double.TryParse(baseStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double baseMs)) client.BaseBackoffMs = baseMs; string capStr = backoff.GetString("maxBackoffInterval"); - if (capStr != null && double.TryParse(capStr, out double capMs)) + if (capStr != null && double.TryParse(capStr, NumberStyles.Float, CultureInfo.InvariantCulture, out double capMs)) client.MaxBackoffMs = capMs; JsonObject overridesJson = backoff.GetJsonObject("statusCodeOverrides"); @@ -129,7 +130,7 @@ private static void ApplyHttpConfig(HTTPClient client, JsonObject httpConfig) string capStr = rateLimit.GetString("maxRetryInterval"); if (capStr != null && int.TryParse(capStr, out int capSec)) - client.MaxRateLimitDuration = TimeSpan.FromSeconds(capSec); + client.MaxRetryAfterCapSeconds = capSec; } } diff --git a/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs b/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs index e73103e..611527a 100644 --- a/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs +++ b/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs @@ -51,6 +51,8 @@ public Analytics AnalyticsRef public int MaxRateLimitRetries { get; set; } = 10; + public int MaxRetryAfterCapSeconds { get; set; } = 300; + public double BaseBackoffMs { get; set; } = 500.0; public double MaxBackoffMs { get; set; } = 60_000.0; @@ -105,6 +107,7 @@ public virtual async Task Upload(byte[] data) // Snapshot config at start of upload to avoid mid-loop mutation int maxRetries = MaxRetries; int maxRateLimitRetries = MaxRateLimitRetries; + int retryAfterCapSeconds = MaxRetryAfterCapSeconds; TimeSpan maxTotalBackoff = MaxTotalBackoffDuration; TimeSpan maxRateLimit = MaxRateLimitDuration; bool backoffEnabled = BackoffEnabled; @@ -146,9 +149,12 @@ public virtual async Task Upload(byte[] data) message: "Error " + response.StatusCode + " uploading to url"); // 429 handling - if (response.StatusCode == 429 && rateLimitEnabled) + if (response.StatusCode == 429) { - TimeSpan? retryAfter = ParseRetryAfter(response.RetryAfterHeader); + if (!rateLimitEnabled) + return true; // rate limiting disabled — discard immediately + + TimeSpan? retryAfter = ParseRetryAfter(response.RetryAfterHeader, retryAfterCapSeconds); if (retryAfter.HasValue) { if (rateLimitStartTime == null) rateLimitStartTime = DateTime.UtcNow; @@ -172,7 +178,7 @@ public virtual async Task Upload(byte[] data) if (action != "retry") AnalyticsRef?.ReportInternalError(AnalyticsErrorType.NetworkServerRejected, message: "Response code: " + response.StatusCode + ". Non-retryable. Discarding batch."); - return action != "retry"; // non-retryable → discard (true); backoff disabled → discard (true) + return true; // non-retryable or backoff disabled → discard } } @@ -181,7 +187,7 @@ public virtual async Task Upload(byte[] data) if (DateTime.UtcNow - firstFailureTime.Value > maxTotalBackoff) { Analytics.Logger.Log(LogLevel.Error, message: "Max total backoff duration exceeded"); - return false; + return true; // discard — budget exhausted, no point retrying next cycle } backoffAttempts++; @@ -189,7 +195,7 @@ public virtual async Task Upload(byte[] data) { Analytics.Logger.Log(LogLevel.Error, message: $"Retries exhausted after {totalAttempts} attempts"); - return false; + return true; // discard — retry budget exhausted } await Task.Delay(TimeSpan.FromMilliseconds(backoffMs)); @@ -222,7 +228,7 @@ private static string GetStatusCodeAction(int statusCode, Dictionary(); var errorHandler = new CapturingErrorHandler(errors); @@ -105,13 +111,22 @@ storageProvider: new InMemoryStorageProvider(), apiHost: rawApiHost, cdnHost: rawCdnHost, - httpClientProvider: httpClientProvider + httpClientProvider: httpClientProvider, + maxRetries: maxRetries ); Console.Error.WriteLine($"[e2e-cli] Initialising analytics (writeKey={writeKey[..Math.Min(8, writeKey.Length)]}…, apiHost={apiHost ?? "default"})"); var analytics = new Analytics(configBuilder); +// If AUTO_SETTINGS is enabled, wait briefly for the settings fetch to complete +// so that httpConfig overrides (BackoffEnabled, MaxRateLimitRetries, etc.) are +// applied before the first upload starts. +bool autoSettings = string.Equals(Environment.GetEnvironmentVariable("AUTO_SETTINGS"), "true", + StringComparison.OrdinalIgnoreCase); +if (autoSettings) + Thread.Sleep(2000); + // ── Process sequences ──────────────────────────────────────────────────────── int totalEvents = 0; @@ -149,23 +164,16 @@ case "track": { - // Set userId state first if provided - if (userId != null && analytics.UserId() != userId) - analytics.Identify(userId); - string eventName = ev.TryGetProperty("event", out var enEl) ? enEl.GetString() ?? "Unknown" : "Unknown"; JsonObject? properties = GetJsonObject(ev, "properties"); - analytics.Track(eventName, properties); + analytics.Track(eventName, properties, userId != null ? e => { ((TrackEvent)e).UserId = userId; return e; } : null); break; } case "page": { - if (userId != null && analytics.UserId() != userId) - analytics.Identify(userId); - string title = ev.TryGetProperty("name", out var nameEl) ? nameEl.GetString() ?? "" : ""; @@ -173,15 +181,12 @@ ? catEl.GetString() ?? "" : ""; JsonObject? properties = GetJsonObject(ev, "properties"); - analytics.Page(title, properties, category); + analytics.Page(title, properties, category, userId != null ? e => { ((PageEvent)e).UserId = userId; return e; } : null); break; } case "screen": { - if (userId != null && analytics.UserId() != userId) - analytics.Identify(userId); - string title = ev.TryGetProperty("name", out var nameEl) ? nameEl.GetString() ?? "" : ""; @@ -189,38 +194,29 @@ ? catEl.GetString() ?? "" : ""; JsonObject? properties = GetJsonObject(ev, "properties"); - analytics.Screen(title, properties, category); + analytics.Screen(title, properties, category, userId != null ? e => { ((ScreenEvent)e).UserId = userId; return e; } : null); break; } case "alias": { - // For alias: previousId becomes the current userId state, newId is the alias target. - // The SDK Alias(newId) uses _userInfo._userId as previousId. string? previousId = ev.TryGetProperty("previousId", out var prevEl) ? prevEl.GetString() : null; string newId = userId ?? (ev.TryGetProperty("newId", out var newIdEl) ? newIdEl.GetString() ?? "" : ""); - - if (previousId != null && analytics.UserId() != previousId) - analytics.Identify(previousId); - - analytics.Alias(newId); + analytics.Alias(newId, previousId != null ? e => { ((AliasEvent)e).PreviousId = previousId; return e; } : null); break; } case "group": { - if (userId != null && analytics.UserId() != userId) - analytics.Identify(userId); - string groupId = ev.TryGetProperty("groupId", out var gidEl) ? gidEl.GetString() ?? "" : ""; JsonObject? traits = GetJsonObject(ev, "traits"); - analytics.Group(groupId, traits); + analytics.Group(groupId, traits, userId != null ? e => { ((GroupEvent)e).UserId = userId; return e; } : null); break; } @@ -237,11 +233,19 @@ Console.Error.WriteLine($"[e2e-cli] Flushing {totalEvents} event(s)…"); analytics.Flush(); -// Give the async pipeline time to upload -Thread.Sleep(5000); +// Wait for the async pipeline to finish flushing + retries. +// Cap at 25s so tests with a 30s timeout always get a result. +int waitMs = Math.Min(Math.Max(10_000, maxRetries * 2_000 + 5_000), 25_000); +Thread.Sleep(waitMs); // ── Output result ───────────────────────────────────────────────────────────── -bool success = errors.Count == 0; +// Combine SDK error handler errors (non-retryable drops) with captured logger errors +// (retry exhaustion, backoff budget exceeded). Either signals final failure. +var logErrors = ((CapturingLogger)Analytics.Logger).Errors; +var allErrors = new List(errors); +allErrors.AddRange(logErrors); + +bool success = allErrors.Count == 0; if (success) { Console.WriteLine($"{{\"success\":true,\"sentBatches\":1}}"); @@ -249,7 +253,7 @@ } else { - string combinedErrors = string.Join("; ", errors); + string combinedErrors = string.Join("; ", allErrors); Console.WriteLine($"{{\"success\":false,\"sentBatches\":0,\"error\":\"{Escape(combinedErrors)}\"}}"); Environment.Exit(1); } @@ -312,3 +316,25 @@ public void OnExceptionThrown(Exception e) _errors.Add(msg); } } + +// ── Logger that captures Error-level messages (retry exhaustion, etc.) ──────── + +class CapturingLogger : Segment.Analytics.Utilities.ISegmentLogger +{ + private readonly List _errors = new List(); + public IReadOnlyList Errors => _errors; + + public void Log(Segment.Analytics.Utilities.LogLevel logLevel, Exception exception = null, string message = null) + { + string text = message ?? exception?.Message ?? ""; + Console.Error.WriteLine($"[analytics][{logLevel}] {text}"); + // Only capture final-failure messages, not transient per-attempt errors. + // Transient errors look like "Error 500 uploading to url". + // Final failures are "Retries exhausted..." and "Max total backoff...". + if (logLevel == Segment.Analytics.Utilities.LogLevel.Error && !string.IsNullOrEmpty(text) + && (text.StartsWith("Retries exhausted") || text.StartsWith("Max total backoff"))) + { + _errors.Add(text); + } + } +} diff --git a/e2e-cli/e2e-config.json b/e2e-cli/e2e-config.json index eea1cca..91e0a5f 100644 --- a/e2e-cli/e2e-config.json +++ b/e2e-cli/e2e-config.json @@ -1,7 +1,9 @@ { "sdk": "csharp", - "test_suites": "basic", - "auto_settings": false, + "test_suites": "basic,retry,retry-settings", + "auto_settings": true, "patch": null, - "env": {} + "env": { + "HTTP_CONFIG_SETTINGS": "true" + } } From 2378654fe48c11dcd101f639d5dcb5012f6fcbcc Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Thu, 21 May 2026 10:57:49 -0400 Subject: [PATCH 3/4] Fix review findings: ParseRetryAfter floor, dynamic status code overrides - Clamp ParseRetryAfter minimum to 1 second to prevent busy-loop when server sends Retry-After: 0 or MaxRetryAfterCapSeconds is misconfigured - Replace hardcoded status code list in ParseStatusCodeOverrides with dynamic key enumeration (matches analytics-kotlin approach), so CDN overrides for any valid HTTP status code (100-599) are respected --- .../Segment/Analytics/Plugins/SegmentDestination.cs | 9 ++++----- .../Segment/Analytics/Utilities/HTTPClient.cs | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs b/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs index 319c787..6e985b9 100644 --- a/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs +++ b/Analytics-CSharp/Segment/Analytics/Plugins/SegmentDestination.cs @@ -137,12 +137,11 @@ private static void ApplyHttpConfig(HTTPClient client, JsonObject httpConfig) private static Dictionary ParseStatusCodeOverrides(JsonObject overridesJson) { var result = new Dictionary(); - // JsonObject iteration — enumerate known entries via string keys - // We use a conservative approach: try common status codes - foreach (int code in new[] { 200, 201, 204, 301, 302, 400, 401, 403, 404, 408, 410, 413, - 422, 429, 460, 499, 500, 501, 502, 503, 504, 505, 508, 511 }) + foreach (string key in overridesJson.Keys) { - string val = overridesJson.GetString(code.ToString()); + if (!int.TryParse(key, out int code) || code < 100 || code > 599) + continue; + string val = overridesJson.GetString(key); if (val == "retry" || val == "drop") result[code] = val; } diff --git a/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs b/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs index 611527a..a0dfe41 100644 --- a/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs +++ b/Analytics-CSharp/Segment/Analytics/Utilities/HTTPClient.cs @@ -229,7 +229,8 @@ private static string GetStatusCodeAction(int statusCode, Dictionary Date: Thu, 21 May 2026 11:50:54 -0400 Subject: [PATCH 4/4] Ignore e2e-cli/build/ output directory --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c5b6d2e..5765e8f 100644 --- a/.gitignore +++ b/.gitignore @@ -402,4 +402,7 @@ ASALocalRun/ .mfractor/ # Local History for Visual Studio -.localhistory/ \ No newline at end of file +.localhistory/ + +# e2e-cli build output +e2e-cli/build/ \ No newline at end of file