diff --git a/CLAUDE.md b/CLAUDE.md index 82470bb..dca2c9e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,7 +35,11 @@ MinimalWorker is a .NET library for simplified background worker registration on - `RunPeriodicBackgroundWorker(IHost, TimeSpan, Delegate)` - Runs after each interval - `RunCronBackgroundWorker(IHost, string, Delegate)` - Runs on cron schedule (UTC) -All return `IWorkerBuilder` for fluent `.WithName()` and `.WithErrorHandler()` configuration. +All return `IWorkerBuilder` for fluent configuration: +- `.WithName(string)` - Set worker name for logs/metrics +- `.WithErrorHandler(Action)` - Handle errors (worker continues) +- `.WithTimeout(TimeSpan)` - Cancel execution if it exceeds timeout +- `.WithRetry(int maxAttempts, TimeSpan? delay)` - Retry failed executions **Source Generator** (`src/MinimalWorker.Generators/`): - `WorkerGenerator.cs` - IIncrementalGenerator that scans invocations @@ -84,6 +88,14 @@ for (int i = 0; i < steps; i++) } ``` +### Timeout and Retry Behavior + +- **Timeout**: Throws `TimeoutException`, cancels the delegate's `CancellationToken` +- **Retry**: Only retries on exceptions (not timeouts or `OperationCanceledException`) +- **Combined**: Timeouts are NOT retried when using both `.WithTimeout()` and `.WithRetry()` +- Error handler is called only after all retries are exhausted +- Both timeout and retry delays respect `TimeProvider` for testability with `FakeTimeProvider` + ## Common Anti-patterns - **Continuous workers**: Run exactly once. Include your own `while` loop if you need repetition diff --git a/README.llm b/README.llm index 3e2ebdb..f018090 100644 --- a/README.llm +++ b/README.llm @@ -50,6 +50,8 @@ public interface IWorkerBuilder { IWorkerBuilder WithName(string name); IWorkerBuilder WithErrorHandler(Action handler); + IWorkerBuilder WithTimeout(TimeSpan timeout); + IWorkerBuilder WithRetry(int maxAttempts = 3, TimeSpan? delay = null); } ``` @@ -211,6 +213,73 @@ app.RunBackgroundWorker((IMissingService missing) => --- +## Timeout Configuration + +Use `.WithTimeout()` to automatically cancel long-running executions: + +```csharp +app.RunPeriodicBackgroundWorker(TimeSpan.FromMinutes(5), async (DataService data, CancellationToken ct) => +{ + await data.ProcessBatch(ct); // Cancelled if exceeds 4 minutes +}) +.WithTimeout(TimeSpan.FromMinutes(4)) +.WithErrorHandler(ex => +{ + if (ex is TimeoutException) Console.WriteLine("Timed out!"); +}); +``` + +**Behavior:** +- `TimeoutException` thrown when timeout exceeded +- CancellationToken passed to delegate is cancelled on timeout +- Timeouts are **NOT retried** when combined with `.WithRetry()` +- Works with all worker types + +--- + +## Retry Configuration + +Use `.WithRetry()` to automatically retry failed executions: + +```csharp +app.RunPeriodicBackgroundWorker(TimeSpan.FromMinutes(5), async (ApiClient api, CancellationToken ct) => +{ + await api.SendData(ct); // Retries up to 3 times +}) +.WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(5)) +.WithErrorHandler(ex => +{ + // Called only after ALL retries exhausted + Console.WriteLine($"All retries failed: {ex.Message}"); +}); +``` + +**Parameters:** +| Parameter | Default | Description | +|-----------|---------|-------------| +| `maxAttempts` | 3 | Maximum execution attempts | +| `delay` | 5 seconds | Wait time between retries | + +**Behavior:** +- Error handler only called after all retries exhausted +- `OperationCanceledException` (shutdown) is never retried +- Timeouts are **NOT retried** + +### Combining Timeout and Retry + +```csharp +app.RunPeriodicBackgroundWorker(TimeSpan.FromMinutes(10), async (SyncService sync, CancellationToken ct) => +{ + await sync.SyncData(ct); +}) +.WithTimeout(TimeSpan.FromMinutes(2)) // Each attempt times out after 2 min +.WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(30)) // Retry failures, not timeouts +.WithName("data-sync") +.WithErrorHandler(ex => logger.LogError(ex, "Sync failed")); +``` + +--- + ## Testing ### Required Package @@ -537,7 +606,9 @@ host.RunPeriodicBackgroundWorker( { await notifications.CleanupExpiredAsync(ct); }) -.WithName("notification-cleanup"); +.WithName("notification-cleanup") +.WithTimeout(TimeSpan.FromMinutes(4)) +.WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(10)); // Cron worker - runs on schedule (UTC), new scope per execution host.RunCronBackgroundWorker( diff --git a/README.md b/README.md index 17224fa..e0af293 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ - ๐Ÿงผ Minimal and clean API - ๐Ÿ“ˆ Built-in telemetry with automatic metrics and distributed tracing - ๐ŸŽ๏ธ AOT Compilation Support +- โฐ Configurable execution timeouts +- ๐Ÿ” Automatic retry with configurable attempts and delays --- @@ -154,6 +156,68 @@ app.RunBackgroundWorker(async (CancellationToken token) => **Note**: This captures singleton services. For scoped services, this approach has limitations. Native DI support for error handlers is being considered for a future release. +### Timeout Configuration + +Use `.WithTimeout()` to automatically cancel long-running worker executions: + +```csharp +app.RunPeriodicBackgroundWorker(TimeSpan.FromMinutes(5), async (DataService data, CancellationToken token) => +{ + await data.ProcessBatch(token); // Will be cancelled if takes > 4 minutes +}) +.WithTimeout(TimeSpan.FromMinutes(4)) +.WithErrorHandler(ex => +{ + if (ex is TimeoutException) + { + Console.WriteLine("Processing timed out!"); + } +}); +``` + +**Behavior**: +- A `TimeoutException` is thrown when the timeout is exceeded +- The `CancellationToken` passed to your delegate is cancelled on timeout +- Timeouts are **not retried** (if using `.WithRetry()`) +- Works with all worker types: continuous, periodic, and cron + +### Retry Configuration + +Use `.WithRetry()` to automatically retry failed worker executions: + +```csharp +app.RunPeriodicBackgroundWorker(TimeSpan.FromMinutes(5), async (ApiClient api, CancellationToken token) => +{ + await api.SendData(token); // Will retry up to 3 times on failure +}) +.WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(5)) +.WithErrorHandler(ex => +{ + // Called only after all retries are exhausted + Console.WriteLine($"All retries failed: {ex.Message}"); +}); +``` + +**Behavior**: +- `maxAttempts` - Maximum number of execution attempts (default: 3) +- `delay` - Time to wait between retry attempts (default: 5 seconds) +- Error handler is only called after all retries are exhausted +- `OperationCanceledException` (graceful shutdown) is never retried +- Timeouts are **not retried** when combined with `.WithTimeout()` + +### Combining Timeout and Retry + +```csharp +app.RunPeriodicBackgroundWorker(TimeSpan.FromMinutes(10), async (SyncService sync, CancellationToken token) => +{ + await sync.SyncData(token); +}) +.WithTimeout(TimeSpan.FromMinutes(2)) // Each attempt times out after 2 minutes +.WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(30)) // Retry regular failures, not timeouts +.WithName("data-sync") +.WithErrorHandler(ex => logger.LogError(ex, "Sync failed")); +``` + #### Startup Dependency Validation MinimalWorker validates that all required dependencies for your workers are registered **during application startup**. If any dependencies are missing, the application will fail immediately with a clear error message: diff --git a/src/MinimalWorker.Generators/WorkerEmitter.cs b/src/MinimalWorker.Generators/WorkerEmitter.cs index 5e576b7..e2114e2 100644 --- a/src/MinimalWorker.Generators/WorkerEmitter.cs +++ b/src/MinimalWorker.Generators/WorkerEmitter.cs @@ -10,12 +10,53 @@ namespace MinimalWorker.Generators; /// internal static class WorkerEmitter { + /// + /// Emits shared infrastructure code that doesn't depend on worker analysis. + /// This is generated via RegisterPostInitializationOutput for better incrementality. + /// + public static string EmitSharedCode() + { + var sb = new StringBuilder(); + + sb.AppendLine("// "); + sb.AppendLine("// MinimalWorker Source Generator - Shared Infrastructure"); + sb.AppendLine("// This file contains shared code that doesn't depend on worker analysis."); + sb.AppendLine("#nullable enable"); + sb.AppendLine(); + sb.AppendLine("using System;"); + sb.AppendLine("using System.Diagnostics;"); + sb.AppendLine("using System.Diagnostics.Metrics;"); + sb.AppendLine("using System.Linq;"); + sb.AppendLine("using System.Threading;"); + sb.AppendLine("using Microsoft.Extensions.Logging;"); + sb.AppendLine(); + sb.AppendLine("namespace MinimalWorker;"); + sb.AppendLine(); + + // Emit ActivityExtensions + EmitActivityExtensions(sb); + + // Emit MinimalWorkerObservability + EmitObservability(sb); + + // Emit WorkerLogMessages + EmitWorkerLogMessages(sb); + + // Emit TimeProvider extensions for timeout/retry support + EmitTimeProviderExtensions(sb); + + return sb.ToString(); + } + + /// + /// Emits worker-specific code based on analyzed worker invocations. + /// public static string EmitSource(List workers) { var sb = new StringBuilder(); - + sb.AppendLine("// "); - sb.AppendLine("// MinimalWorker Source Generator"); + sb.AppendLine("// MinimalWorker Source Generator - Worker Initializers"); sb.AppendLine("// AOT-compatible background worker registration"); sb.AppendLine("#nullable enable"); sb.AppendLine(); @@ -31,6 +72,15 @@ public static string EmitSource(List workers) sb.AppendLine(); sb.AppendLine("namespace MinimalWorker;"); sb.AppendLine(); + + // Emit the extension method that wires up all workers + EmitWorkerExtension(sb, workers); + + return sb.ToString(); + } + + private static void EmitActivityExtensions(StringBuilder sb) + { sb.AppendLine("/// "); sb.AppendLine("/// Extension methods for Activity to support exception recording."); sb.AppendLine("/// "); @@ -56,6 +106,10 @@ public static string EmitSource(List workers) sb.AppendLine(" }"); sb.AppendLine("}"); sb.AppendLine(); + } + + private static void EmitObservability(StringBuilder sb) + { sb.AppendLine("/// "); sb.AppendLine("/// Observability instruments for MinimalWorker."); sb.AppendLine("/// ActivitySource and Meter are automatically consumed by OpenTelemetry when configured."); @@ -257,14 +311,6 @@ public static string EmitSource(List workers) sb.AppendLine(" }"); sb.AppendLine("}"); sb.AppendLine(); - - // Emit high-performance logging messages class - EmitWorkerLogMessages(sb); - - // Emit the extension method that wires up all workers - EmitWorkerExtension(sb, workers); - - return sb.ToString(); } private static void EmitWorkerLogMessages(StringBuilder sb) @@ -310,6 +356,40 @@ private static void EmitWorkerLogMessages(StringBuilder sb) sb.AppendLine(); } + private static void EmitTimeProviderExtensions(StringBuilder sb) + { + sb.AppendLine("/// "); + sb.AppendLine("/// Extension methods for TimeProvider to support delay and timeout operations."); + sb.AppendLine("/// "); + sb.AppendLine("internal static class TimeProviderExtensions"); + sb.AppendLine("{"); + sb.AppendLine(" /// "); + sb.AppendLine(" /// Creates a delay that respects the TimeProvider (works with FakeTimeProvider in tests)."); + sb.AppendLine(" /// "); + sb.AppendLine(" internal static async Task Delay(this TimeProvider timeProvider, TimeSpan delay, CancellationToken cancellationToken = default)"); + sb.AppendLine(" {"); + sb.AppendLine(" if (delay <= TimeSpan.Zero) return;"); + sb.AppendLine(" "); + sb.AppendLine(" var tcs = new TaskCompletionSource();"); + sb.AppendLine(" using var timer = timeProvider.CreateTimer(_ => tcs.TrySetResult(true), null, delay, Timeout.InfiniteTimeSpan);"); + sb.AppendLine(" using var registration = cancellationToken.Register(() => tcs.TrySetCanceled(cancellationToken));"); + sb.AppendLine(" await tcs.Task.ConfigureAwait(false);"); + sb.AppendLine(" }"); + sb.AppendLine(); + sb.AppendLine(" /// "); + sb.AppendLine(" /// Creates a CancellationTokenSource that cancels after the specified timeout using the TimeProvider."); + sb.AppendLine(" /// The returned CTS is also linked to the parent token for graceful shutdown support."); + sb.AppendLine(" /// "); + sb.AppendLine(" internal static (CancellationTokenSource Cts, ITimer Timer) CreateTimeoutCts(this TimeProvider timeProvider, TimeSpan timeout, CancellationToken parentToken)"); + sb.AppendLine(" {"); + sb.AppendLine(" var cts = CancellationTokenSource.CreateLinkedTokenSource(parentToken);"); + sb.AppendLine(" var timer = timeProvider.CreateTimer(_ => cts.Cancel(), null, timeout, Timeout.InfiniteTimeSpan);"); + sb.AppendLine(" return (cts, timer);"); + sb.AppendLine(" }"); + sb.AppendLine("}"); + sb.AppendLine(); + } + private static void EmitWorkerExtension(StringBuilder sb, List workers) { // Build worker map first to determine signatures @@ -422,9 +502,13 @@ private static void EmitContinuousWorkerInit(StringBuilder sb, WorkerInvocationM sb.AppendLine(" var token = lifetime.ApplicationStopping;"); sb.AppendLine(" var workerId = registration.Id.ToString();"); sb.AppendLine(" var workerName = registration.DisplayName;"); + sb.AppendLine(" var timeout = registration.Timeout;"); + sb.AppendLine(" var retryMaxAttempts = registration.RetryMaxAttempts ?? 1;"); + sb.AppendLine(" var retryDelay = registration.RetryDelay ?? TimeSpan.Zero;"); sb.AppendLine(); sb.AppendLine(" // Cache service lookups at initialization (outside Task.Run)"); sb.AppendLine(" var workerLogger = host.Services.GetService()?.CreateLogger($\"MinimalWorker.{workerName}\");"); + sb.AppendLine(" var timeProvider = host.Services.GetService() ?? TimeProvider.System;"); sb.AppendLine(); sb.AppendLine(" // Register worker for status tracking"); sb.AppendLine($" MinimalWorkerObservability.RegisterWorker(workerId, workerName, \"{workerType}\");"); @@ -437,6 +521,7 @@ private static void EmitContinuousWorkerInit(StringBuilder sb, WorkerInvocationM sb.AppendLine(" _ = Task.Run(async () =>"); sb.AppendLine(" {"); sb.AppendLine(" using var scope = host.Services.CreateScope();"); + sb.AppendLine(); // Resolve DI parameters EmitParameterResolution(sb, worker, " ", "scope"); @@ -447,32 +532,90 @@ private static void EmitContinuousWorkerInit(StringBuilder sb, WorkerInvocationM EmitActivitySetup(sb, workerType, " "); sb.AppendLine(); sb.AppendLine(" var stopwatch = Stopwatch.StartNew();"); + sb.AppendLine(" Exception? lastException = null;"); + sb.AppendLine(" var succeeded = false;"); sb.AppendLine(); - sb.AppendLine(" try"); + sb.AppendLine(" // Retry loop"); + sb.AppendLine(" for (var attempt = 1; attempt <= retryMaxAttempts && !token.IsCancellationRequested; attempt++)"); sb.AppendLine(" {"); + sb.AppendLine(" CancellationTokenSource? timeoutCts = null;"); + sb.AppendLine(" ITimer? timeoutTimer = null;"); + sb.AppendLine(" try"); + sb.AppendLine(" {"); + sb.AppendLine(" // Create execution token with optional timeout (uses TimeProvider for testability)"); + sb.AppendLine(" if (timeout.HasValue)"); + sb.AppendLine(" {"); + sb.AppendLine(" (timeoutCts, timeoutTimer) = timeProvider.CreateTimeoutCts(timeout.Value, token);"); + sb.AppendLine(" }"); + sb.AppendLine(" var executionToken = timeoutCts?.Token ?? token;"); + sb.AppendLine(); - // Invoke the delegate - EmitDelegateInvocation(sb, worker, delegateType, " "); + // Note: For continuous workers, we need to substitute the token in the delegate parameters + // We'll emit a local token variable that references executionToken + EmitContinuousWorkerDelegateInvocation(sb, worker, delegateType, " "); // Record success - EmitSuccessRecording(sb, " "); + EmitSuccessRecording(sb, " "); + sb.AppendLine(" succeeded = true;"); + sb.AppendLine(" break; // Success - exit retry loop"); + sb.AppendLine(" }"); + sb.AppendLine(" catch (OperationCanceledException) when (token.IsCancellationRequested)"); + sb.AppendLine(" {"); + sb.AppendLine(" // Graceful shutdown - exit without error, don't retry"); + sb.AppendLine(" activity?.SetStatus(ActivityStatusCode.Ok);"); + sb.AppendLine(" succeeded = true; // Mark as success to avoid error handling"); + sb.AppendLine(" break;"); + sb.AppendLine(" }"); + sb.AppendLine(" catch (OperationCanceledException) when (timeout.HasValue)"); + sb.AppendLine(" {"); + sb.AppendLine(" // Timeout - don't retry timeouts"); + sb.AppendLine(" lastException = new TimeoutException($\"Worker '{workerName}' execution timed out after {timeout.Value}.\");"); + sb.AppendLine(" break;"); + sb.AppendLine(" }"); + sb.AppendLine(" catch (Exception ex)"); + sb.AppendLine(" {"); + sb.AppendLine(" lastException = ex;"); + sb.AppendLine(" if (attempt < retryMaxAttempts && retryDelay > TimeSpan.Zero && !token.IsCancellationRequested)"); + sb.AppendLine(" {"); + sb.AppendLine(" // Wait before retry (uses TimeProvider for testability)"); + sb.AppendLine(" try { await timeProvider.Delay(retryDelay, token); } catch (OperationCanceledException) { break; }"); + sb.AppendLine(" }"); + sb.AppendLine(" }"); + sb.AppendLine(" finally"); + sb.AppendLine(" {"); + sb.AppendLine(" timeoutTimer?.Dispose();"); + sb.AppendLine(" timeoutCts?.Dispose();"); + sb.AppendLine(" }"); sb.AppendLine(" }"); - sb.AppendLine(" catch (OperationCanceledException)"); - sb.AppendLine(" {"); - sb.AppendLine(" // Graceful shutdown - exit without error"); - sb.AppendLine(" activity?.SetStatus(ActivityStatusCode.Ok);"); - sb.AppendLine(" }"); - sb.AppendLine(" catch (Exception ex)"); - sb.AppendLine(" {"); - EmitErrorHandling(sb, " "); - sb.AppendLine(" }"); - sb.AppendLine(" finally"); + sb.AppendLine(); + sb.AppendLine(" // Handle failure after all retries exhausted"); + sb.AppendLine(" if (!succeeded && lastException != null)"); sb.AppendLine(" {"); - EmitDurationRecording(sb, " "); - sb.AppendLine(" // Deactivate worker when it completes"); - EmitWorkerDeactivation(sb, workerType, " "); + sb.AppendLine(" activity?.SetStatus(ActivityStatusCode.Error, lastException.Message);"); + sb.AppendLine(" activity?.RecordException(lastException);"); + sb.AppendLine(); + sb.AppendLine(" var errorTags = tags;"); + sb.AppendLine(" errorTags.Add(\"exception.type\", lastException.GetType().FullName);"); + sb.AppendLine(" MinimalWorkerObservability.ErrorCounter.Add(1, errorTags);"); + sb.AppendLine(" MinimalWorkerObservability.RecordFailure(workerId);"); + sb.AppendLine(); + sb.AppendLine(" if (workerLogger != null) WorkerLogMessages.WorkerExecutionFailed(workerLogger, workerName, lastException);"); + sb.AppendLine(); + sb.AppendLine(" if (registration.OnError != null)"); + sb.AppendLine(" {"); + sb.AppendLine(" registration.OnError(lastException);"); + sb.AppendLine(" }"); + sb.AppendLine(" else"); + sb.AppendLine(" {"); + sb.AppendLine(" if (workerLogger != null) WorkerLogMessages.WorkerFatalError(workerLogger, workerName, lastException);"); + sb.AppendLine(" BackgroundWorkerExtensions.TerminateOnFatalError(lastException);"); + sb.AppendLine(" }"); sb.AppendLine(" }"); + sb.AppendLine(); + EmitDurationRecording(sb, " "); + sb.AppendLine(" // Deactivate worker when it completes"); + EmitWorkerDeactivation(sb, workerType, " "); sb.AppendLine(" }, token);"); } @@ -489,6 +632,9 @@ private static void EmitPeriodicWorkerInit(StringBuilder sb, WorkerInvocationMod sb.AppendLine(" var workerId = registration.Id.ToString();"); sb.AppendLine(" var workerName = registration.DisplayName;"); sb.AppendLine(" var token = lifetime.ApplicationStopping;"); + sb.AppendLine(" var timeout = registration.Timeout;"); + sb.AppendLine(" var retryMaxAttempts = registration.RetryMaxAttempts ?? 1;"); + sb.AppendLine(" var retryDelay = registration.RetryDelay ?? TimeSpan.Zero;"); sb.AppendLine(); sb.AppendLine(" // Cache service lookups at initialization (outside Task.Run)"); sb.AppendLine(" var workerLogger = host.Services.GetService()?.CreateLogger($\"MinimalWorker.{workerName}\");"); @@ -516,37 +662,97 @@ private static void EmitPeriodicWorkerInit(StringBuilder sb, WorkerInvocationMod EmitActivitySetup(sb, workerType, " ", new[] { "activity?.SetTag(\"worker.schedule\", scheduleString);" }); sb.AppendLine(); sb.AppendLine(" var stopwatch = Stopwatch.StartNew();"); + sb.AppendLine(" Exception? lastException = null;"); + sb.AppendLine(" var succeeded = false;"); sb.AppendLine(); - sb.AppendLine(" try"); + sb.AppendLine(" // Retry loop"); + sb.AppendLine(" for (var attempt = 1; attempt <= retryMaxAttempts && !token.IsCancellationRequested; attempt++)"); sb.AppendLine(" {"); - sb.AppendLine(" using var scope = host.Services.CreateScope();"); + sb.AppendLine(" CancellationTokenSource? timeoutCts = null;"); + sb.AppendLine(" ITimer? timeoutTimer = null;"); + sb.AppendLine(" try"); + sb.AppendLine(" {"); + sb.AppendLine(" using var scope = host.Services.CreateScope();"); + sb.AppendLine(); + sb.AppendLine(" // Create execution token with optional timeout (uses TimeProvider for testability)"); + sb.AppendLine(" if (timeout.HasValue)"); + sb.AppendLine(" {"); + sb.AppendLine(" (timeoutCts, timeoutTimer) = timeProvider.CreateTimeoutCts(timeout.Value, token);"); + sb.AppendLine(" }"); + sb.AppendLine(" var executionToken = timeoutCts?.Token ?? token;"); + sb.AppendLine(); - // Resolve DI parameters - EmitParameterResolution(sb, worker, " ", "scope"); + // Resolve DI parameters - use executionToken for CancellationToken + EmitParameterResolutionWithExecutionToken(sb, worker, " ", "scope"); sb.AppendLine(); // Invoke the delegate - EmitDelegateInvocation(sb, worker, delegateType, " "); + EmitDelegateInvocation(sb, worker, delegateType, " "); // Record success - EmitSuccessRecording(sb, " "); + EmitSuccessRecording(sb, " "); + sb.AppendLine(" succeeded = true;"); + sb.AppendLine(" break; // Success - exit retry loop"); + sb.AppendLine(" }"); + sb.AppendLine(" catch (OperationCanceledException) when (token.IsCancellationRequested)"); + sb.AppendLine(" {"); + sb.AppendLine(" // Graceful shutdown - exit without error, don't retry"); + sb.AppendLine(" activity?.SetStatus(ActivityStatusCode.Ok);"); + sb.AppendLine(" succeeded = true; // Mark as success to avoid error handling"); + sb.AppendLine(" break;"); + sb.AppendLine(" }"); + sb.AppendLine(" catch (OperationCanceledException) when (timeout.HasValue)"); + sb.AppendLine(" {"); + sb.AppendLine(" // Timeout - don't retry timeouts"); + sb.AppendLine(" lastException = new TimeoutException($\"Worker '{workerName}' execution timed out after {timeout.Value}.\");"); + sb.AppendLine(" break;"); + sb.AppendLine(" }"); + sb.AppendLine(" catch (Exception ex)"); + sb.AppendLine(" {"); + sb.AppendLine(" lastException = ex;"); + sb.AppendLine(" if (attempt < retryMaxAttempts && retryDelay > TimeSpan.Zero && !token.IsCancellationRequested)"); + sb.AppendLine(" {"); + sb.AppendLine(" // Wait before retry (uses TimeProvider for testability)"); + sb.AppendLine(" try { await timeProvider.Delay(retryDelay, token); } catch (OperationCanceledException) { break; }"); + sb.AppendLine(" }"); + sb.AppendLine(" }"); + sb.AppendLine(" finally"); + sb.AppendLine(" {"); + sb.AppendLine(" timeoutTimer?.Dispose();"); + sb.AppendLine(" timeoutCts?.Dispose();"); + sb.AppendLine(" }"); sb.AppendLine(" }"); - sb.AppendLine(" catch (OperationCanceledException)"); - sb.AppendLine(" {"); - sb.AppendLine(" // Graceful shutdown - exit without error"); - sb.AppendLine(" activity?.SetStatus(ActivityStatusCode.Ok);"); - sb.AppendLine(" break;"); - sb.AppendLine(" }"); - sb.AppendLine(" catch (Exception ex)"); - sb.AppendLine(" {"); - EmitErrorHandling(sb, " "); - sb.AppendLine(" }"); - sb.AppendLine(" finally"); + sb.AppendLine(); + sb.AppendLine(" // Handle failure after all retries exhausted"); + sb.AppendLine(" if (!succeeded && lastException != null)"); sb.AppendLine(" {"); - EmitDurationRecording(sb, " "); + sb.AppendLine(" activity?.SetStatus(ActivityStatusCode.Error, lastException.Message);"); + sb.AppendLine(" activity?.RecordException(lastException);"); + sb.AppendLine(); + sb.AppendLine(" var errorTags = tags;"); + sb.AppendLine(" errorTags.Add(\"exception.type\", lastException.GetType().FullName);"); + sb.AppendLine(" MinimalWorkerObservability.ErrorCounter.Add(1, errorTags);"); + sb.AppendLine(" MinimalWorkerObservability.RecordFailure(workerId);"); + sb.AppendLine(); + sb.AppendLine(" if (workerLogger != null) WorkerLogMessages.WorkerExecutionFailed(workerLogger, workerName, lastException);"); + sb.AppendLine(); + sb.AppendLine(" if (registration.OnError != null)"); + sb.AppendLine(" {"); + sb.AppendLine(" registration.OnError(lastException);"); + sb.AppendLine(" }"); + sb.AppendLine(" else"); + sb.AppendLine(" {"); + sb.AppendLine(" if (workerLogger != null) WorkerLogMessages.WorkerFatalError(workerLogger, workerName, lastException);"); + sb.AppendLine(" BackgroundWorkerExtensions.TerminateOnFatalError(lastException);"); + sb.AppendLine(" }"); sb.AppendLine(" }"); + sb.AppendLine(); + sb.AppendLine(" // Check for graceful shutdown after retry loop"); + sb.AppendLine(" if (token.IsCancellationRequested) break;"); + sb.AppendLine(); + EmitDurationRecording(sb, " "); sb.AppendLine(" }"); sb.AppendLine(" }"); sb.AppendLine(" catch (OperationCanceledException)"); @@ -574,6 +780,9 @@ private static void EmitCronWorkerInit(StringBuilder sb, WorkerInvocationModel w sb.AppendLine(" var workerId = registration.Id.ToString();"); sb.AppendLine(" var workerName = registration.DisplayName;"); sb.AppendLine(" var token = lifetime.ApplicationStopping;"); + sb.AppendLine(" var timeout = registration.Timeout;"); + sb.AppendLine(" var retryMaxAttempts = registration.RetryMaxAttempts ?? 1;"); + sb.AppendLine(" var retryDelay = registration.RetryDelay ?? TimeSpan.Zero;"); sb.AppendLine(); sb.AppendLine(" // Cache service lookups at initialization (outside Task.Run)"); sb.AppendLine(" var workerLogger = host.Services.GetService()?.CreateLogger($\"MinimalWorker.{workerName}\");"); @@ -615,37 +824,97 @@ private static void EmitCronWorkerInit(StringBuilder sb, WorkerInvocationModel w }); sb.AppendLine(); sb.AppendLine(" var stopwatch = Stopwatch.StartNew();"); + sb.AppendLine(" Exception? lastException = null;"); + sb.AppendLine(" var succeeded = false;"); sb.AppendLine(); - sb.AppendLine(" try"); + sb.AppendLine(" // Retry loop"); + sb.AppendLine(" for (var attempt = 1; attempt <= retryMaxAttempts && !token.IsCancellationRequested; attempt++)"); sb.AppendLine(" {"); - sb.AppendLine(" using var scope = host.Services.CreateScope();"); + sb.AppendLine(" CancellationTokenSource? timeoutCts = null;"); + sb.AppendLine(" ITimer? timeoutTimer = null;"); + sb.AppendLine(" try"); + sb.AppendLine(" {"); + sb.AppendLine(" using var scope = host.Services.CreateScope();"); + sb.AppendLine(); + sb.AppendLine(" // Create execution token with optional timeout (uses TimeProvider for testability)"); + sb.AppendLine(" if (timeout.HasValue)"); + sb.AppendLine(" {"); + sb.AppendLine(" (timeoutCts, timeoutTimer) = timeProvider.CreateTimeoutCts(timeout.Value, token);"); + sb.AppendLine(" }"); + sb.AppendLine(" var executionToken = timeoutCts?.Token ?? token;"); + sb.AppendLine(); - // Resolve DI parameters - EmitParameterResolution(sb, worker, " ", "scope"); + // Resolve DI parameters - use executionToken for CancellationToken + EmitParameterResolutionWithExecutionToken(sb, worker, " ", "scope"); sb.AppendLine(); // Invoke the delegate - EmitDelegateInvocation(sb, worker, delegateType, " "); + EmitDelegateInvocation(sb, worker, delegateType, " "); // Record success - EmitSuccessRecording(sb, " "); + EmitSuccessRecording(sb, " "); + sb.AppendLine(" succeeded = true;"); + sb.AppendLine(" break; // Success - exit retry loop"); + sb.AppendLine(" }"); + sb.AppendLine(" catch (OperationCanceledException) when (token.IsCancellationRequested)"); + sb.AppendLine(" {"); + sb.AppendLine(" // Graceful shutdown - exit without error, don't retry"); + sb.AppendLine(" activity?.SetStatus(ActivityStatusCode.Ok);"); + sb.AppendLine(" succeeded = true; // Mark as success to avoid error handling"); + sb.AppendLine(" break;"); + sb.AppendLine(" }"); + sb.AppendLine(" catch (OperationCanceledException) when (timeout.HasValue)"); + sb.AppendLine(" {"); + sb.AppendLine(" // Timeout - don't retry timeouts"); + sb.AppendLine(" lastException = new TimeoutException($\"Worker '{workerName}' execution timed out after {timeout.Value}.\");"); + sb.AppendLine(" break;"); + sb.AppendLine(" }"); + sb.AppendLine(" catch (Exception ex)"); + sb.AppendLine(" {"); + sb.AppendLine(" lastException = ex;"); + sb.AppendLine(" if (attempt < retryMaxAttempts && retryDelay > TimeSpan.Zero && !token.IsCancellationRequested)"); + sb.AppendLine(" {"); + sb.AppendLine(" // Wait before retry (uses TimeProvider for testability)"); + sb.AppendLine(" try { await timeProvider.Delay(retryDelay, token); } catch (OperationCanceledException) { break; }"); + sb.AppendLine(" }"); + sb.AppendLine(" }"); + sb.AppendLine(" finally"); + sb.AppendLine(" {"); + sb.AppendLine(" timeoutTimer?.Dispose();"); + sb.AppendLine(" timeoutCts?.Dispose();"); + sb.AppendLine(" }"); sb.AppendLine(" }"); - sb.AppendLine(" catch (OperationCanceledException)"); - sb.AppendLine(" {"); - sb.AppendLine(" // Graceful shutdown - exit without error"); - sb.AppendLine(" activity?.SetStatus(ActivityStatusCode.Ok);"); - sb.AppendLine(" break;"); - sb.AppendLine(" }"); - sb.AppendLine(" catch (Exception ex)"); - sb.AppendLine(" {"); - EmitErrorHandling(sb, " "); - sb.AppendLine(" }"); - sb.AppendLine(" finally"); + sb.AppendLine(); + sb.AppendLine(" // Handle failure after all retries exhausted"); + sb.AppendLine(" if (!succeeded && lastException != null)"); sb.AppendLine(" {"); - EmitDurationRecording(sb, " "); + sb.AppendLine(" activity?.SetStatus(ActivityStatusCode.Error, lastException.Message);"); + sb.AppendLine(" activity?.RecordException(lastException);"); + sb.AppendLine(); + sb.AppendLine(" var errorTags = tags;"); + sb.AppendLine(" errorTags.Add(\"exception.type\", lastException.GetType().FullName);"); + sb.AppendLine(" MinimalWorkerObservability.ErrorCounter.Add(1, errorTags);"); + sb.AppendLine(" MinimalWorkerObservability.RecordFailure(workerId);"); + sb.AppendLine(); + sb.AppendLine(" if (workerLogger != null) WorkerLogMessages.WorkerExecutionFailed(workerLogger, workerName, lastException);"); + sb.AppendLine(); + sb.AppendLine(" if (registration.OnError != null)"); + sb.AppendLine(" {"); + sb.AppendLine(" registration.OnError(lastException);"); + sb.AppendLine(" }"); + sb.AppendLine(" else"); + sb.AppendLine(" {"); + sb.AppendLine(" if (workerLogger != null) WorkerLogMessages.WorkerFatalError(workerLogger, workerName, lastException);"); + sb.AppendLine(" BackgroundWorkerExtensions.TerminateOnFatalError(lastException);"); + sb.AppendLine(" }"); sb.AppendLine(" }"); + sb.AppendLine(); + sb.AppendLine(" // Check for graceful shutdown after retry loop"); + sb.AppendLine(" if (token.IsCancellationRequested) break;"); + sb.AppendLine(); + EmitDurationRecording(sb, " "); sb.AppendLine(" }"); sb.AppendLine(" }"); sb.AppendLine(" catch (OperationCanceledException)"); @@ -722,6 +991,25 @@ private static void EmitParameterResolution(StringBuilder sb, WorkerInvocationMo } } + /// + /// Emits DI parameter resolution code using executionToken for CancellationToken parameters. + /// Used when timeout support is needed. + /// + private static void EmitParameterResolutionWithExecutionToken(StringBuilder sb, WorkerInvocationModel worker, string indent, string scopeVar) + { + foreach (var param in worker.Parameters) + { + if (param.IsCancellationToken) + { + sb.AppendLine($"{indent}var {param.Name} = executionToken;"); + } + else + { + sb.AppendLine($"{indent}var {param.Name} = {scopeVar}.ServiceProvider.GetRequiredService<{param.Type}>();"); + } + } + } + /// /// Emits delegate invocation code. /// @@ -739,6 +1027,26 @@ private static void EmitDelegateInvocation(StringBuilder sb, WorkerInvocationMod } } + /// + /// Emits delegate invocation code for continuous workers, using executionToken for CancellationToken parameters. + /// + private static void EmitContinuousWorkerDelegateInvocation(StringBuilder sb, WorkerInvocationModel worker, string delegateType, string indent) + { + // For continuous workers, parameters are resolved outside the retry loop, + // but we need to pass executionToken instead of the resolved token parameter + var paramNames = string.Join(", ", worker.Parameters.Select(p => + p.IsCancellationToken ? "executionToken" : p.Name)); + + if (worker.IsAsync) + { + sb.AppendLine($"{indent}await (({delegateType})registration.Action)({paramNames});"); + } + else + { + sb.AppendLine($"{indent}(({delegateType})registration.Action)({paramNames});"); + } + } + /// /// Emits success recording code (execution counter, success tracking, activity status). /// diff --git a/src/MinimalWorker.Generators/WorkerGenerator.cs b/src/MinimalWorker.Generators/WorkerGenerator.cs index b1a5aa5..48577e4 100644 --- a/src/MinimalWorker.Generators/WorkerGenerator.cs +++ b/src/MinimalWorker.Generators/WorkerGenerator.cs @@ -18,6 +18,13 @@ public class WorkerGenerator : IIncrementalGenerator { public void Initialize(IncrementalGeneratorInitializationContext context) { + // Register shared code that doesn't depend on worker analysis (emitted once) + context.RegisterPostInitializationOutput(static ctx => + { + ctx.AddSource("MinimalWorker.Shared.g.cs", + SourceText.From(WorkerEmitter.EmitSharedCode(), Encoding.UTF8)); + }); + // Find all invocations of RunBackgroundWorker, RunPeriodicBackgroundWorker, and RunCronBackgroundWorker var workerInvocations = context.SyntaxProvider .CreateSyntaxProvider( @@ -25,8 +32,8 @@ public void Initialize(IncrementalGeneratorInitializationContext context) transform: static (ctx, _) => GetWorkerInvocation(ctx)) .Where(static m => m is not null); - // Combine and generate - context.RegisterSourceOutput(workerInvocations.Collect(), + // Combine and generate worker-specific code + context.RegisterSourceOutput(workerInvocations.Collect(), static (spc, workers) => Execute(spc, workers!)); } diff --git a/src/MinimalWorker/BackgroundWorkerExtensions.cs b/src/MinimalWorker/BackgroundWorkerExtensions.cs index 9b4f560..92725bb 100644 --- a/src/MinimalWorker/BackgroundWorkerExtensions.cs +++ b/src/MinimalWorker/BackgroundWorkerExtensions.cs @@ -73,6 +73,86 @@ public interface IWorkerBuilder /// /// IWorkerBuilder WithErrorHandler(Action handler); + + /// + /// Sets a timeout for each worker execution. If the execution exceeds this duration, it will be cancelled. + /// + /// The maximum duration for each execution. Must be greater than . + /// The builder instance for method chaining. + /// + /// + /// The timeout applies to each individual execution, not the worker's total lifetime. + /// + /// + /// When a timeout occurs: + /// + /// + /// The passed to the delegate is cancelled + /// A is raised (handled by error handler if configured) + /// Periodic/Cron workers: Continue running and will execute on next schedule + /// Continuous workers: Worker stops + /// + /// + /// Important: The delegate must respect the for timeout to work effectively. + /// + /// + /// + /// + /// // Data sync with 4 minute timeout (runs every 5 minutes) + /// host.RunPeriodicBackgroundWorker(TimeSpan.FromMinutes(5), async (api, token) => + /// { + /// await api.SyncDataAsync(token); + /// }) + /// .WithTimeout(TimeSpan.FromMinutes(4)) + /// .WithErrorHandler(ex => + /// { + /// if (ex is TimeoutException) + /// logger.LogWarning("Sync timed out, will retry next interval"); + /// else + /// logger.LogError(ex, "Sync failed"); + /// }); + /// + /// + IWorkerBuilder WithTimeout(TimeSpan timeout); + + /// + /// Configures automatic retry behavior for failed worker executions. + /// + /// The maximum number of retry attempts. Must be at least 1. Default is 3. + /// The delay between retry attempts. Must be greater than . Default is 5 seconds. + /// The builder instance for method chaining. + /// + /// + /// Retry behavior varies by worker type: + /// + /// + /// Periodic/Cron workers: Retries occur within the current execution window before moving to next scheduled run + /// Continuous workers: Retries continue until success or max attempts exhausted + /// + /// + /// After all retry attempts are exhausted: + /// + /// + /// The error handler is invoked (if configured) + /// Without an error handler, the application terminates (fail-fast) + /// Periodic/Cron workers: Will run again on the next scheduled interval + /// + /// + /// Note: Retries do not occur for (graceful shutdown) or (if timeout is configured). + /// + /// + /// + /// + /// // Retry API calls up to 5 times with 10 second delay + /// host.RunPeriodicBackgroundWorker(TimeSpan.FromMinutes(5), async (api, token) => + /// { + /// await api.SyncDataAsync(token); + /// }) + /// .WithRetry(maxAttempts: 5, delay: TimeSpan.FromSeconds(10)) + /// .WithErrorHandler(ex => logger.LogError(ex, "Sync failed after all retries")); + /// + /// + IWorkerBuilder WithRetry(int maxAttempts = 3, TimeSpan? delay = null); } /// @@ -98,6 +178,29 @@ public IWorkerBuilder WithErrorHandler(Action handler) _registration.OnError = handler; return this; } + + public IWorkerBuilder WithTimeout(TimeSpan timeout) + { + if (timeout <= TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(timeout), timeout, "Timeout must be greater than zero."); + + _registration.Timeout = timeout; + return this; + } + + public IWorkerBuilder WithRetry(int maxAttempts = 3, TimeSpan? delay = null) + { + if (maxAttempts < 1) + throw new ArgumentOutOfRangeException(nameof(maxAttempts), maxAttempts, "Max attempts must be at least 1."); + + var actualDelay = delay ?? TimeSpan.FromSeconds(5); + if (actualDelay <= TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(delay), delay, "Delay must be greater than zero."); + + _registration.RetryMaxAttempts = maxAttempts; + _registration.RetryDelay = actualDelay; + return this; + } } /// @@ -533,6 +636,27 @@ public class WorkerRegistration /// public Action? OnError { get; set; } + /// + /// Gets or sets the optional timeout for each worker execution. + /// Set via . + /// If null, no timeout is applied. + /// + public TimeSpan? Timeout { get; set; } + + /// + /// Gets or sets the maximum number of retry attempts on failure. + /// Set via . + /// If null, no automatic retries are performed. + /// + public int? RetryMaxAttempts { get; set; } + + /// + /// Gets or sets the delay between retry attempts. + /// Set via . + /// Only used when is configured. + /// + public TimeSpan? RetryDelay { get; set; } + /// /// Gets or sets the unique signature based on worker type and parameter types. /// Format: "{WorkerType}:{Param1Type},{Param2Type},...". diff --git a/test/MinimalWorker.Test/Helpers/WorkerTestHelper.cs b/test/MinimalWorker.Test/Helpers/WorkerTestHelper.cs index ac2a3c2..b07ed81 100644 --- a/test/MinimalWorker.Test/Helpers/WorkerTestHelper.cs +++ b/test/MinimalWorker.Test/Helpers/WorkerTestHelper.cs @@ -5,6 +5,25 @@ namespace MinimalWorker.Test.Helpers; +/// +/// Extension methods for TimeProvider to use in tests. +/// +public static class TimeProviderTestExtensions +{ + /// + /// Creates a delay that respects the TimeProvider (works with FakeTimeProvider in tests). + /// + public static async Task Delay(this TimeProvider timeProvider, TimeSpan delay, CancellationToken cancellationToken = default) + { + if (delay <= TimeSpan.Zero) return; + + var tcs = new TaskCompletionSource(); + using var timer = timeProvider.CreateTimer(_ => tcs.TrySetResult(true), null, delay, Timeout.InfiniteTimeSpan); + using var registration = cancellationToken.Register(() => tcs.TrySetCanceled(cancellationToken)); + await tcs.Task.ConfigureAwait(false); + } +} + /// /// Helper class for testing workers with FakeTimeProvider. /// Advances time automatically to trigger periodic and cron workers without real delays. diff --git a/test/MinimalWorker.Test/RetryTests.cs b/test/MinimalWorker.Test/RetryTests.cs new file mode 100644 index 0000000..839f509 --- /dev/null +++ b/test/MinimalWorker.Test/RetryTests.cs @@ -0,0 +1,347 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using MinimalWorker.Test.Helpers; + +namespace MinimalWorker.Test; + +public class RetryTests +{ + [Fact] + public async Task PeriodicWorker_WithRetry_Should_Retry_On_Failure() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var attemptCount = 0; + var errorHandlerCalled = false; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + (CancellationToken token) => + { + Interlocked.Increment(ref attemptCount); + if (attemptCount < 3) + { + throw new InvalidOperationException("Simulated failure"); + } + return Task.CompletedTask; + }) + .WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(10)) + .WithErrorHandler(ex => + { + errorHandlerCalled = true; + }); + + // Act + await host.StartAsync(); + // Advance time: 1 min for periodic tick, then enough for retries + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(2), steps: 24); + await host.StopAsync(); + + // Assert - Should succeed on 3rd attempt, error handler not called + Assert.True(attemptCount >= 3, $"Expected at least 3 attempts, got {attemptCount}"); + Assert.False(errorHandlerCalled, "Error handler should not be called when retry succeeds"); + } + + [Fact] + public async Task PeriodicWorker_WithRetry_Should_Call_ErrorHandler_After_All_Retries_Exhausted() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var attemptCount = 0; + var errorHandlerCalled = false; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + (CancellationToken token) => + { + Interlocked.Increment(ref attemptCount); + throw new InvalidOperationException("Always fails"); +#pragma warning disable CS0162 // Unreachable code detected + return Task.CompletedTask; +#pragma warning restore CS0162 + }) + .WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(10)) + .WithErrorHandler(ex => + { + errorHandlerCalled = true; + }); + + // Act + await host.StartAsync(); + // Advance time: 1 min for periodic tick, then enough for all retries + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(2), steps: 24); + await host.StopAsync(); + + // Assert - Should exhaust all retries and call error handler + Assert.True(attemptCount >= 3, $"Expected at least 3 attempts, got {attemptCount}"); + Assert.True(errorHandlerCalled, "Error handler should be called after all retries exhausted"); + } + + [Fact] + public void PeriodicWorker_WithRetry_Should_Throw_For_Zero_MaxAttempts() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + + using var host = Host.CreateDefaultBuilder().Build(); + + // Act & Assert + var exception = Assert.Throws(() => + { + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + (CancellationToken token) => Task.CompletedTask) + .WithRetry(maxAttempts: 0); + }); + + Assert.Equal("maxAttempts", exception.ParamName); + } + + [Fact] + public void PeriodicWorker_WithRetry_Should_Throw_For_Negative_Delay() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + + using var host = Host.CreateDefaultBuilder().Build(); + + // Act & Assert + var exception = Assert.Throws(() => + { + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + (CancellationToken token) => Task.CompletedTask) + .WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(-5)); + }); + + Assert.Equal("delay", exception.ParamName); + } + + [Fact] + public async Task PeriodicWorker_WithRetry_Should_Use_Configured_Attempts() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var attemptCount = 0; + var errorHandlerCalled = false; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + (CancellationToken token) => + { + Interlocked.Increment(ref attemptCount); + throw new InvalidOperationException("Always fails"); +#pragma warning disable CS0162 // Unreachable code detected + return Task.CompletedTask; +#pragma warning restore CS0162 + }) + .WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(10)) + .WithErrorHandler(ex => + { + errorHandlerCalled = true; + }); + + // Act + await host.StartAsync(); + // Advance time for periodic tick and retries + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(2), steps: 24); + await host.StopAsync(); + + // Assert - 3 attempts configured + Assert.True(attemptCount >= 3, $"Expected at least 3 attempts, got {attemptCount}"); + Assert.True(errorHandlerCalled, "Error handler should be called after retries exhausted"); + } + + [Fact] + public async Task CronWorker_WithRetry_Should_Retry_On_Failure() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var attemptCount = 0; + var errorHandlerCalled = false; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + // Every minute + host.RunCronBackgroundWorker( + "* * * * *", + (CancellationToken token) => + { + Interlocked.Increment(ref attemptCount); + if (attemptCount < 2) + { + throw new InvalidOperationException("Simulated failure"); + } + return Task.CompletedTask; + }) + .WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(10)) + .WithErrorHandler(ex => + { + errorHandlerCalled = true; + }); + + // Act + await host.StartAsync(); + // Advance time for cron trigger and retries + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(2), steps: 24); + await host.StopAsync(); + + // Assert - Should succeed on 2nd attempt + Assert.True(attemptCount >= 2, $"Expected at least 2 attempts, got {attemptCount}"); + Assert.False(errorHandlerCalled, "Error handler should not be called when retry succeeds"); + } + + [Fact] + public async Task ContinuousWorker_WithRetry_Should_Retry_On_Failure() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var attemptCount = 0; + var errorHandlerCalled = false; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + host.RunBackgroundWorker((CancellationToken token) => + { + attemptCount++; + if (attemptCount < 3) + { + throw new InvalidOperationException("Simulated failure"); + } + return Task.CompletedTask; + }) + .WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(10)) + .WithErrorHandler(ex => + { + errorHandlerCalled = true; + }); + + // Act + await host.StartAsync(); + // Advance time for retries (2 retries * 10s delay = 20s minimum) + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(1), steps: 12); + await host.StopAsync(); + + // Assert - Should succeed on 3rd attempt + Assert.True(attemptCount >= 3, $"Expected at least 3 attempts, got {attemptCount}"); + Assert.False(errorHandlerCalled, "Error handler should not be called when retry succeeds"); + } + + [Fact] + public async Task ContinuousWorker_WithRetry_Should_Call_ErrorHandler_After_All_Retries_Exhausted() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var attemptCount = 0; + var errorHandlerCalled = false; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + host.RunBackgroundWorker((CancellationToken token) => + { + Interlocked.Increment(ref attemptCount); + throw new InvalidOperationException("Always fails"); +#pragma warning disable CS0162 // Unreachable code detected + return Task.CompletedTask; +#pragma warning restore CS0162 + }) + .WithRetry(maxAttempts: 2, delay: TimeSpan.FromSeconds(10)) + .WithErrorHandler(ex => + { + errorHandlerCalled = true; + }); + + // Act + await host.StartAsync(); + // Advance time for retries + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(1), steps: 12); + await host.StopAsync(); + + // Assert + Assert.True(attemptCount >= 2, $"Expected at least 2 attempts, got {attemptCount}"); + Assert.True(errorHandlerCalled, "Error handler should be called after all retries exhausted"); + } + + [Fact] + public async Task PeriodicWorker_WithTimeoutAndRetry_Should_Not_Retry_Timeouts() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var attemptCount = 0; + Exception? caughtException = null; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + async (CancellationToken token) => + { + attemptCount++; + // Simulate long-running task that will timeout + await timeProvider.Delay(TimeSpan.FromMinutes(30), token); + }) + .WithTimeout(TimeSpan.FromMinutes(2)) + .WithRetry(maxAttempts: 3, delay: TimeSpan.FromSeconds(10)) + .WithErrorHandler(ex => + { + caughtException = ex; + }); + + // Act + await host.StartAsync(); + // Advance time: 1 min for periodic tick, then 2+ min for timeout + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(4), steps: 8); + await host.StopAsync(); + + // Assert - Timeout should not be retried (only 1 attempt per timeout) + Assert.True(caughtException is TimeoutException, $"Should have caught TimeoutException, got {caughtException?.GetType().Name}"); + } +} diff --git a/test/MinimalWorker.Test/TimeoutTests.cs b/test/MinimalWorker.Test/TimeoutTests.cs new file mode 100644 index 0000000..16a22d4 --- /dev/null +++ b/test/MinimalWorker.Test/TimeoutTests.cs @@ -0,0 +1,209 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using MinimalWorker.Test.Helpers; + +namespace MinimalWorker.Test; + +public class TimeoutTests +{ + [Fact] + public async Task PeriodicWorker_WithTimeout_Should_Cancel_Long_Running_Execution() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var timeoutOccurred = false; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + async (CancellationToken token) => + { + // This simulates a long-running task that exceeds the timeout + await timeProvider.Delay(TimeSpan.FromMinutes(30), token); + }) + .WithTimeout(TimeSpan.FromMinutes(2)) + .WithErrorHandler(ex => + { + if (ex is TimeoutException) + { + timeoutOccurred = true; + } + }); + + // Act + await host.StartAsync(); + // Advance time: 1 min for periodic tick, then 2+ min for timeout to trigger + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(4), steps: 8); + await host.StopAsync(); + + // Assert + Assert.True(timeoutOccurred, "Timeout should have occurred for long-running execution"); + } + + [Fact] + public async Task PeriodicWorker_WithTimeout_Should_Not_Timeout_Fast_Execution() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var executionCount = 0; + var errorCount = 0; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + (CancellationToken token) => + { + Interlocked.Increment(ref executionCount); + return Task.CompletedTask; // Fast execution + }) + .WithTimeout(TimeSpan.FromMinutes(5)) + .WithErrorHandler(ex => + { + Interlocked.Increment(ref errorCount); + }); + + // Act + await host.StartAsync(); + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(5), steps: 10); + await host.StopAsync(); + + // Assert + Assert.True(executionCount >= 4, $"Expected at least 4 executions, got {executionCount}"); + Assert.Equal(0, errorCount); + } + + [Fact] + public void PeriodicWorker_WithTimeout_Should_Throw_For_Zero_Timeout() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + + using var host = Host.CreateDefaultBuilder().Build(); + + // Act & Assert + var exception = Assert.Throws(() => + { + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + (CancellationToken token) => Task.CompletedTask) + .WithTimeout(TimeSpan.Zero); + }); + + Assert.Equal("timeout", exception.ParamName); + } + + [Fact] + public void PeriodicWorker_WithTimeout_Should_Throw_For_Negative_Timeout() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + + using var host = Host.CreateDefaultBuilder().Build(); + + // Act & Assert + var exception = Assert.Throws(() => + { + host.RunPeriodicBackgroundWorker( + TimeSpan.FromMinutes(1), + (CancellationToken token) => Task.CompletedTask) + .WithTimeout(TimeSpan.FromSeconds(-10)); + }); + + Assert.Equal("timeout", exception.ParamName); + } + + [Fact] + public async Task CronWorker_WithTimeout_Should_Cancel_Long_Running_Execution() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var timeoutOccurred = false; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + // Every minute + host.RunCronBackgroundWorker( + "* * * * *", + async (CancellationToken token) => + { + // Long-running task that will timeout + await timeProvider.Delay(TimeSpan.FromMinutes(30), token); + }) + .WithTimeout(TimeSpan.FromMinutes(2)) + .WithErrorHandler(ex => + { + if (ex is TimeoutException) + { + timeoutOccurred = true; + } + }); + + // Act + await host.StartAsync(); + // Advance time: 1 min for cron trigger, then 2+ min for timeout + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(4), steps: 8); + await host.StopAsync(); + + // Assert + Assert.True(timeoutOccurred, "Timeout should have occurred for long-running cron execution"); + } + + [Fact] + public async Task ContinuousWorker_WithTimeout_Should_Cancel_Long_Running_Execution() + { + // Arrange + BackgroundWorkerExtensions.ClearRegistrations(); + var timeoutOccurred = false; + var timeProvider = WorkerTestHelper.CreateTimeProvider(); + + using var host = Host.CreateDefaultBuilder() + .ConfigureServices(services => + { + services.AddSingleton(timeProvider); + }) + .Build(); + + host.RunBackgroundWorker(async (CancellationToken token) => + { + // Long-running task that exceeds timeout + await timeProvider.Delay(TimeSpan.FromMinutes(30), token); + }) + .WithTimeout(TimeSpan.FromMinutes(2)) + .WithErrorHandler(ex => + { + if (ex is TimeoutException) + { + timeoutOccurred = true; + } + }); + + // Act + await host.StartAsync(); + // Advance time past the timeout + await WorkerTestHelper.AdvanceTimeAsync(timeProvider, TimeSpan.FromMinutes(3), steps: 6); + await host.StopAsync(); + + // Assert + Assert.True(timeoutOccurred, "Timeout should have occurred for long-running continuous worker"); + } +}