diff --git a/README.md b/README.md index 57afbe85fe..30e94dbebe 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ Notes: 3. Use Aspire CLI 13.6.5 or newer (`aspire update --self` to update). The AppHost uses the CLI bundle for the dashboard and resource orchestrator; when no CLI is installed, the SDK downloads the matching version through DNX. 4. Try `dotnet build -mt` for multithreaded builds. Performance depends on the workload and machine. +See the [background task developer guide](docs/background-tasks.md) for task lifecycle, API contracts, cancellation, bulk execution, and extension points. + ## UI Development The Svelte 5 UI in `src/Exceptionless.Web/ClientApp` is the only application frontend and serves at `/`. Aspire starts Svelte together with the API. Primary hash-route bookmarks resolve to the current pages, and existing signed-in sessions are retained. diff --git a/benchmarks/BackgroundTasks/BackgroundTasks.csproj b/benchmarks/BackgroundTasks/BackgroundTasks.csproj new file mode 100644 index 0000000000..64784b27ff --- /dev/null +++ b/benchmarks/BackgroundTasks/BackgroundTasks.csproj @@ -0,0 +1,12 @@ + + + Exe + net11.0 + enable + enable + true + + + + + diff --git a/benchmarks/BackgroundTasks/Program.cs b/benchmarks/BackgroundTasks/Program.cs new file mode 100644 index 0000000000..2e3049ba50 --- /dev/null +++ b/benchmarks/BackgroundTasks/Program.cs @@ -0,0 +1,176 @@ +using System.Diagnostics; +using System.Text.Json; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Services.BackgroundTasks; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Repositories.Utility; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging.Abstractions; + +// A manual service microbenchmark, deliberately outside the correctness test suite. +// Compare the same command before and after a change. This does not measure Redis network latency. +string label = args.FirstOrDefault() ?? "current"; +bool stress = args.Skip(1).Contains("--stress", StringComparer.Ordinal); +int[] sizes = args.Length > 1 && !stress ? args.Skip(1).Select(Int32.Parse).ToArray() : [100, 1_000, 10_000, 50_001]; +Workload[] workloads = stress + ? [new("large-selection-history", 1_000, 2, 1_000), new("many-active-tasks", 1_000, 200, 0), new("same-second-history", 1_000, 2, 1_000, SameSecond: true)] + : sizes.Select(size => new Workload("history", size, 2, 0)).ToArray(); +foreach (var workload in workloads) +{ + int historyCount = workload.HistoryCount; + using var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { CloneValues = true, MaxItems = null }); + using var messageBus = new InMemoryMessageBus(); + using var queue = new InMemoryQueue(); + var measuredCache = new MeasuredCacheClient(cache); + var locks = new CacheLockProvider(cache, messageBus); + var tasks = new BackgroundTaskService(measuredCache, locks, messageBus, queue, + new ResiliencePolicyProvider(), new BackgroundTaskOptions(), TimeProvider.System, NullLogger.Instance); + + const string organizationId = "537650f3b77efe23a47914f4"; + var createdUtc = DateTime.UtcNow.AddSeconds(-historyCount - workload.ActiveCount); + createdUtc = new DateTime(createdUtc.Ticks - createdUtc.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Utc); + var ids = new List(historyCount + workload.ActiveCount); + var selectionIds = Enumerable.Range(0, workload.SelectionIds).Select(_ => ObjectId.GenerateNewId().ToString()).ToList(); + for (int i = 0; i < historyCount + workload.ActiveCount; i++) + { + bool active = i < workload.ActiveCount; + var taskCreatedUtc = workload.SameSecond && !active + ? createdUtc.AddSeconds(workload.ActiveCount).AddTicks(i - workload.ActiveCount) + : createdUtc.AddSeconds(i); + var task = new BackgroundTask + { + // Model servers whose ObjectId machine bytes disagree with precise creation order inside one second. + Id = workload.SameSecond && !active + ? new ObjectId(taskCreatedUtc, machine: historyCount + workload.ActiveCount - i, pid: 1, increment: 0).ToString() + : ObjectId.GenerateNewId(taskCreatedUtc).ToString(), + OrganizationId = organizationId, + Type = "benchmark.work", + Description = "Benchmark task", + CreatedByUserId = "537650f3b77efe23a47914f5", + CreatedUtc = taskCreatedUtc, + UpdatedUtc = taskCreatedUtc, + Status = active ? BackgroundTaskStatus.Running : BackgroundTaskStatus.Completed, + Progress = active ? 10 : 100, + Bulk = selectionIds.Count == 0 ? null : new BulkOperationDetails + { + Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = selectionIds } + } + }; + ids.Add(task.Id); + await cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(task.Id), task, BackgroundTaskService.TaskLifetime); + } + + // Seed equivalent persisted state directly so index construction is outside the measurement. + await cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(organizationId), ids, BackgroundTaskService.TaskLifetime); + await cache.ListAddAsync(BackgroundTaskService.GetActiveIndexCacheKey(organizationId), ids.Take(workload.ActiveCount), BackgroundTaskService.TaskLifetime); + async Task RunAsync(string operation) + { + if (operation == "list") + { + var result = await tasks.GetByOrganizationAsync(organizationId); + return result.Count == Math.Min(historyCount, BackgroundTaskService.ListLimit) + workload.ActiveCount + && result[0].Id == ids[^1] + && result.Count(t => t.IsActive) == workload.ActiveCount; + } + + var updated = await tasks.UpdateAsync(ids[0], task => + { + task.Progress = operation == "progress" ? 20 : 99; + return operation == "progress"; + }); + return updated is { Progress: 20, IsActive: true }; + } + + foreach (string operation in new[] { "list", "progress", "no-op" }) + { + for (int i = 0; i < 3; i++) + await RunAsync(operation); + + int iterations = historyCount >= 10_000 ? 7 : 21; + var milliseconds = new List(iterations); + var allocatedBytes = new List(iterations); + bool correct = true; + for (int i = 0; i < iterations; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + measuredCache.Reset(); + long beforeAllocation = GC.GetTotalAllocatedBytes(precise: true); + long started = Stopwatch.GetTimestamp(); + correct &= await RunAsync(operation); + milliseconds.Add(Stopwatch.GetElapsedTime(started).TotalMilliseconds); + allocatedBytes.Add(GC.GetTotalAllocatedBytes(precise: true) - beforeAllocation); + } + + milliseconds.Sort(); + allocatedBytes.Sort(); + Console.WriteLine(JsonSerializer.Serialize(new + { + label, + workload = workload.Name, + operation, + historyCount, + activeCount = workload.ActiveCount, + selectionIds = workload.SelectionIds, + iterations, + medianMilliseconds = milliseconds[iterations / 2], + medianAllocatedBytes = allocatedBytes[iterations / 2], + measuredCache.ListReads, + measuredCache.SingleReads, + measuredCache.BatchReads, + measuredCache.ListWrites, + measuredCache.ExpirationWrites, + correct + })); + } +} + +sealed record Workload(string Name, int HistoryCount, int ActiveCount, int SelectionIds, bool SameSecond = false); + +// Instrument calls while preserving the real cache implementation and its cloning behavior. +sealed class MeasuredCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient +{ + public int ListReads { get; private set; } + public int SingleReads { get; private set; } + public int BatchReads { get; private set; } + public int ListWrites { get; private set; } + public int ExpirationWrites { get; private set; } + + public void Reset() => ListReads = SingleReads = BatchReads = ListWrites = ExpirationWrites = 0; + + Task> ICacheClient.GetAsync(string key) + { + SingleReads++; + return UnscopedCache.GetAsync(key); + } + + Task>> ICacheClient.GetAllAsync(IEnumerable keys) + { + BatchReads++; + return UnscopedCache.GetAllAsync(keys); + } + + Task>> ICacheClient.GetListAsync(string key, int? page, int pageSize) + { + ListReads++; + return UnscopedCache.GetListAsync(key, page, pageSize); + } + + Task ICacheClient.ListAddAsync(string key, IEnumerable values, TimeSpan? expiresIn) + { + ListWrites++; + return UnscopedCache.ListAddAsync(key, values, expiresIn); + } + + Task ICacheClient.SetExpirationAsync(string key, TimeSpan expiresIn) + { + ExpirationWrites++; + return UnscopedCache.SetExpirationAsync(key, expiresIn); + } +} diff --git a/benchmarks/BackgroundTasks/README.md b/benchmarks/BackgroundTasks/README.md new file mode 100644 index 0000000000..fa42f7b29e --- /dev/null +++ b/benchmarks/BackgroundTasks/README.md @@ -0,0 +1,85 @@ +# Background task service benchmark + +This manual executable measures task listing, active progress updates, and skipped mutations using the real `BackgroundTaskService` and Foundatio in-memory cache. It is outside the normal test suite and solution. It adds no benchmark package or production dependency. + +## Run + +From the repository root with the .NET 11 SDK: + +```sh +dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current +``` + +The first argument labels the results. Optional following arguments replace the finished-history sizes: + +```sh +dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current 100 1000 10000 50001 +``` + +The stress profile checks larger task payloads, a larger active index, and creation bursts across servers: + +```sh +dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current --stress +``` + +| Stress workload | Finished tasks | Active tasks | Explicit selection IDs per task | Creation order | +| --- | ---: | ---: | ---: | --- | +| Large selection history | 1,000 | 2 | 1,000 | Separate seconds | +| Many active tasks | 1,000 | 200 | 0 | Separate seconds | +| Same-second history | 1,000 | 2 | 1,000 | All finished tasks in one second, with ID order opposite to precise creation order | + +To compare revisions, create a separate worktree at the baseline, copy this folder's `Program.cs` and `.csproj` into the same location there, and run the same command in each checkout. Keep the SDK, configuration, dependencies, and machine identical. Run the measurements sequentially without simultaneous builds or test suites. + +## What is measured + +- Seed each history size plus the profile's older active tasks. Setup is outside the measurement. Cache values are cloned as in application use; capacity eviction is disabled so it does not change the selected workload. +- Warm each operation three times, then take 21 samples for smaller histories or seven for histories of 10,000 or more. +- Report median elapsed time and total managed allocation per call, including asynchronous work. Force collection outside each sample to make retained setup state comparable. +- Count service-level index reads, individual and batched record reads, index renewals, and expiry updates. The measuring decorator delegates directly to the underlying cache. +- Check that listing returns the newest finished task, the expected count, and every active task. Check that progress updates retain the running task and requested progress. A skipped mutation deliberately changes its local copy before returning false; the returned result must still describe the stored progress. + +Allocation means bytes allocated during an operation, not retained process memory. This is a service microbenchmark, not an Elasticsearch throughput test or a Redis latency forecast. `GetAllAsync` enables batching, but provider behavior and Redis Cluster hash-slot placement determine actual network requests. The default profile uses two active entries; the stress profile measures a larger active index separately. + +## Recorded comparison + +Measured on Linux x64, .NET SDK 11.0.100, Release, Foundatio 13.0.5. The baseline is [`fc804c08`](https://github.com/exceptionless/Exceptionless/commit/fc804c084506f5f78fc80b11209b49e437423587). The candidate is the service implementation in [`54f7cf26`](https://github.com/exceptionless/Exceptionless/commit/54f7cf264bbeb3f6a79227edd5e1c3972d56bf55), before the same-second ordering and skipped-mutation corrections. [comparison.json](comparison.json) contains every recorded sample summary and the candidate service blob identifier. + +| Finished history | Baseline listing | Candidate listing | Baseline allocation | Candidate allocation | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 0.1340 ms | 0.1573 ms | 0.050 MB | 0.061 MB | +| 1,000 | 0.9845 ms | 0.6104 ms | 0.308 MB | 0.188 MB | +| 10,000 | 16.6519 ms | 5.2518 ms | 10.578 MB | 1.364 MB | +| 50,001 | 152.1701 ms | 17.0455 ms | 233.967 MB | 6.241 MB | + +The 50,001 baseline result is **incorrect**: the old page cap omits the newest task. Its timing is included to expose the old work and failure, not as a comparison between two correct implementations. All candidate result checks passed. + +At 10,000 finished tasks, both versions return the correct result; the candidate used about 87% fewer allocated bytes and 68% less elapsed time. The candidate performs two index reads and two batched record reads for these fixtures. The baseline performs 22 index reads and 52 individual record reads at 10,000 tasks, rising to 101 index reads at 50,001. + +The small-history tradeoff is visible: at 100 tasks, dictionary batching adds about 12 KB of allocation and 0.023 ms of in-process work. It reduces sequential cache calls, which this local benchmark does not price as remote latency. Avoid adding provider-specific branches or caching layers to optimize that small absolute cost without a representative workload. + +### Retention cost + +Progress updates now renew the small active index and its key expiry so a task remains discoverable for the lifetime of its saved record. They do not rewrite the history index. With two active tasks, median allocation increased from 6,640 to 9,128 bytes at the first three history sizes: about 2.5 KB per update. At 1,000 history entries, the observed median was 0.0773 ms before and 0.1047 ms after. Submillisecond timings vary with runtime and host scheduling; the extra renewal and expiry calls are the stable change. + +Terminal updates renew history before removing active membership. These measurements cover active progress, not terminal-history writes. Index retention is a correctness obligation; report its write cost alongside read improvements instead of claiming that every operation became faster. + +## Remaining scaling boundary + +The portable cache API has no ordered recent-history query. Listing still reads and sorts all live history IDs, and returning all active tasks requires materializing that active set. Correct ordering across servers also requires reading every candidate in the cutoff second, even when only a few finished records are requested. The service trims finished candidates between bounded read batches, so retained task payloads scale with active tasks, the requested limit, and one batch rather than the entire burst. A future store with ordered discovery can change that access pattern when real volume warrants it; the [task guide](../../docs/background-tasks.md) describes that extraction boundary. + +## Correctness and stress comparison + +[extended-comparison.json](extended-comparison.json) compares `54f7cf26` with the corrected service, using identical expanded benchmark source. It records both service blob identifiers, the benchmark blob, all 42 summaries, and the same environment and sampling rules. All 21 candidate checks passed. The baseline fails every skipped-mutation return check and the same-second listing check. + +| Listing workload | Before | After | Allocation before | Allocation after | Record batches after | +| --- | ---: | ---: | ---: | ---: | ---: | +| 1,000 history, each with 1,000 selection IDs | 0.5614 ms | 0.5584 ms | 625,304 B | 623,520 B | 2 | +| 1,000 history, 200 active tasks | 1.0354 ms | 0.8396 ms | 371,792 B | 363,496 B | 5 | +| 10,000 history, 2 active tasks | 3.0749 ms | 3.0660 ms | 1,364,320 B | 1,362,536 B | 2 | +| 1,000 same-second tasks with large selections | 0.7746 ms **incorrect** | 5.0146 ms | 625,640 B **incorrect** | 9,282,976 B | 21 | + +Correct same-second ordering costs more reads and decoding allocation because the old version silently discarded newer records. The corrected service retains only 50 finished candidates between batches, with at most another 50 candidates during trimming; it does not retain all 1,000 task payloads. This is an algorithmic bound, not a measured peak-memory claim. + +The successful progress path keeps the same cache-call counts and allocation in these fixtures. At 200 active tasks it allocates 26,248 bytes per update, versus 9,208 with two active tasks; the cache's active-index work is a real scaling cost. A skipped mutation adds one read only on that path: 488 bytes for the small record and 8,896 bytes when its selection contains 1,000 IDs. This upholds the stored-result contract without cloning every successful update. + +Small-history elapsed times varied in both directions: the 100-task list measured 0.1344 ms before and 0.1722 ms after, while its allocation fell by 1,784 bytes. These results do not support a claim that every operation became faster. The stable benefits are bounded retained history payloads, fewer temporary sorting allocations, unchanged successful-update cache calls, and correct results under creation bursts. diff --git a/benchmarks/BackgroundTasks/comparison.json b/benchmarks/BackgroundTasks/comparison.json new file mode 100644 index 0000000000..29bae3a484 --- /dev/null +++ b/benchmarks/BackgroundTasks/comparison.json @@ -0,0 +1,238 @@ +{ + "environment": { + "platform": "Linux x64", + "sdk": "11.0.100", + "configuration": "Release", + "foundatio": "13.0.5" + }, + "baseline_commit": "fc804c084506f5f78fc80b11209b49e437423587", + "candidate_service_blob": "cd9773fe7feafda0cbabb1e44edf1527b37536f0", + "baseline": [ + { + "label": "fc804c08", + "operation": "list", + "historyCount": 100, + "iterations": 21, + "medianMilliseconds": 0.134, + "medianAllocatedBytes": 49640, + "ListReads": 2, + "SingleReads": 52, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "progress", + "historyCount": 100, + "iterations": 21, + "medianMilliseconds": 0.0717, + "medianAllocatedBytes": 6640, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "list", + "historyCount": 1000, + "iterations": 21, + "medianMilliseconds": 0.9845, + "medianAllocatedBytes": 308328, + "ListReads": 4, + "SingleReads": 52, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "progress", + "historyCount": 1000, + "iterations": 21, + "medianMilliseconds": 0.0773, + "medianAllocatedBytes": 6640, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "list", + "historyCount": 10000, + "iterations": 7, + "medianMilliseconds": 16.6519, + "medianAllocatedBytes": 10577920, + "ListReads": 22, + "SingleReads": 52, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "progress", + "historyCount": 10000, + "iterations": 7, + "medianMilliseconds": 0.1462, + "medianAllocatedBytes": 6640, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "list", + "historyCount": 50001, + "iterations": 7, + "medianMilliseconds": 152.1701, + "medianAllocatedBytes": 233967176, + "ListReads": 101, + "SingleReads": 52, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "fc804c08", + "operation": "progress", + "historyCount": 50001, + "iterations": 7, + "medianMilliseconds": 0.1139, + "medianAllocatedBytes": 6584, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + } + ], + "candidate": [ + { + "label": "candidate", + "operation": "list", + "historyCount": 100, + "iterations": 21, + "medianMilliseconds": 0.1573, + "medianAllocatedBytes": 61232, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate", + "operation": "progress", + "historyCount": 100, + "iterations": 21, + "medianMilliseconds": 0.0643, + "medianAllocatedBytes": 9128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate", + "operation": "list", + "historyCount": 1000, + "iterations": 21, + "medianMilliseconds": 0.6104, + "medianAllocatedBytes": 188400, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate", + "operation": "progress", + "historyCount": 1000, + "iterations": 21, + "medianMilliseconds": 0.1047, + "medianAllocatedBytes": 9128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate", + "operation": "list", + "historyCount": 10000, + "iterations": 7, + "medianMilliseconds": 5.2518, + "medianAllocatedBytes": 1364296, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate", + "operation": "progress", + "historyCount": 10000, + "iterations": 7, + "medianMilliseconds": 0.1774, + "medianAllocatedBytes": 9128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate", + "operation": "list", + "historyCount": 50001, + "iterations": 7, + "medianMilliseconds": 17.0455, + "medianAllocatedBytes": 6241400, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate", + "operation": "progress", + "historyCount": 50001, + "iterations": 7, + "medianMilliseconds": 0.1606, + "medianAllocatedBytes": 9128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + } + ] +} diff --git a/benchmarks/BackgroundTasks/extended-comparison.json b/benchmarks/BackgroundTasks/extended-comparison.json new file mode 100644 index 0000000000..6450439704 --- /dev/null +++ b/benchmarks/BackgroundTasks/extended-comparison.json @@ -0,0 +1,741 @@ +{ + "recordedUtcDate": "2026-11-14", + "environment": { + "os": "Linux x64", + "sdk": "11.0.100", + "configuration": "Release", + "foundatio": "13.0.5", + "cache": "InMemoryCacheClient", + "cloneValues": true, + "maxItems": null + }, + "baselineCommit": "54f7cf264bbeb3f6a79227edd5e1c3972d56bf55", + "baselineServiceBlob": "cd9773fe7feafda0cbabb1e44edf1527b37536f0", + "candidateServiceBlob": "2224e293332b8927e8852a8ddddeb62c9111fe4d", + "benchmarkProgramBlob": "2e3049ba50050610ae519bdc5865aefa1b3717de", + "method": "Three warmups, then 21 samples per operation (seven for 10,000 or more history entries); median elapsed time and total managed allocated bytes; GC outside each sample; baseline and candidate run sequentially with identical benchmark source and no concurrent test suites or builds.", + "limitations": [ + "Allocation is total bytes allocated per call, not retained or peak memory.", + "In-process cache measurements do not forecast Redis latency or Elasticsearch throughput.", + "The baseline same-second listing is incorrect; its lower cost omits required candidates.", + "Baseline no-op mutation returns the uncommitted callback copy and fails the contract check.", + "Timings are host-sensitive; operation counts and allocation expose stable work differences.", + "Terminal-history writes and bulk execution are outside this benchmark." + ], + "measurements": [ + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "list", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.1344, + "medianAllocatedBytes": 61256, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "progress", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0616, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "no-op", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0404, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.4538, + "medianAllocatedBytes": 188424, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0663, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0433, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "list", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 3.0749, + "medianAllocatedBytes": 1364320, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "progress", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.1048, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "no-op", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.0474, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "list", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 27.6162, + "medianAllocatedBytes": 6241424, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "progress", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.2345, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "no-op", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.1602, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "large-selection-history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.5614, + "medianAllocatedBytes": 625304, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "large-selection-history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0973, + "medianAllocatedBytes": 26024, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "large-selection-history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0729, + "medianAllocatedBytes": 14128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "many-active-tasks", + "operation": "list", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 1.0354, + "medianAllocatedBytes": 371792, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 5, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "many-active-tasks", + "operation": "progress", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.1029, + "medianAllocatedBytes": 26248, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "many-active-tasks", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0432, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "same-second-history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.7746, + "medianAllocatedBytes": 625640, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "same-second-history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.1199, + "medianAllocatedBytes": 26024, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "same-second-history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0734, + "medianAllocatedBytes": 14128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "list", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.1722, + "medianAllocatedBytes": 59472, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "progress", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0456, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "no-op", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.03, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.6504, + "medianAllocatedBytes": 186640, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0692, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0418, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "list", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 3.066, + "medianAllocatedBytes": 1362536, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "progress", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.0656, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "no-op", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.0393, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "list", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 21.9808, + "medianAllocatedBytes": 6239640, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "progress", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.234, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "no-op", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.1751, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "large-selection-history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.5584, + "medianAllocatedBytes": 623520, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "large-selection-history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0947, + "medianAllocatedBytes": 26024, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "large-selection-history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0528, + "medianAllocatedBytes": 23024, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "many-active-tasks", + "operation": "list", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.8396, + "medianAllocatedBytes": 363496, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 5, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "many-active-tasks", + "operation": "progress", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.1004, + "medianAllocatedBytes": 26248, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "many-active-tasks", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0434, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "same-second-history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 5.0146, + "medianAllocatedBytes": 9282976, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 21, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "same-second-history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0854, + "medianAllocatedBytes": 26024, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "same-second-history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0584, + "medianAllocatedBytes": 23024, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + } + ] +} diff --git a/docs/_config.ts b/docs/_config.ts index 20f2af36dc..3667a21c17 100644 --- a/docs/_config.ts +++ b/docs/_config.ts @@ -15,6 +15,7 @@ site.ignore( ".gitignore", ".npmrc", "_site", + "background-tasks.md", "deno.json", "deno.lock", "news/index.md", diff --git a/docs/background-tasks.md b/docs/background-tasks.md new file mode 100644 index 0000000000..7bcf5d09ff --- /dev/null +++ b/docs/background-tasks.md @@ -0,0 +1,192 @@ +# Background tasks + +Background tasks let users follow work performed by `WorkItemJob`: bulk actions, project data resets, and sample-data generation. The task record describes progress; the work queue controls delivery. This guide covers the application contracts and the boundary for a future Foundatio extraction. + +## Ownership and persistence + +| Component | Owns | Source | +| --- | --- | --- | +| `BackgroundTaskService` | Cached records, organization indexes, serialized updates, cancellation requests, progress notifications | [Service](../src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs) | +| `WorkItemJob` and its queue | Delivery, acknowledgment, work-item retries, execution locks | [Registration](../src/Exceptionless.Core/Bootstrapper.cs) | +| `BackgroundTaskTrackingWorkItemHandler` | Progress and lifecycle tracking around an existing ordinary work-item handler | [Tracking wrapper](../src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs) | +| `BulkOperationService` and `BulkOperationWorkItemHandler` | Destructive-operation admission, batch execution, checkpoints, bounded continuation, selection cleanup | [Admission](../src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs), [worker](../src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs) | +| Selection resolver and action handler | Event membership, current access/retention rules, and the action's domain side effects | [Contracts](../src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs) | +| HTTP and WebSocket adapters | Authentication, organization access, public projections, client discovery and reconciliation | [API handler](../src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs), [client](../src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts) | + +The task record is the source of truth for saved progress. Counts and the bulk cursor share one record and are saved together. The queue payload carries the task identifier and the information required by its handler. Notifications are best effort; polling reads the saved state when a notification is delayed or lost. + +Task records use the configured `ICacheClient` and expire seven days after their last save. They provide limited history rather than a durable audit log. Index entries have independent per-member expiration. Every active update renews active membership; a terminal update renews finished-history membership. This keeps long-running work discoverable without rewriting the larger history index on every progress report. The service also refreshes the index key's lifetime because renewing an existing member does not extend the key in every provider. + +Index cleanup re-reads suspected stale entries under the same task-update lease. It skips busy tasks, retains tasks that became active, and removes history only when the record is still absent. This avoids waiting behind progress writes or deleting a retry's renewed membership. Readers clean up within their existing batches; normal active reads acquire no additional locks. + +Task mutations require isolated snapshots from the cache: serialized providers provide this boundary, and the in-memory provider must use `CloneValues = true`. A callback must retain the task's identity and ownership. A future store adapter must preserve these requirements rather than exposing a shared mutable task instance. + +Updates are serialized while their update lease remains valid. Like admission locks, these cache leases are not fencing tokens: a process or store call that outlives its lease can race a newer writer. A future durable store needs conditional revisions or equivalent fencing before promising stronger write isolation. + +Record and index writes are separate operations. A cache failure can leave incomplete discovery even when a record was saved; a no-op mutation does not repair every prior index failure. History refresh precedes active removal so normal completion has overlapping membership and a failed refresh preserves the existing fallback. Treat this cache-backed history as best effort, and use a store with stronger indexing guarantees if a future consumer requires transactional discovery. + +## Creating and updating a task + +Ordinary callers use `CreateAndEnqueueAsync`. Authorization and project validation happen before this call. The factory runs after the task has its identifier, so the work item can reference it: + +```csharp +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; + +static Task QueueSampleDataAsync( + BackgroundTaskService tasks, + string organizationId, + string projectId, + string userId) +{ + return tasks.CreateAndEnqueueAsync(new BackgroundTask + { + OrganizationId = organizationId, + ProjectId = projectId, + Type = BackgroundTaskTypes.GenerateProjectSampleData, + Description = "Generate sample data", + CreatedByUserId = userId + }, task => new GenerateSampleEventsWorkItem + { + OrganizationId = organizationId, + ProjectId = projectId, + TaskId = task.Id, + EventCount = 250, + DaysBack = 7 + }); +} +``` + +This uses the same work item as [SampleDataService](../src/Exceptionless.Core/Utility/SampleDataService.cs). `CanCancel` defaults to false because this handler cannot stop midway safely. Set it to true only for an implementation that honors cancellation. + +| Service method | Contract | +| --- | --- | +| `CreateAndEnqueueAsync` | Creates and indexes a queued task, then prepares and enqueues its work item. Factory or queue failures fail a still-queued task. | +| `GetAsync` | Returns the stored record, or null when absent. Internal callers must enforce their own access policy. | +| `GetByIdsAsync` | Returns existing tasks once per ID in input order, omitting missing records. Cache reads contain at most 50 records; callers enforce scope. | +| `GetByOrganizationAsync` | Returns all discoverable active tasks plus up to `limit` finished tasks, newest first; the default finished limit is 50. | +| `PruneIndexesAsync` | Rechecks suspected stale IDs under their update leases. Removes finished tasks from the active index and missing tasks from both indexes; skips busy, active, or foreign tasks. | +| `UpdateAsync` | Locks one task, reloads it, runs a synchronous mutation, and saves only when the callback returns true. Returns the stored state, including after a skipped mutation, or null when the record is absent. | +| `CancelAsync` | Records a cancellation request and returns the current task outcome, including a distinct non-cancellable result. | +| `IsCancelRequestedAsync` | Lets a cooperative worker check the persisted cancellation request at a safe boundary. | + +`AddAsync` followed by `EnqueueAsync` is the lower-level path used by bulk admission, where selection state must be saved before enqueueing. Keep that sequence inside the owning operation service. Raw persistence is private so callers use the locking, indexing, and notification path. + +Task storage and queue submission are separate operations. An enqueue exception can occur after the queue accepted the item; cleanup changes only a task that is still queued and preserves an outcome already recorded by the worker. Keep work-item factories free of domain side effects. The returned creation snapshot may already be older than worker progress by the time it reaches the client. + +An `UpdateAsync` callback should only inspect and change the supplied task. Perform expensive work outside it, then checkpoint the result. The service owns `UpdatedUtc`: every committed mutation receives a timestamp strictly newer than the previous record, including when a clock repeats or moves backward. The persistence lock is released before publishing; a slow message bus does not hold up another mutation. + +Returning false discards the mutation and reloads the stored state for the result. This extra read occurs only on skipped updates; successful progress updates do not clone another task just to support rollback. Advisory publication has no caller cancellation token, so a transport timeout reported as `TaskCanceledException` is logged like other publication failures. It cannot reverse an accepted enqueue or a committed update. + +## Lifecycle and retry rules + +`IsActive` means `Queued` or `Running`. The worker owns valid transitions for its task type; the service supplies serialized mutations. + +| Event | Saved behavior | +| --- | --- | +| Accepted submission | `Queued`, with creation and update timestamps | +| Work starts or reports progress | `Running`; the first start timestamp is retained | +| Successful completion | `Completed`, completion timestamp, progress 100 | +| Ordinary handler throws | `Failed` with a generic public message; the queue may retry it | +| Ordinary queue retries a failed task | `Running` again; completion time, message, and percent are reset, and active membership is restored | +| Bulk transient error | Keep the task active and continue from its saved cursor; three consecutive transient failures end it as `Failed` | +| Bulk permanent error, lost selection, or exhausted checkpoint retries | `Failed`; the task does not reopen automatically | +| Duplicate delivery of completed or cancelled tracked work | Skip execution | + +A non-success ordinary work-item result, such as shutdown, leaves tracking active so queue delivery can resume it. Sample generation returns non-success when shutdown leaves a required batch unstarted; shutdown after its final required batch remains successful. Ordinary queue retries can replay earlier side effects. The wrapper has no provider-neutral retry-exhaustion signal; repeated non-success deliveries can therefore exhaust retries while their task remains active. Bulk shutdown also retains its saved progress and selection. An ordinary work item with no `TaskId` runs without tracking. If an ordinary task record disappears, its existing queue payload can still run; a bulk task requires its saved record and selection and stops if either is lost. + +`Failed` therefore describes a failed attempt for an ordinary tracked work item and a terminal outcome for a bulk operation. Consumers must allow a newer ordinary-work update to move `Failed` back to `Running`. Retry policy belongs to the queue or bulk worker, not to a status label alone. + +The tracking wrapper keeps the initial start decision strict so a storage failure cannot accidentally run completed or cancelled work. Subsequent progress and outcome tracking is best effort. The wrapper forwards the job's original progress callback before tracking I/O, including execution-lock renewal when the handler enables it. A tracking transport failure cannot replace the handler's original exception. Cancellation of the actual work and its original progress callback still propagates normally. + +## Cancellation and staleness + +Cancellation is cooperative. A queued cancellable task can be marked `Cancelled` immediately. A running bulk task checks the request between batches, finishes the batch already in flight, then records `Cancelled`. Completed side effects are retained. + +`StaleAfter` is two hours without a saved update. Staleness is computed when needed; it is not a separate status or a timer that automatically kills work. An abandoned task can be cleared through cancellation even when its type normally cannot be stopped. Bulk admission can retire an abandoned conflicting operation before accepting a replacement. + +A worker checks saved activity at the next batch boundary. Retirement therefore stops it from starting another batch even if it missed the admission decision while paused. This cannot undo side effects from a batch that was already in flight. + +Cancelling a finished task is idempotent. A successful cancellation request can still return `Running` while the worker finishes its current batch. Continue observing the task until its state changes. The task snapshot does not expose a separate cancellation-pending field. + +## HTTP and notification contracts + +All routes are under `/api/v2`. Task lookup and cancellation enforce organization access; inaccessible records return the same not-found response as missing records. + +| Route | Result | +| --- | --- | +| `POST /bulk-operations` | HTTP 202 with the accepted task; conflicting destructive work returns 409 | +| `GET /tasks/{id}` | One task snapshot; missing or inaccessible tasks return 404 | +| `GET /organizations/{id}/tasks` | All active tasks plus the newest 50 finished tasks | +| `POST /tasks/{id}/cancel` | HTTP 202 with the current snapshot; a live non-cancellable task returns 422 | + +The HTTP [view model](../src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs) includes status, counts, message, creator, timestamps, and a selection summary for bulk work. That summary can contain the filter, original time expression, resolved UTC bounds, and explicit ID count. It omits the ID array, operation parameters, pinned membership, snapshot identifier, and resume cursor. The smaller [BackgroundTaskUpdated message](../src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs) also omits the selection summary. + +Messages can be duplicated, delayed, or delivered out of order. Compare `UpdatedUtc` at .NET tick precision before applying them; JavaScript's `Date` alone discards submillisecond precision. Unknown task identifiers trigger rediscovery, including when the initial list request has not finished. Bursts share the required reconciliation request. A distinct task created after that request's snapshot can be discovered by the next poll. + +Known progress updates do not cancel list reads. At the cache commit, the client merges the HTTP result with changes received since the read began: newer task revisions win, newly inserted or changed tasks survive omission from an older response, and unchanged omitted records can disappear. This uses one baseline per in-flight query and releases it after the read; it does not buffer every progress message. + +The panel polls the current organization every three seconds while any task is active and once a minute while idle. Its fixed cadence changes only with the organization or active/idle state, so progress revisions cannot continually restart it. A tick skips hidden tabs and an already-running query; returning to an organization also refetches its list. Idle polling covers missed notifications, work started elsewhere, and ordinary queue retries after a failed attempt; it does not guess when a provider has exhausted retries. The idle cost is one task-list request per minute per visible tab. Notification delivery remains an optimization for latency. + +Dismissal and completion handling follow the task's outcome revision. A newer terminal outcome can appear and refresh affected data even when the client missed its intervening `Running` state. Repeated copies of the same outcome do not repeat that refresh. If a previously active task disappears from a later list, the panel also refreshes affected data once: the task may have finished outside the 50-task history window or its record may have expired. Omission alone does not establish its final status. + +The first successful task list also reconciles affected event and stack queries when it contains finished tasks. This covers completion during delayed initial discovery or missed notifications. Old finished history can cause one additional refresh on first discovery for that organization; later identical polls do not. Reconciliation supersedes reads started before the outcome, including uncached initial reads, and simultaneous completions share the replacement request. This guarantees a new query attempt, while normal Elasticsearch search visibility still applies. + +## Bulk selection and execution + +Explicit ID selection accepts at most 1,000 distinct IDs. Larger requests use a filter. The resolver captures filter membership during preparation: a complete small selection keeps fixed IDs, while a larger selection keeps an Elasticsearch point-in-time snapshot and search-after cursor. Relative dates use a fixed UTC anchor and invariant formatting. + +For filter selections, current organization/project scope, search-plan rules, and retention can narrow the eligible set. Later arrivals or edits cannot expand it. Missing, expired, or legacy selection state fails safely and requires starting a new operation; the worker does not silently re-run a live filter. Empty selections have an explicit persisted representation because the application serializer omits empty collections. + +Preparation runs before the organization admission lock. Admission serializes destructive bulk operations that overlap the same project while its lease is valid; an organization-wide operation overlaps every project. Each batch executes at most the configured number of IDs, with a default batch size of 500. Event deletion fetches only identity, daily-index routing, organization, project, and stack fields, then uses the same deletion service as individual deletes. + +Admission renews its lease between bounded conflict-read batches, before stale-task retirement, and before indexing the candidate. Monotonic elapsed time detects a scan or renewal that outlived the prior lease; that request returns busy instead of admitting work from an old snapshot. Cache leases are not fencing tokens: a process suspended across lease expiry during the final store writes, or a cache that loses a live lock key, cannot provide a cross-store transactional exclusion guarantee. Handlers must still tolerate replay and already-running side effects. + +The snapshot has a two-hour idle lease, renewed while paging. The cached immutable selection also has its own seven-day retention, renewed as the active worker checkpoints so a task older than seven days can still resume. Renewing that expiry does not rewrite the selection payload. Terminal or rejected work releases the snapshot best effort; active retries and shutdown retain it. Expiry bounds the resource when cleanup cannot reach the backend. + +The [pinned Foundatio.Repositories implementation](https://github.com/FoundatioFx/Foundatio.Repositories/blob/690e3f7d2c9a81cad2f658e10c29cc609f1e79f7/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReadOnlyRepositoryBase.cs) can open a snapshot and then throw before returning its identifier during preparation. The caller cannot release an identifier it never received; that snapshot remains bounded by its idle lease. Fixing this boundary belongs in the provider's failed-query cleanup, not in a second Elasticsearch access path in the task service. + +Completion does not force an Elasticsearch index refresh. Deletion uses the repository's normal eventual search consistency, so search results can briefly retain deleted events until the index refreshes. + +Usage accounting after successful deletion is best effort. Its transport failures, including cancellation without a caller token, do not turn a completed deletion into a failed batch or prevent accounting for other projects. A failure of the actual deletion still propagates. + +Deletion and checkpoint persistence are separate writes. The worker retries saving a known batch result before considering another execution. A process can still stop after a side effect but before its checkpoint is saved, so handlers must tolerate replay. Missing items are reported as failed; counts are not an exactly-once ledger of external side effects. + +## Adding work + +For an ordinary tracked job: + +1. Implement the existing work-item handler and give its work-item type an optional `TaskId` through `IBackgroundTaskWorkItem`. +2. Register it through `TrackWithBackgroundTask` in `Bootstrapper`, following reset-data or sample-data generation. +3. Create it through `CreateAndEnqueueAsync` after authorization, and report progress through `WorkItemContext.ReportProgressAsync` so execution-lock renewal still happens. +4. Define which side effects may replay, whether cancellation is safe, and which queue failures are retryable. + +For a bulk action, implement `IBulkOperationHandler` with a unique operation type, authorization, parameter validation, description, and bounded batch execution. Reuse the existing entity resolver when membership semantics match. A new entity family also needs an `IBulkSelectionResolver` that prepares stable membership, returns opaque resumable cursors, and releases its resources. Keep all domain side effects in the existing domain service. + +## Scaling and a future Foundatio boundary + +Keep immutable selection data separate from frequently updated progress where the existing format permits it. Process one batch at a time, project only required fields, and use the cache's bulk-read operation for groups of task records. Listing reads each index once, orders the IDs, and loads records in batches of at most 50. It has no silent history-page cutoff. Conflict detection uses the same bounded record reads while checking the active index. + +ObjectId order determines the creation second, but IDs from different servers do not determine creation order inside that second. Listing reads the full cutoff second and selects by precise `CreatedUtc`, then ID. It retains only the requested finished history between batches, plus all active tasks. A burst in the cutoff second increases record reads without retaining the burst's entire task payload. + +Returning all active tasks inherently requires work proportional to that active set. The generic cache-list API does not provide a portable ordered recent-history query, so listing still reads and orders the live history IDs. `GetAllAsync` lets providers combine reads; it does not promise one network round trip on every provider or Redis Cluster topology. Active-index renewal adds writes to a progress update, while the history index is renewed only on terminal updates. Measure these operations separately when changing retention or storage. + +| Reusable responsibility | Keep in the application adapter | +| --- | --- | +| Task identity, status, progress, timestamps, and serialized state updates | ObjectId HTTP routes and Exceptionless model interfaces | +| Store retention, discovery, bounded reads, and cancellation signaling | Organization/project/user authorization and admission policy | +| Worker tracking and progress reporting using existing Foundatio primitives | Event filters, plan limits, retention policy, Elasticsearch snapshots, deletion accounting | +| Optional advisory change notifications | Public HTTP/WebSocket projections, UI text, and task-panel behavior | + +The current `BackgroundTask` includes organization/project/user fields and `BulkOperationDetails`; it is still an application model. Preserve that honest boundary until another consumer establishes the common contract. An extraction should define identity, owner scope, private checkpoint serialization, retention/index consistency, and retry ownership explicitly, while reusing `ICacheClient`, queues, locks, messaging, resilience, and `TimeProvider`. Introduce a storage abstraction when it is needed to offer indexed discovery or durability that the cache contract cannot provide. Avoid a generic payload bag or a parallel queue/state-machine framework solely to anticipate reuse. + +## Verification and operations + +Service tests use real Foundatio in-memory infrastructure and controlled time. Regression coverage belongs at the failure boundary: factory/index/queue failures, concurrent updates, delayed notifications, cancellation, retention, full selection paging, and checkpoint recovery. Integration tests exercise real serialization, Elasticsearch routing, authorization, usage accounting, and selection mutation. Browser tests cover discovery, concurrent task completion, and cancellation access. + +Provider behavior remains part of this contract. Foundatio 13.0.5's [in-memory expiry cleanup](https://github.com/FoundatioFx/Foundatio/blob/be10785cfed4e1850350dfaa024e1c3f25b7fac7/src/Foundatio/Caching/InMemoryCacheClient.cs) can observe an expired entry and then remove a concurrently renewed replacement by key. The controlled-time retention regression exposed its `Removed item was not expired` diagnostic. This provider race is not repaired by task-service locking; the cache-loss recovery described above still applies. It is distinct from the Redis provider and should be addressed in the in-memory cache implementation before relying on stronger retention guarantees there. + +Start with the focused classes under [BackgroundTasks](../tests/Exceptionless.Tests/Services/BackgroundTasks) and [BulkOperations](../tests/Exceptionless.Tests/Services/BulkOperations), then the [worker tests](../tests/Exceptionless.Tests/Jobs/WorkItemHandlers), endpoint tests, and the full CI run. Manual performance measurements live in [benchmarks/BackgroundTasks](../benchmarks/BackgroundTasks); they report service time, allocation, and cache calls separately from provider/network throughput. + +Deploy the updated job before or with the web application. Cache loss can remove progress and prepared selections; treat that as an interrupted operation requiring user-visible recovery. This feature adds no database migration or index. Preserve public DTOs and message shapes when changing internal storage or moving reusable components to Foundatio. diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index ee36fdf0d6..4a08e2c6f5 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -26,6 +26,8 @@ using Exceptionless.Core.Seed; using Exceptionless.Core.Serialization; using Exceptionless.Core.Services; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; using Exceptionless.Core.Services.SourceMaps; using Exceptionless.Core.Utility; using Exceptionless.Core.Validation; @@ -93,6 +95,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(s => { var handlers = new WorkItemHandlers(); + handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); @@ -102,12 +105,12 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); - handlers.Register(s.GetRequiredService); + handlers.Register(() => TrackWithBackgroundTask(s, s.GetRequiredService())); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); - handlers.Register(s.GetRequiredService); + handlers.Register(() => TrackWithBackgroundTask(s, s.GetRequiredService())); return handlers; }); @@ -235,6 +238,13 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO }); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(typeof(IBulkOperationHandler), typeof(Bootstrapper).Assembly); + services.AddSingleton(typeof(IBulkSelectionResolver), typeof(Bootstrapper).Assembly); services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(); services.AddSingleton(); @@ -530,6 +540,11 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log logger.LogWarning("Jobs running in process"); } + private static IWorkItemHandler TrackWithBackgroundTask(IServiceProvider container, IWorkItemHandler handler) + { + return new BackgroundTaskTrackingWorkItemHandler(handler, container.GetRequiredService(), container.GetRequiredService()); + } + private static IQueue CreateQueue(IServiceProvider container, TimeSpan? workItemTimeout = null) where T : class { var loggerFactory = container.GetRequiredService(); diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs new file mode 100644 index 0000000000..11185a1e64 --- /dev/null +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs @@ -0,0 +1,170 @@ +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Queues; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs.WorkItemHandlers; + +/// +/// Wraps the handler of a work item that is tracked by a background task. Progress, success and failure are written to the +/// task in the job process that runs the work. Progress reaches the job's original callback before tracking I/O, preserving +/// lock renewal when enabled. Work items without a task id pass straight through. +/// +internal sealed class BackgroundTaskTrackingWorkItemHandler : IWorkItemHandler +{ + private readonly IWorkItemHandler _inner; + private readonly BackgroundTaskService _taskService; + private readonly TimeProvider _timeProvider; + + public BackgroundTaskTrackingWorkItemHandler(IWorkItemHandler inner, BackgroundTaskService taskService, TimeProvider timeProvider) + { + _inner = inner; + _taskService = taskService; + _timeProvider = timeProvider; + } + + public bool AutoRenewLockOnProgress + { + get => _inner.AutoRenewLockOnProgress; + set => _inner.AutoRenewLockOnProgress = value; + } + + public ILogger Log + { + get => _inner.Log; + set => _inner.Log = value; + } + + public Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) + { + return _inner.GetWorkItemLockAsync(workItem, cancellationToken); + } + + public void LogProcessingQueueEntry(IQueueEntry queueEntry, Type workItemDataType, object workItem) + { + _inner.LogProcessingQueueEntry(queueEntry, workItemDataType, workItem); + } + + public void LogAutoCompletedQueueEntry(IQueueEntry queueEntry, Type workItemDataType, object workItem) + { + _inner.LogAutoCompletedQueueEntry(queueEntry, workItemDataType, workItem); + } + + public async Task HandleItemAsync(WorkItemContext context) + { + if (context.Data is not IBackgroundTaskWorkItem { TaskId: { Length: > 0 } taskId }) + { + await _inner.HandleItemAsync(context); + return; + } + + string? lastMessage = null; + var tracked = new WorkItemContext(context.Data, context.JobId, context.WorkItemLock, context.CancellationToken, async (progress, message) => + { + // Give the job its renewal opportunity before tracking waits on cache or messaging. + await context.ReportProgressAsync(progress, message); + + lastMessage = message ?? lastMessage; + await TryUpdateAsync(taskId, task => + { + if (!task.IsActive) + return false; + + task.Start(UtcNow); + task.Progress = Math.Clamp(progress, 0, 100); + task.Message = message; + return true; + }); + }); + + if (!await StartAsync(taskId)) + return; + + try + { + await _inner.HandleItemAsync(tracked); + } + catch (Exception) + { + await TryUpdateAsync(taskId, task => + { + if (!task.IsActive) + return false; + + // The exception is logged by the job; its message can contain internal details, so the user sees a generic one. + task.Finish(BackgroundTaskStatus.Failed, "An error occurred. It will be retried automatically if possible.", UtcNow); + return true; + }); + + throw; + } + + context.Result = tracked.Result; + + // A result that is not a success (for example a shutdown) means the queue runs the work item again, so the task keeps running. + if (!tracked.Result.IsSuccess) + return; + + await TryUpdateAsync(taskId, task => + { + if (!task.IsActive) + return false; + + task.Finish(BackgroundTaskStatus.Completed, lastMessage ?? "Completed.", UtcNow); + return true; + }); + } + + private DateTime UtcNow => _timeProvider.GetUtcNow().UtcDateTime; + + /// + /// Marks the task running. A queue retry runs the work item again after it failed, which brings the task back to running + /// and back into the active index. A cancelled or completed task must not run again, so a failed start decision must + /// be retried by the queue rather than treated as a missing task record. + /// + private async Task StartAsync(string taskId) + { + bool wasFinished = false; + var task = await _taskService.UpdateAsync(taskId, current => + { + if (current.Status is BackgroundTaskStatus.Cancelled or BackgroundTaskStatus.Completed) + return false; + + wasFinished = current.Status is BackgroundTaskStatus.Failed; + current.Start(UtcNow); + if (wasFinished) + { + current.CompletedUtc = null; + current.Message = null; + current.Progress = 0; + } + + return true; + }); + + return task is not { Status: BackgroundTaskStatus.Cancelled or BackgroundTaskStatus.Completed }; + } + + private async Task TryUpdateAsync(string taskId, Func update) + { + BackgroundTask? result = null; + await TryAsync(taskId, async () => result = await _taskService.UpdateAsync(taskId, update)); + return result; + } + + // Tracking is best effort and uses no caller cancellation token. A transport failure must not replace the worker's outcome. + private async Task TryAsync(string taskId, Func action) + { + try + { + await action(); + } + catch (Exception ex) + { + Log.LogWarning(ex, "Unable to update background task {TaskId}: {Message}", taskId, ex.Message); + } + } +} diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs new file mode 100644 index 0000000000..2ea69febab --- /dev/null +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -0,0 +1,399 @@ +using System.Text.Json; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories.Base; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs.WorkItemHandlers; + +/// +/// Runs a stored bulk task in batches. After every batch the counts and the resume cursor are saved and a progress +/// message is published; between batches it checks for cancellation. A restarted job continues from the saved cursor. +/// +public class BulkOperationWorkItemHandler : WorkItemHandlerBase +{ + /// + /// How many temporary errors in a row (without a batch completing in between) end the task. + /// + public const int MaxTransientFailures = 3; + + /// + /// Saved as the cursor once the last batch is done, so a run that restarts before the task is marked complete finds + /// nothing left to do instead of starting the selection over. + /// + internal const string ExhaustedCursor = "exhausted"; + + private readonly BulkOperationService _service; + private readonly BackgroundTaskService _taskService; + private readonly IReadOnlyDictionary _resolvers; + private readonly BulkOperationOptions _options; + private readonly ILockProvider _lockProvider; + private readonly TimeProvider _timeProvider; + private readonly IResiliencePolicy _checkpointPolicy; + + public BulkOperationWorkItemHandler(BulkOperationService service, BackgroundTaskService taskService, IEnumerable resolvers, BulkOperationOptions options, + ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory) + { + _service = service; + _taskService = taskService; + _resolvers = resolvers.ToDictionary(r => r.Entity); + _options = options; + _lockProvider = lockProvider; + _timeProvider = timeProvider; + _checkpointPolicy = new ResiliencePolicyBuilder(Log, timeProvider) + .WithMaxAttempts(MaxTransientFailures) + .WithDelay(options.RetryDelay) + .WithShouldRetry((_, exception) => !IsPermanent(exception)) + .Build(); + + // Progress is reported after every batch. Renewing the queue entry and the task lock then keeps a long run from + // timing out and being delivered to a second worker while this one is still deleting. + AutoRenewLockOnProgress = true; + } + + public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) + { + string cacheKey = $"{nameof(BulkOperationWorkItem)}:{((BulkOperationWorkItem)workItem).TaskId}"; + return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(15), cancellationToken); + } + + public override async Task HandleItemAsync(WorkItemContext context) + { + var workItem = context.GetData()!; + ArgumentException.ThrowIfNullOrWhiteSpace(workItem.TaskId); + + BackgroundTask? task = null; + BackgroundTask? selectionTask = null; + string? pinnedSelection = null; + string? latestSelectionCursor = null; + try + { + task = await _taskService.GetAsync(workItem.TaskId); + if (task is null) + { + // The record expired or the cache was cleared. Say so, otherwise the progress panel waits forever. + Log.LogWarning("Bulk operation {TaskId} no longer exists", workItem.TaskId); + await _service.PublishLostAsync(workItem); + return; + } + + selectionTask = task; + latestSelectionCursor = task.Bulk?.Cursor; + using var _ = Log.BeginScope(new ExceptionlessState().Organization(task.OrganizationId).Tag("BulkOperation")); + if (!task.IsActive) + { + Log.LogInformation("Skipping bulk operation {TaskId} because it is already {Status}", task.Id, task.Status); + return; + } + + var handler = _service.GetHandler(task.Type); + if (handler is null || !_resolvers.TryGetValue(handler.Entity, out var resolver)) + { + await FinishAsync(workItem, BackgroundTaskStatus.Failed, $"Unknown bulk operation type '{task.Type}'."); + return; + } + + if (await _taskService.IsCancelRequestedAsync(task.Id)) + { + await FinishAsync(workItem, BackgroundTaskStatus.Cancelled, "Cancelled by user."); + return; + } + + if (task.Bulk!.Selection.Kind is BulkOperationSelectionKind.Filter) + { + pinnedSelection = await _service.GetPinnedSelectionAsync(task.Id); + if (pinnedSelection is null) + { + await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The saved selection expired, so the operation could not continue."); + return; + } + + await _service.RefreshSelectionExpirationAsync(task); + } + + task = await _taskService.UpdateAsync(task.Id, current => + { + if (!current.IsActive) + return false; + + current.Start(UtcNow); + return true; + }); + + if (task is null) + { + await _service.PublishLostAsync(workItem); + return; + } + + if (task.Status is not BackgroundTaskStatus.Running) + return; + + Log.LogInformation("Running bulk operation {TaskId} {Type} from cursor {Cursor}", task.Id, task.Type, task.Bulk!.Cursor); + await context.ReportProgressAsync(GetPercent(task), task.Description); + + while (true) + { + if (context.CancellationToken.IsCancellationRequested) + { + // The host is shutting down. Leave the task running so the queue retry resumes it from the cursor. + context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {task.Id} for shutdown"); + return; + } + + // Admission can retire a stale worker while it is between batches. Observe the saved outcome before + // starting another action, including when the old worker resumes after its replacement was admitted. + task = await _taskService.GetAsync(workItem.TaskId); + if (task is null) + { + await _service.PublishLostAsync(workItem); + return; + } + + if (!task.IsActive) + return; + + if (await _taskService.IsCancelRequestedAsync(task.Id)) + { + await FinishAsync(workItem, BackgroundTaskStatus.Cancelled, $"Cancelled by user after {task.Processed:N0} of {task.Total:N0}."); + return; + } + + var page = await GetPageAsync(task, resolver, pinnedSelection, context.CancellationToken); + latestSelectionCursor = page.NextCursor ?? latestSelectionCursor; + if (page.Ids.Count == 0) + break; + + // The user who started the task is authorized once, when it is created. Each batch re-checks the scope + // (organization and project) but not the user: someone removed from the organization mid-run still finishes + // the deletion they legitimately started, and any member of the organization can cancel it. + var result = await handler.ExecuteBatchAsync(task, page.Ids, context.CancellationToken); + + task = await SaveBatchAsync(task, page, result); + + if (task is null) + { + await _service.PublishLostAsync(workItem); + return; + } + + if (!task.IsActive) + return; + + await _service.RefreshSelectionExpirationAsync(task); + await context.ReportProgressAsync(GetPercent(task), $"{task.Processed:N0} of {task.Total:N0}"); + + if (page.NextCursor is null) + break; + } + + task = await _taskService.UpdateAsync(task.Id, current => + { + if (!current.IsActive) + return false; + + current.Total = current.Processed; + current.Finish(BackgroundTaskStatus.Completed, current.Failed > 0 ? $"Completed with {current.Failed:N0} failures." : "Completed.", UtcNow); + return true; + }); + + if (task is null) + await _service.PublishLostAsync(workItem); + } + catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested) + { + context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {workItem.TaskId} for shutdown"); + } + catch (BatchCheckpointException ex) + { + Log.LogError(ex, "Bulk operation {TaskId} could not save completed batch progress: {Message}", workItem.TaskId, ex.Message); + await FinishAsync(workItem, BackgroundTaskStatus.Failed, ex.Message); + } + catch (BulkSelectionRejectedException ex) + { + Log.LogWarning(ex, "Bulk operation {TaskId} selection is no longer available: {Message}", workItem.TaskId, ex.Message); + await FinishAsync(workItem, BackgroundTaskStatus.Failed, ex.Message); + } + catch (Exception ex) when (!IsPermanent(ex)) + { + // Elasticsearch, cache and network hiccups: continue in a new work item from the saved counts and cursor. Rethrowing + // would use the queue's retries, which cover the whole work item, so a task that makes progress between errors + // would run out of deliveries while still marked running. After MaxTransientFailures in a row, give up. + var failed = await _taskService.UpdateAsync(workItem.TaskId, current => + { + if (!current.IsActive) + return false; + + current.Bulk!.RetryCount++; + current.Message = current.Bulk!.RetryCount >= MaxTransientFailures ? "The operation stopped because of repeated errors." : "Retrying after a temporary error."; + if (current.Bulk!.RetryCount >= MaxTransientFailures) + current.Finish(BackgroundTaskStatus.Failed, current.Message, UtcNow); + + return true; + }); + + if (failed is { IsActive: false }) + { + Log.LogError(ex, "Bulk operation {TaskId} failed after repeated errors: {Message}", workItem.TaskId, ex.Message); + return; + } + + Log.LogWarning(ex, "Bulk operation {TaskId} hit a temporary error: {Message}", workItem.TaskId, ex.Message); + if (failed is null) + { + await _service.PublishLostAsync(workItem); + return; + } + + try + { + if (_options.RetryDelay > TimeSpan.Zero) + await Task.Delay(_options.RetryDelay * failed.Bulk!.RetryCount, _timeProvider, context.CancellationToken); + + await _service.ContinueAsync(workItem); + } + catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested) + { + context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {workItem.TaskId} for shutdown"); + } + catch (Exception continuationException) + { + Log.LogError(continuationException, "Bulk operation {TaskId} could not queue another run: {Message}", workItem.TaskId, continuationException.Message); + await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The operation stopped because another run could not be queued."); + } + } + catch (Exception ex) + { + Log.LogError(ex, "Bulk operation {TaskId} failed: {Message}", workItem.TaskId, ex.Message); + await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The operation stopped because of an unexpected error."); + } + finally + { + try + { + // FinishAsync and enqueue acknowledgement races can change the stored outcome without changing the local + // task. Only terminal/lost work releases its selection; active continuation and shutdown must retain it. + var saved = await _taskService.GetAsync(workItem.TaskId); + if (saved?.IsActive != true) + { + var finished = saved ?? selectionTask; + if (finished is null && workItem.Type is not null) + { + // The selection may outlive a lost task record. The work item still identifies its resolver. + finished = new BackgroundTask + { + Id = workItem.TaskId, + OrganizationId = workItem.OrganizationId, + Type = workItem.Type, + Bulk = new BulkOperationDetails { Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter } } + }; + } + + if (finished is not null) + await _service.ReleaseSelectionAsync(finished, pinnedSelection, latestSelectionCursor); + } + } + catch (Exception ex) + { + // A bounded snapshot lease is the fallback when the task store or search backend is unavailable. + Log.LogWarning(ex, "Could not check bulk operation {TaskId} for selection cleanup: {Message}", workItem.TaskId, ex.Message); + } + } + } + + private async Task SaveBatchAsync(BackgroundTask task, BulkSelectionPage page, BulkBatchResult result) + { + // Keep the known result while retrying its checkpoint, including during shutdown. Replaying a successful delete + // would classify its now-missing ids as failures or omit them from a filter selection's counts. A process stopping + // before any checkpoint is saved can still lose that result; the deletion and task record are separate writes. + try + { + return await _checkpointPolicy.ExecuteAsync(async _ => await _taskService.UpdateAsync(task.Id, current => + { + // A save can succeed before releasing its update lock throws. The prior cursor makes another attempt harmless + // if that checkpoint was committed already, or if the task has since advanced or finished. + if (!current.IsActive || !String.Equals(current.Bulk!.Cursor, task.Bulk!.Cursor, StringComparison.Ordinal)) + return false; + + current.Processed += page.Ids.Count; + current.Succeeded += result.Succeeded; + current.Failed += result.Failed; + current.Bulk.Cursor = page.NextCursor ?? ExhaustedCursor; + current.Bulk.RetryCount = 0; + current.Total = Math.Max(current.Total, current.Processed); + current.Message = result.Message; + current.Progress = GetPercent(current); + return true; + }), CancellationToken.None); + } + catch (Exception ex) + { + // Replaying a batch known to have succeeded would lose its result again. Stop if its bounded checkpoint + // retries are exhausted, keeping the last durable counts and making the uncertain outcome explicit. + throw new BatchCheckpointException(ex); + } + } + + /// + /// Errors that retrying cannot fix. Everything else (Elasticsearch, cache and network failures surface as many different + /// exception types) is treated as temporary and bounded by . + /// + internal static bool IsPermanent(Exception exception) + { + for (var current = exception; current is not null; current = current.InnerException) + { + if (current is DocumentLimitExceededException or ArgumentException or InvalidOperationException or NotSupportedException or FormatException or JsonException or InvalidCastException or NullReferenceException or KeyNotFoundException) + return true; + } + + return false; + } + + private async Task GetPageAsync(BackgroundTask task, IBulkSelectionResolver resolver, string? pinnedSelection, CancellationToken cancellationToken) + { + int size = Math.Max(1, _options.BatchSize); + if (task.Bulk!.Cursor == ExhaustedCursor) + return new BulkSelectionPage([], null); + + if (task.Bulk!.Selection.Kind is not BulkOperationSelectionKind.Ids) + return await resolver.GetPageAsync(task, pinnedSelection, task.Bulk!.Cursor, size, cancellationToken); + + // The cursor of an id selection is the index of the next id to process. + var ids = task.Bulk!.Selection.Ids ?? []; + int start = Int32.TryParse(task.Bulk!.Cursor, out int parsed) ? parsed : 0; + var batch = ids.Skip(start).Take(size).ToList(); + int next = start + batch.Count; + return new BulkSelectionPage(batch, next < ids.Count ? next.ToString() : null); + } + + private DateTime UtcNow => _timeProvider.GetUtcNow().UtcDateTime; + + private async Task FinishAsync(BulkOperationWorkItem workItem, BackgroundTaskStatus status, string message) + { + var task = await _taskService.UpdateAsync(workItem.TaskId, current => + { + if (!current.IsActive) + return false; + + current.Finish(status, message, UtcNow); + return true; + }); + + if (task is null) + await _service.PublishLostAsync(workItem); + } + + private static int GetPercent(BackgroundTask task) + { + return task.Total <= 0 ? 0 : (int)Math.Min(99, task.Processed * 100 / task.Total); + } + + private sealed class BatchCheckpointException(Exception innerException) + : InvalidOperationException("The operation stopped because its progress could not be saved. Some items may have been processed.", innerException); +} diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs index f88d91c33f..f8611080de 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs @@ -84,18 +84,26 @@ public override async Task HandleItemAsync(WorkItemContext context) for (int p = 0; p < projectList.Count; p++) { + int projectEventCount = eventsPerProject + (p < remainder ? 1 : 0); + if (projectEventCount == 0) + continue; + if (context.CancellationToken.IsCancellationRequested) - break; + { + context.Result = JobResult.CancelledWithMessage("Stopping sample event generation for shutdown"); + return; + } var project = projectList[p]; - int projectEventCount = eventsPerProject + (p < remainder ? 1 : 0); - var events = generator.Generate(organization.Id, project.Id, projectEventCount, minDate, utcNow); foreach (var batch in events.Chunk(batchSize)) { if (context.CancellationToken.IsCancellationRequested) - break; + { + context.Result = JobResult.CancelledWithMessage("Stopping sample event generation for shutdown"); + return; + } await _eventPipeline.RunAsync(batch, organization, project, allowExtendedEventDateRange: true); totalProcessed += batch.Length; @@ -138,7 +146,10 @@ private async Task GenerateProjectSampleEventsAsync(WorkItemContext context, Ran foreach (var batch in events.Chunk(batchSize)) { if (context.CancellationToken.IsCancellationRequested) - break; + { + context.Result = JobResult.CancelledWithMessage("Stopping sample event generation for shutdown"); + return; + } await _eventPipeline.RunAsync(batch, organization, project, allowExtendedEventDateRange: true); totalProcessed += batch.Length; diff --git a/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs new file mode 100644 index 0000000000..504febcffb --- /dev/null +++ b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs @@ -0,0 +1,76 @@ +using System.Text.Json.Serialization; +using Exceptionless.Core.Models.BulkOperations; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Core.Models.BackgroundTasks; + +/// +/// Long-running work that a user can follow in the progress panel: a bulk action, a project data reset, sample data +/// generation. Stored in the cache with a limited lifetime and indexed per organization; it is intentionally not +/// persisted in Elasticsearch. Everything a bulk batch changes (counts and cursor) lives in this one record so a +/// batch is saved atomically. +/// +public class BackgroundTask : IIdentity, IOwnedByOrganization, IHaveDates +{ + public string Id { get; set; } = null!; + public string OrganizationId { get; set; } = null!; + public string? ProjectId { get; set; } + + /// What the task does, e.g. events.delete or projects.reset-data. + public string Type { get; set; } = null!; + + public string Description { get; set; } = null!; + public BackgroundTaskStatus Status { get; set; } = BackgroundTaskStatus.Queued; + + /// Percent complete, 0 to 100. + public int Progress { get; set; } + + /// Item counts. All zero for tasks that only report a percent. + public long Total { get; set; } + public long Processed { get; set; } + public long Succeeded { get; set; } + public long Failed { get; set; } + + public string? Message { get; set; } + + /// Whether the user can stop the task. Tasks that cannot be stopped midway are not cancellable. + public bool CanCancel { get; set; } + + public string CreatedByUserId { get; set; } = null!; + public DateTime CreatedUtc { get; set; } + public DateTime? StartedUtc { get; set; } + public DateTime? CompletedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } + + /// Only set for bulk actions: what they apply to and where they resume. + public BulkOperationDetails? Bulk { get; set; } + + [JsonIgnore] + public bool IsActive => Status is BackgroundTaskStatus.Queued or BackgroundTaskStatus.Running; + + /// Marks the task as running. The start time is only recorded the first time. + public void Start(DateTime utcNow) + { + Status = BackgroundTaskStatus.Running; + StartedUtc ??= utcNow; + } + + public void Finish(BackgroundTaskStatus status, string? message, DateTime utcNow) + { + Status = status; + Message = message; + CompletedUtc = utcNow; + if (status is BackgroundTaskStatus.Completed) + Progress = 100; + } +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum BackgroundTaskStatus +{ + Queued, + Running, + Completed, + Failed, + Cancelled +} diff --git a/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs new file mode 100644 index 0000000000..ff51c9a86b --- /dev/null +++ b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs @@ -0,0 +1,8 @@ +namespace Exceptionless.Core.Models.BackgroundTasks; + +/// The types of tasks that are not bulk actions. Bulk actions use the key of their handler, e.g. events.delete. +public static class BackgroundTaskTypes +{ + public const string ResetProjectData = "projects.reset-data"; + public const string GenerateProjectSampleData = "projects.sample-data"; +} diff --git a/src/Exceptionless.Core/Models/BulkOperations/BulkOperationSelection.cs b/src/Exceptionless.Core/Models/BulkOperations/BulkOperationSelection.cs new file mode 100644 index 0000000000..1bfec8a567 --- /dev/null +++ b/src/Exceptionless.Core/Models/BulkOperations/BulkOperationSelection.cs @@ -0,0 +1,50 @@ +using System.Text.Json.Serialization; + +namespace Exceptionless.Core.Models.BulkOperations; + +/// The part of a that only bulk actions have. +public class BulkOperationDetails +{ + public BulkOperationSelection Selection { get; set; } = null!; + public Dictionary Parameters { get; set; } = new(); + + /// Consecutive batches that failed with a temporary error. Reset by the next successful batch. + public int RetryCount { get; set; } + + /// Opaque position of the last batch that finished; the job resumes after it. + public string? Cursor { get; set; } +} + +/// The kind of item an operation targets. Each entity has one selection resolver. +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum BulkOperationEntity +{ + Events +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum BulkOperationSelectionKind +{ + Ids, + Filter +} + +/// +/// Either an explicit list of ids or a filter. A filter selection always carries the absolute UTC window that was +/// resolved when the operation was created, so items that arrive afterwards are never included. +/// +public class BulkOperationSelection +{ + public BulkOperationSelectionKind Kind { get; set; } + public List? Ids { get; set; } + public string? Filter { get; set; } + + /// The relative time expression the window was resolved from. Informational only. + public string? Time { get; set; } + + public DateTime? UtcStart { get; set; } + public DateTime? UtcEnd { get; set; } + + /// Items stored after this instant (UTC) are never part of a filter selection. + public DateTime? UtcCreatedBefore { get; set; } +} diff --git a/src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs b/src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs new file mode 100644 index 0000000000..d16f69c23e --- /dev/null +++ b/src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs @@ -0,0 +1,25 @@ +using Exceptionless.Core.Models.BackgroundTasks; + +namespace Exceptionless.Core.Messaging.Models; + +/// +/// Published after every progress report and every status change of a background task. Intentionally independent of the +/// transport that delivers it to clients, and free of the selection and parameters. +/// +public record BackgroundTaskUpdated +{ + public required string Id { get; set; } + public required string OrganizationId { get; set; } + public string? ProjectId { get; set; } + public required string Type { get; set; } + public required string Description { get; set; } + public BackgroundTaskStatus Status { get; set; } + public int Progress { get; set; } + public long Total { get; set; } + public long Processed { get; set; } + public long Succeeded { get; set; } + public long Failed { get; set; } + public string? Message { get; set; } + public bool CanCancel { get; set; } + public DateTime UpdatedUtc { get; set; } +} diff --git a/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs new file mode 100644 index 0000000000..523bd67151 --- /dev/null +++ b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs @@ -0,0 +1,15 @@ +namespace Exceptionless.Core.Models.WorkItems; + +public record BulkOperationWorkItem : IBackgroundTaskWorkItem +{ + public required string TaskId { get; init; } + + /// + /// The organization, type and description are carried so a final state can still be published if the task + /// record is gone when the job runs. + /// + public required string OrganizationId { get; init; } + + public string? Type { get; init; } + public string? Description { get; init; } +} diff --git a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs index 4bf9a8db67..ecc627b735 100644 --- a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs +++ b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs @@ -1,9 +1,10 @@ namespace Exceptionless.Core.Models.WorkItems; -public record GenerateSampleEventsWorkItem +public record GenerateSampleEventsWorkItem : IBackgroundTaskWorkItem { public string? OrganizationId { get; init; } public string? ProjectId { get; init; } public int EventCount { get; init; } = 250; public int DaysBack { get; init; } = 7; + public string? TaskId { get; init; } } diff --git a/src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs new file mode 100644 index 0000000000..3c3f7d499d --- /dev/null +++ b/src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs @@ -0,0 +1,10 @@ +namespace Exceptionless.Core.Models.WorkItems; + +/// +/// A work item whose progress is tracked by a background task, so the user can follow it in the progress panel. +/// +public interface IBackgroundTaskWorkItem +{ + /// The tracking task, or null when the work item was queued without one. + string? TaskId { get; } +} diff --git a/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs index 4d4410c6c0..929a05b55b 100644 --- a/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs +++ b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs @@ -1,7 +1,8 @@ namespace Exceptionless.Core.Models.WorkItems; -public record ResetProjectDataWorkItem +public record ResetProjectDataWorkItem : IBackgroundTaskWorkItem { public required string OrganizationId { get; init; } public required string ProjectId { get; init; } + public string? TaskId { get; init; } } diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs index 43065cc189..a41f8c7dc2 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs @@ -8,6 +8,16 @@ namespace Exceptionless.Core.Repositories.Queries; public class EventFieldsQueryVisitor : ChainableQueryVisitor { public override async Task VisitAsync(GroupNode node, IQueryVisitorContext context) + { + node.Field = GetCustomFieldName(node.Field, GetGroupChildTerms(node)) ?? node.Field; + foreach (var child in node.Children) + await child.AcceptAsync(this, context); + } + + /// + /// The terms of a group's unfielded children, which pick the stored type of a custom field set on the group. + /// + internal static string[] GetGroupChildTerms(GroupNode node) { var childTerms = new List(); if (node.Left is TermNode { Field: null, Term: not null } leftTermNode) @@ -32,9 +42,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte childTerms.Add(rightTermRangeNode.Max); } - node.Field = GetCustomFieldName(node.Field, childTerms.ToArray()) ?? node.Field; - foreach (var child in node.Children) - await child.AcceptAsync(this, context); + return childTerms.ToArray(); } public override Task VisitAsync(TermNode node, IQueryVisitorContext context) @@ -67,7 +75,10 @@ public override Task VisitAsync(MissingNode node, IQueryVisitorContext context) return Task.CompletedTask; } - private string? GetCustomFieldName(string? field, string?[] terms) + /// + /// Maps a custom data or reference field to the stored idx field. The suffix is the stored type, picked from the terms. + /// + internal static string? GetCustomFieldName(string? field, string?[] terms) { if (String.IsNullOrEmpty(field)) return null; diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs new file mode 100644 index 0000000000..8c86a76275 --- /dev/null +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -0,0 +1,436 @@ +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Repositories.Utility; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services.BackgroundTasks; + +public class BackgroundTaskOptions +{ + /// How long one attempt to lock a task waits. A busy lock is retried by the resilience policy. + public TimeSpan LockAcquireTimeout { get; set; } = TimeSpan.FromSeconds(5); +} + +/// +/// Stores background tasks in the cache (with a limited lifetime), keeps a per-organization index and an index of the +/// active tasks, and publishes every change so the progress panel can follow along. +/// +public class BackgroundTaskService +{ + public static readonly TimeSpan TaskLifetime = TimeSpan.FromDays(7); + + /// An active task that has not reported progress for this long is considered abandoned. + public static readonly TimeSpan StaleAfter = TimeSpan.FromHours(2); + + /// How many finished tasks the list shows, in addition to all active tasks. + public const int ListLimit = 50; + + /// Every cache key and lock resource of background tasks starts with this prefix. + public const string KeyPrefix = "background-tasks"; + + /// Bounds task payloads loaded together, including selections attached to bulk tasks. + internal const int ReadBatchSize = 50; + private static readonly TimeSpan UpdateLockLifetime = TimeSpan.FromSeconds(30); + private readonly ICacheClient _cache; + private readonly ILockProvider _lockProvider; + private readonly IMessagePublisher _publisher; + private readonly IQueue _workItemQueue; + private readonly IResiliencePolicy _lockPolicy; + private readonly BackgroundTaskOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public BackgroundTaskService(ICacheClient cache, ILockProvider lockProvider, IMessagePublisher publisher, IQueue workItemQueue, + IResiliencePolicyProvider resiliencePolicyProvider, BackgroundTaskOptions options, TimeProvider timeProvider, ILogger logger) + { + _cache = cache; + _lockProvider = lockProvider; + _publisher = publisher; + _workItemQueue = workItemQueue; + _options = options; + _timeProvider = timeProvider; + _logger = logger; + + // Several workers and web instances update the same record, so a busy lock is normal: wait a little, then try again. + _lockPolicy = resiliencePolicyProvider.GetPolicy(b => b.WithMaxAttempts(5).WithLinearDelay(TimeSpan.FromMilliseconds(250)), logger, timeProvider); + } + + public static string GetTaskCacheKey(string id) => $"{KeyPrefix}:{id}"; + public static string GetCancelCacheKey(string id) => $"{KeyPrefix}:{id}:cancel"; + public static string GetOrganizationIndexCacheKey(string organizationId) => $"{KeyPrefix}:organization:{organizationId}"; + public static string GetActiveIndexCacheKey(string organizationId) => $"{KeyPrefix}:organization:{organizationId}:active"; + public static string GetUpdateLockResource(string id) => $"{KeyPrefix}:update:{id}"; + public static string GetCreateLockResource(string organizationId) => $"{KeyPrefix}:create:{organizationId}"; + + public async Task GetAsync(string id) + { + if (String.IsNullOrEmpty(id)) + return null; + + var cached = await _cache.GetAsync(GetTaskCacheKey(id)); + return cached.HasValue ? cached.Value : null; + } + + /// + /// Returns the existing tasks in input order, once per id. Missing records are omitted. Reads are batched so a + /// remote cache can retrieve several records together; callers remain responsible for checking their scope. + /// + public async Task> GetByIdsAsync(IEnumerable ids) + { + var tasks = new List(); + foreach (var keys in ids.Distinct(StringComparer.Ordinal).Select(GetTaskCacheKey).Chunk(ReadBatchSize)) + { + var cached = await _cache.GetAllAsync(keys); + foreach (string key in keys) + { + if (cached.TryGetValue(key, out var value) && value.HasValue && value.Value is { } task) + tasks.Add(task); + } + } + + return tasks; + } + + /// + /// All active tasks and the newest finished tasks of the organization, newest first. Only finished history is limited. + /// + public async Task> GetByOrganizationAsync(string organizationId, int limit = ListLimit) + { + limit = Math.Max(0, limit); + var activeIds = (await GetActiveIdsAsync(organizationId)).ToHashSet(StringComparer.Ordinal); + // One snapshot avoids repeated full-list cloning in memory and offset shifts while entries expire. + var indexed = await _cache.GetListAsync(GetOrganizationIndexCacheKey(organizationId)); + // ObjectIds order whole seconds; their machine/process bytes can reverse creation order within a second. + var newestIds = (indexed.HasValue ? indexed.Value : []) + .Concat(activeIds) + .Distinct(StringComparer.Ordinal) + .OrderByDescending(id => id, StringComparer.Ordinal); + + var tasks = new List(); + var finished = new List(); + string? cutoffSecond = null; + // Read the entire cutoff second before choosing its newest tasks by CreatedUtc, including across cache batches. + foreach (var batch in newestIds.Where(id => finished.Count < limit || activeIds.Contains(id) + || (cutoffSecond is not null && id.StartsWith(cutoffSecond, StringComparison.Ordinal))).Chunk(ReadBatchSize)) + { + var found = await GetByIdsAsync(batch); + var expired = batch.Except(found.Select(task => task.Id), StringComparer.Ordinal).ToArray(); + foreach (var task in found) + { + if (!String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal)) + continue; + + if (task.IsActive) + tasks.Add(task); + else + { + finished.Add(task); + if (finished.Count == limit) + cutoffSecond = task.Id[..8]; + } + } + + // A burst within one second must not retain every task's payload while its remaining batches are read. + if (finished.Count > limit) + { + finished.Sort(CompareNewestFirst); + finished.RemoveRange(limit, finished.Count - limit); + } + + if (expired.Length > 0) + await PruneIndexesAsync(organizationId, expired); + } + + tasks.AddRange(finished); + tasks.Sort(CompareNewestFirst); + return tasks; + } + + /// The ids of the organization's tasks that have not finished. + public async Task> GetActiveIdsAsync(string organizationId) + { + // Completion removes ids concurrently. Read one snapshot so a removal cannot shift an unread id past a page offset. + var ids = await _cache.GetListAsync(GetActiveIndexCacheKey(organizationId)); + return ids.HasValue ? ids.Value.Distinct(StringComparer.Ordinal).ToList() : []; + } + + /// + /// Rechecks suspected stale ids under their update locks before removing index entries. Busy tasks are skipped for + /// a later read; active tasks and tasks owned by another organization are retained. Only missing tasks lose history. + /// + public async Task PruneIndexesAsync(string organizationId, IReadOnlyCollection ids) + { + foreach (string id in ids.Distinct(StringComparer.Ordinal)) + { + await using var updateLock = await _lockProvider.TryAcquireAsync(GetUpdateLockResource(id), UpdateLockLifetime, TimeSpan.Zero); + if (updateLock is null) + continue; + + var task = await GetAsync(id); + if (task is not null && (task.IsActive || !String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal))) + continue; + + if (task is null) + await _cache.ListRemoveAsync(GetOrganizationIndexCacheKey(organizationId), [id]); + + await _cache.ListRemoveAsync(GetActiveIndexCacheKey(organizationId), [id]); + } + } + + /// + /// Gives the task an id, stores it as queued and adds it to the indexes. Nothing is published or queued yet. + /// + public async Task AddAsync(BackgroundTask task) + { + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + task.Id = ObjectId.GenerateNewId(utcNow).ToString(); + task.Status = BackgroundTaskStatus.Queued; + task.CreatedUtc = utcNow; + task.UpdatedUtc = utcNow; + + await SaveAsync(task); + try + { + if (await _cache.ListAddAsync(GetOrganizationIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime) != 1) + throw new InvalidOperationException("The background task could not be added to the organization index."); + + if (await _cache.ListAddAsync(GetActiveIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime) != 1) + throw new InvalidOperationException("The background task could not be added to the active index."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to index background task {TaskId}: {Message}", task.Id, ex.Message); + await FailToStartAsync(task.Id); + throw; + } + } + + /// + /// Adds a task and queues the work item that runs it. The work item is created after the task has its id. + /// If preparation fails, the task is marked failed and no work is queued. + /// + public async Task CreateAndEnqueueAsync(BackgroundTask task, Func createWorkItem) where T : class + { + await AddAsync(task); + + T workItem; + try + { + workItem = createWorkItem(task); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to prepare background task {TaskId}: {Message}", task.Id, ex.Message); + await FailToStartAsync(task.Id); + throw; + } + + await EnqueueAsync(task, workItem); + return task; + } + + /// + /// Prepares and queues the work item of a task that was added and publishes it. When preparation or queueing fails, + /// a task that is still queued is failed and the error is rethrown. Worker progress and cancellation are preserved. + /// + public async Task EnqueueAsync(BackgroundTask task, T workItem, Func? beforeEnqueue = null) where T : class + { + try + { + if (beforeEnqueue is not null) + await beforeEnqueue(); + + await _workItemQueue.EnqueueAsync(workItem); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to queue background task {TaskId}: {Message}", task.Id, ex.Message); + await FailToStartAsync(task.Id); + throw; + } + + await PublishAsync(task); + } + + /// + /// Queues another run of a work item, e.g. to continue a long task after a temporary error. The queue's retries cover a + /// whole work item, so continuing in a new work item does not use up the retries of the current one. + /// + public Task ContinueAsync(T workItem) where T : class + { + return _workItemQueue.EnqueueAsync(workItem); + } + + /// + /// Loads the task, lets change it, then saves and publishes it. The change is skipped when + /// returns false. Returns the stored task, or null when it no longer exists. + /// + /// + /// Tasks live in the cache rather than in Elasticsearch, so the repository's PatchAsync does not apply. The lock makes the + /// load, change and save one step; a busy lock is retried with the resilience policy before giving up. + /// + public async Task UpdateAsync(string id, Func update) + { + BackgroundTask? task; + await using (var updateLock = await _lockPolicy.ExecuteAsync(async ct => + await _lockProvider.AcquireAsync(GetUpdateLockResource(id), UpdateLockLifetime, _options.LockAcquireTimeout), CancellationToken.None)) + { + task = await GetAsync(id); + if (task is null) + return null; + + var previousUpdatedUtc = task.UpdatedUtc; + if (!update(task)) + { + // A declined mutation returns the stored state even if the callback changed its copy. + return await GetAsync(id); + } + + // Clients use this stamp to reject delayed notifications, including when a clock repeats or moves backwards. + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + task.UpdatedUtc = utcNow > previousUpdatedUtc ? utcNow : previousUpdatedUtc.AddTicks(1); + await SaveAsync(task); + if (task.IsActive) + { + // Active discovery must live as long as progress, including tasks that run longer than TaskLifetime. + await RefreshIndexAsync(GetActiveIndexCacheKey(task.OrganizationId), task.Id); + } + else + { + // Renew history on completion, rather than rewriting the large history index on every progress report. + // Keep active membership until history is saved so a concurrent list cannot miss a long-running task. + await RefreshIndexAsync(GetOrganizationIndexCacheKey(task.OrganizationId), task.Id); + await _cache.ListRemoveAsync(GetActiveIndexCacheKey(task.OrganizationId), [task.Id]); + } + } + + // The state is committed; a slow publisher must not block another update or cancellation. + await PublishAsync(task); + return task; + } + + /// + /// Requests cancellation. A queued (or abandoned) task is cancelled right away; a running bulk task stops at the next + /// batch boundary, after the batch that is in flight finishes. Tasks that cannot be stopped are rejected while they are + /// alive, but an abandoned one can always be cleared. + /// + public async Task CancelAsync(string id) + { + var task = await GetAsync(id); + if (task is null) + return BackgroundTaskCancelResult.NotFound; + + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + if (!task.IsActive) + return BackgroundTaskCancelResult.Cancelled(task); + + bool isAbandoned = utcNow - task.UpdatedUtc >= StaleAfter; + if (!task.CanCancel && !isAbandoned) + return BackgroundTaskCancelResult.NotCancellable(task); + + if (!await _cache.SetAsync(GetCancelCacheKey(id), true, TaskLifetime)) + throw new InvalidOperationException("The cancellation request could not be saved."); + + var updated = await UpdateAsync(id, current => + { + if (!current.IsActive) + return false; + + bool isRunningNow = current.Status is BackgroundTaskStatus.Running && utcNow - current.UpdatedUtc < StaleAfter; + if (isRunningNow) + return false; + + current.Finish(BackgroundTaskStatus.Cancelled, current.Status is BackgroundTaskStatus.Queued ? "Cancelled before it started." : "Cancelled after it stopped responding.", utcNow); + return true; + }); + + return updated is null ? BackgroundTaskCancelResult.NotFound : BackgroundTaskCancelResult.Cancelled(updated); + } + + public async Task IsCancelRequestedAsync(string id) + { + return (await _cache.GetAsync(GetCancelCacheKey(id))).HasValue; + } + + private async Task SaveAsync(BackgroundTask task) + { + if (!await _cache.SetAsync(GetTaskCacheKey(task.Id), task, TaskLifetime)) + throw new InvalidOperationException("The background task could not be saved."); + } + + public Task PublishAsync(BackgroundTask task) + { + return PublishAsync(new BackgroundTaskUpdated + { + Id = task.Id, + OrganizationId = task.OrganizationId, + ProjectId = task.ProjectId, + Type = task.Type, + Description = task.Description, + Status = task.Status, + Progress = task.Progress, + Total = task.Total, + Processed = task.Processed, + Succeeded = task.Succeeded, + Failed = task.Failed, + Message = task.Message, + CanCancel = task.CanCancel, + UpdatedUtc = task.UpdatedUtc + }); + } + + public async Task PublishAsync(BackgroundTaskUpdated message) + { + try + { + await _publisher.PublishAsync(message); + } + catch (Exception ex) + { + // No caller token is passed, so a transport cancellation is also an advisory notification failure. + _logger.LogWarning(ex, "Failed to publish background task update for {TaskId}: {Message}", message.Id, ex.Message); + } + } + + private Task FailToStartAsync(string id) + { + return UpdateAsync(id, current => + { + // Queueing may have succeeded before its acknowledgement failed, allowing a worker or user to advance the task. + if (current.Status is not BackgroundTaskStatus.Queued) + return false; + + current.Finish(BackgroundTaskStatus.Failed, "The task could not be started. Please try again.", _timeProvider.GetUtcNow().UtcDateTime); + return true; + }); + } + + private async Task RefreshIndexAsync(string key, string id) + { + await _cache.ListAddAsync(key, [id], TaskLifetime); + // Refreshing an existing list member does not extend the key TTL in every cache provider. + await _cache.SetExpirationAsync(key, TaskLifetime); + } + + private static int CompareNewestFirst(BackgroundTask left, BackgroundTask right) + { + int byCreated = right.CreatedUtc.CompareTo(left.CreatedUtc); + return byCreated != 0 ? byCreated : StringComparer.Ordinal.Compare(right.Id, left.Id); + } +} + +public record BackgroundTaskCancelResult +{ + public BackgroundTask? Task { get; init; } + public bool IsNotCancellable { get; init; } + + public static readonly BackgroundTaskCancelResult NotFound = new(); + public static BackgroundTaskCancelResult Cancelled(BackgroundTask task) => new() { Task = task }; + public static BackgroundTaskCancelResult NotCancellable(BackgroundTask task) => new() { Task = task, IsNotCancellable = true }; +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs new file mode 100644 index 0000000000..283b96b3e8 --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -0,0 +1,296 @@ +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; +using Foundatio.Caching; +using Foundatio.Lock; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Exceptionless.Core.Services.BulkOperations; + +public class BulkOperationOptions +{ + /// How many ids are resolved and executed per batch. + public int BatchSize { get; set; } = 500; + + /// How long to wait before continuing after a temporary error. Multiplied by the number of errors in a row. + public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(5); +} + +/// +/// The bulk specific part of background tasks: the registry of handlers and selection resolvers, counting and pinning a +/// selection, and the one-active-destructive-task-per-project rule. Storage, indexes and notifications are +/// . +/// +public class BulkOperationService +{ + private static readonly TimeSpan CreateLockLifetime = TimeSpan.FromSeconds(30); + private readonly BackgroundTaskService _taskService; + private readonly ICacheClient _cache; + private readonly ILockProvider _lockProvider; + private readonly IReadOnlyDictionary _handlers; + private readonly IReadOnlyDictionary _resolvers; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public BulkOperationService(BackgroundTaskService taskService, ICacheClient cache, ILockProvider lockProvider, + IEnumerable handlers, IEnumerable resolvers, TimeProvider timeProvider, ILogger? logger = null) + { + _taskService = taskService; + _cache = cache; + _lockProvider = lockProvider; + _handlers = handlers.ToDictionary(h => h.Type, StringComparer.Ordinal); + _resolvers = resolvers.ToDictionary(r => r.Entity); + _timeProvider = timeProvider; + _logger = logger ?? NullLogger.Instance; + } + + public static string GetPinnedSelectionCacheKey(string id) => $"{BackgroundTaskService.KeyPrefix}:{id}:selection"; + + public IBulkOperationHandler? GetHandler(string type) => _handlers.GetValueOrDefault(type); + + /// + /// Queues another run that continues from the saved cursor. The queue's retries cover a whole work item, so a long task + /// continues in a new work item after a temporary error instead of using up the retries of the current one. + /// + public Task ContinueAsync(BulkOperationWorkItem workItem) => _taskService.ContinueAsync(workItem); + + public async Task GetPinnedSelectionAsync(string id) + { + var cached = await _cache.GetAsync(GetPinnedSelectionCacheKey(id)); + return cached.HasValue ? cached.Value : null; + } + + /// Keeps an active task's immutable selection available without rewriting its payload. + public Task RefreshSelectionExpirationAsync(BackgroundTask task) + { + return task.Bulk?.Selection.Kind is BulkOperationSelectionKind.Filter + ? _cache.SetExpirationAsync(GetPinnedSelectionCacheKey(task.Id), BackgroundTaskService.TaskLifetime) + : Task.CompletedTask; + } + + /// Releases a rejected or terminal task's selection without changing its durable outcome. + public async Task ReleaseSelectionAsync(BackgroundTask task, string? pinnedSelection = null, string? cursor = null) + { + if (task.Bulk?.Selection.Kind is not BulkOperationSelectionKind.Filter + || GetHandler(task.Type) is not { } handler || !_resolvers.TryGetValue(handler.Entity, out var resolver)) + return; + + try + { + pinnedSelection ??= await GetPinnedSelectionAsync(task.Id); + await resolver.ReleaseAsync(task, pinnedSelection, cursor ?? task.Bulk.Cursor); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not release the selection for bulk operation {TaskId}: {Message}", task.Id, ex.Message); + } + } + + /// + /// Stores and enqueues the task unless it conflicts with an active destructive task on the same project. + /// Admission uses a renewed per-organization lease; a scan that outlives its lease returns busy. + /// + public async Task CreateAsync(BackgroundTask task) + { + var handler = GetHandler(task.Type) ?? throw new ArgumentException($"Unknown bulk operation type '{task.Type}'.", nameof(task)); + var bulk = task.Bulk ?? throw new ArgumentException("A bulk task needs its bulk details.", nameof(task)); + + task.CanCancel = true; + + string? pinnedSelection = null; + bool enqueueAttempted = false; + bool retainSelection = false; + try + { + // Count and pin before taking the lock, so a slow search never holds up other requests. + if (bulk.Selection.Kind is BulkOperationSelectionKind.Filter) + { + if (!_resolvers.TryGetValue(handler.Entity, out var resolver)) + throw new InvalidOperationException($"No selection resolver is registered for {handler.Entity}."); + + task.CreatedUtc = _timeProvider.GetUtcNow().UtcDateTime; + var preparation = await resolver.PrepareAsync(task, CancellationToken.None); + pinnedSelection = preparation.PinnedState; + if (!preparation.Check.IsOk) + return BulkOperationCreateResult.Rejected(preparation.Check); + + task.Total = preparation.Total; + } + else + { + task.Total = bulk.Selection.Ids?.Count ?? 0; + } + + task.Description = handler.Describe(task); + + long leaseStarted = _timeProvider.GetTimestamp(); + await using var createLock = await _lockProvider.TryAcquireAsync(BackgroundTaskService.GetCreateLockResource(task.OrganizationId), CreateLockLifetime, TimeSpan.FromSeconds(10)); + if (createLock is null) + return BulkOperationCreateResult.Busy(); + + async Task RenewCreateLockAsync() + { + long renewalStarted = _timeProvider.GetTimestamp(); + if (_timeProvider.GetElapsedTime(leaseStarted, renewalStarted) >= CreateLockLifetime) + return false; + + await createLock.RenewAsync(CreateLockLifetime); + if (_timeProvider.GetElapsedTime(leaseStarted) >= CreateLockLifetime) + return false; + + leaseStarted = renewalStarted; + return true; + } + + // Start conservatively before acquisition, then establish a fresh lease. A slow scan may renew it, but an + // expired request must not admit work from its earlier snapshot after another creator has acquired the lock. + if (!await RenewCreateLockAsync()) + return BulkOperationCreateResult.Busy(); + + if (handler.IsDestructive) + { + var conflict = await FindConflictingTaskAsync(task, RenewCreateLockAsync); + if (conflict is not null) + return conflict; + } + + if (!await RenewCreateLockAsync()) + return BulkOperationCreateResult.Busy(); + + await _taskService.AddAsync(task); + enqueueAttempted = true; + await _taskService.EnqueueAsync(task, new BulkOperationWorkItem + { + Description = task.Description, + TaskId = task.Id, + OrganizationId = task.OrganizationId, + Type = task.Type + }, async () => + { + // A selection that cannot be saved must fail the task just like an enqueue failure, so an unqueued task + // cannot remain active and block the project's next operation. + if (pinnedSelection is not null && !await _cache.SetAsync(GetPinnedSelectionCacheKey(task.Id), pinnedSelection, BackgroundTaskService.TaskLifetime)) + throw new InvalidOperationException("The operation's selection could not be saved."); + }); + + retainSelection = true; + return BulkOperationCreateResult.Created(task); + } + finally + { + if (!retainSelection && pinnedSelection is not null) + { + var cleanupTask = task; + if (enqueueAttempted) + { + try + { + // An enqueue can succeed before its acknowledgement is lost. The worker's saved state wins; + // never close an accepted active task's snapshot just because the creating request failed. + var saved = await _taskService.GetAsync(task.Id); + retainSelection = saved?.IsActive == true; + cleanupTask = saved ?? task; + } + catch (Exception ex) + { + retainSelection = true; + _logger.LogWarning(ex, "Could not check bulk operation {TaskId} before releasing its selection: {Message}", task.Id, ex.Message); + } + } + + if (!retainSelection) + await ReleaseSelectionAsync(cleanupTask, pinnedSelection); + } + } + } + + /// + /// Tells clients that a task can no longer run because its record is gone (for example the cache was cleared). + /// The work item carries enough to publish a final state so the progress panel does not wait forever. + /// + public Task PublishLostAsync(BulkOperationWorkItem workItem) + { + return _taskService.PublishAsync(new BackgroundTaskUpdated + { + Description = workItem.Description ?? "Bulk operation", + Id = workItem.TaskId, + Message = "The operation's progress was lost and it could not continue.", + OrganizationId = workItem.OrganizationId, + Status = BackgroundTaskStatus.Failed, + Type = workItem.Type ?? String.Empty, + UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime + }); + } + + private async Task FindConflictingTaskAsync(BackgroundTask candidate, Func> renewCreateLock) + { + // Only active tasks can conflict, and every one of them is in the active index no matter how much history + // the organization has. + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + BackgroundTask? conflict = null; + var activeIds = await _taskService.GetActiveIdsAsync(candidate.OrganizationId); + foreach (var batch in activeIds.Chunk(BackgroundTaskService.ReadBatchSize)) + { + if (!await renewCreateLock()) + return BulkOperationCreateResult.Busy(); + + var tasks = await _taskService.GetByIdsAsync(batch); + var finished = batch.Except(tasks.Where(task => task.IsActive).Select(task => task.Id), StringComparer.Ordinal).ToArray(); + foreach (var active in tasks.Where(task => task.IsActive)) + { + var existing = active; + if (GetHandler(existing.Type) is not { IsDestructive: true }) + continue; + + // An organization wide task can touch every project, so it conflicts with any project's task. + bool overlaps = existing.ProjectId is null || candidate.ProjectId is null || String.Equals(existing.ProjectId, candidate.ProjectId, StringComparison.Ordinal); + if (!overlaps) + continue; + + if (utcNow - existing.UpdatedUtc >= BackgroundTaskService.StaleAfter) + { + if (!await renewCreateLock()) + return BulkOperationCreateResult.Busy(); + + // Retire it before admitting a replacement, so a delayed delivery cannot run it alongside the new one. + var retired = await _taskService.UpdateAsync(existing.Id, current => + { + if (!current.IsActive || utcNow - current.UpdatedUtc < BackgroundTaskService.StaleAfter) + return false; + + current.Finish(BackgroundTaskStatus.Failed, "Stopped because it made no progress for two hours.", utcNow); + return true; + }); + + if (retired is null || !retired.IsActive) + continue; + + existing = retired; + } + + conflict ??= existing; + } + + if (finished.Length > 0) + await _taskService.PruneIndexesAsync(candidate.OrganizationId, finished); + } + + return conflict is null ? null : BulkOperationCreateResult.Conflicting(conflict); + } +} + +public record BulkOperationCreateResult +{ + public BackgroundTask? Task { get; init; } + public BackgroundTask? Conflict { get; init; } + public bool IsBusy { get; init; } + public BulkOperationCheckResult? Rejection { get; init; } + + public static BulkOperationCreateResult Created(BackgroundTask task) => new() { Task = task }; + public static BulkOperationCreateResult Conflicting(BackgroundTask conflict) => new() { Conflict = conflict }; + public static BulkOperationCreateResult Busy() => new() { IsBusy = true }; + public static BulkOperationCreateResult Rejected(BulkOperationCheckResult rejection) => new() { Rejection = rejection }; +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs new file mode 100644 index 0000000000..148414a812 --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs @@ -0,0 +1,73 @@ +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Foundatio.Repositories; + +namespace Exceptionless.Core.Services.BulkOperations; + +/// +/// Deletes events. Each batch goes through , the same code the single-item +/// DELETE /events/{ids} endpoint uses, so usage accounting and notifications are identical. +/// +public class DeleteEventsBulkOperationHandler : IBulkOperationHandler +{ + public const string OperationType = "events.delete"; + + private readonly IEventRepository _eventRepository; + private readonly EventDeletionService _eventDeletionService; + + public DeleteEventsBulkOperationHandler(IEventRepository eventRepository, EventDeletionService eventDeletionService) + { + _eventRepository = eventRepository; + _eventDeletionService = eventDeletionService; + } + + public string Type => OperationType; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => true; + + public Task AuthorizeAsync(BulkOperationRequest request) + { + // Same rule as DELETE /events/{ids}: any user of the organization (or a global admin) may delete its events. + if (!request.Actor.CanAccessOrganization(request.Organization.Id)) + return Task.FromResult(BulkOperationCheckResult.Denied("Organization not found.")); + + if (request.Project is not null && !String.Equals(request.Project.OrganizationId, request.Organization.Id, StringComparison.Ordinal)) + return Task.FromResult(BulkOperationCheckResult.Denied("Project not found.")); + + return Task.FromResult(BulkOperationCheckResult.Ok); + } + + public Task ValidateAsync(BulkOperationRequest request) + { + if (request.Parameters.Count > 0) + return Task.FromResult(BulkOperationCheckResult.Invalid("parameters", $"The {Type} operation does not take parameters.")); + + return Task.FromResult(BulkOperationCheckResult.Ok); + } + + public string Describe(BackgroundTask task) + { + return task.Bulk!.Selection.Kind is BulkOperationSelectionKind.Ids + ? $"Delete {task.Total:N0} selected events" + : $"Delete {task.Total:N0} events matching filter"; + } + + public async Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) + { + // Deletion needs scope, daily-index routing and notification fields, not the event's potentially large payload. + var events = await _eventRepository.GetByIdsAsync(ids.ToArray(), o => o.Cache(false) + .Include(e => e.Id, e => e.Date, e => e.OrganizationId, e => e.ProjectId, e => e.StackId)); + + // The task may only touch its own organization (and project), no matter what ids it was given. + var inScope = events + .Where(e => String.Equals(e.OrganizationId, task.OrganizationId, StringComparison.Ordinal) + && (task.ProjectId is null || String.Equals(e.ProjectId, task.ProjectId, StringComparison.Ordinal))) + .ToList(); + + await _eventDeletionService.DeleteAsync(inScope); + + int failed = ids.Count - inScope.Count; + return new BulkBatchResult(inScope.Count, failed, failed > 0 ? $"{failed:N0} events were not found." : null); + } +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs new file mode 100644 index 0000000000..c091aac15f --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -0,0 +1,374 @@ +using System.Globalization; +using Elastic.Clients.Elasticsearch.Mapping; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Base; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Repositories.Options; +using Exceptionless.Core.Repositories.Queries; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.DateTimeExtensions; +using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Extensions; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; +using Foundatio.Serializer; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services.BulkOperations; + +/// +/// Pages the ids of events that match an organization or project scope, a filter and an absolute UTC window, using the +/// same query building as the events list so the selection is exactly what the user was looking at. +/// +public class EventBulkSelectionResolver : IBulkSelectionResolver +{ + private const string CreatedUtcField = "created_utc"; + private const string PremiumSearchUpgradeMessage = "Please upgrade your plan to use premium search features."; + private const string SnapshotUnavailableMessage = "The saved event selection expired or is unavailable. Please start the operation again."; + private readonly IEventRepository _eventRepository; + private readonly IOrganizationRepository _organizationRepository; + private readonly IProjectRepository _projectRepository; + private readonly PersistentEventQueryValidator _validator; + private readonly ITextSerializer _serializer; + private readonly ExceptionlessElasticConfiguration _configuration; + private readonly AppOptions _appOptions; + private readonly BulkOperationOptions _options; + private readonly ILogger _logger; + + public EventBulkSelectionResolver(IEventRepository eventRepository, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, + PersistentEventQueryValidator validator, ITextSerializer serializer, ExceptionlessElasticConfiguration configuration, AppOptions appOptions, BulkOperationOptions options) + { + _eventRepository = eventRepository; + _organizationRepository = organizationRepository; + _projectRepository = projectRepository; + _validator = validator; + _serializer = serializer; + _configuration = configuration; + _appOptions = appOptions; + _options = options; + _logger = configuration.LoggerFactory.CreateLogger(); + } + + private ValueTask IsDateFieldAsync(string field) + { + return ValueTask.FromResult(_configuration.Events.MappingResolver.GetMappingProperty(field, followAlias: true) is DateProperty); + } + + public BulkOperationEntity Entity => BulkOperationEntity.Events; + + public async Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection) + { + var result = await _validator.ValidateQueryAsync(selection.Filter); + if (!result.IsValid) + return BulkOperationCheckResult.Invalid("selection.filter", result.Message ?? "Invalid filter."); + + if (result.UsesPremiumFeatures && !organization.HasPremiumFeatures) + return BulkOperationCheckResult.PlanLimit(PremiumSearchUpgradeMessage); + + return BulkOperationCheckResult.Ok; + } + + public async Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken) + { + var (appFilter, check) = await GetAppFilterAsync(task); + if (!check.IsOk) + return BulkSelectionPreparation.Rejected(check); + + if (appFilter is null) + return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, 0, + _serializer.SerializeToString(new PinnedEventSelection { Version = 1, IsEmpty = true })); + + // Counting and paging must use the same relative-date anchor, even when preparation runs after the selected day. + string? frozenFilter = await RelativeDateFreezer.FreezeAsync(task.Bulk!.Selection.Filter, + task.Bulk.Selection.UtcCreatedBefore ?? task.CreatedUtc, IsDateFieldAsync); + + var pinned = CreatePinnedSelection(appFilter); + // Resolve stack terms once and open an event snapshot in the same search that counts the selection. A created-at + // cutoff alone cannot pin mutable event fields such as the value of an active session or its user description. + var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection, pinned) + .FilterExpression(frozenFilter) + .Include(e => e.Id) + .SortExpression($"-{EventIndex.Alias.Date}") + .EnforceEventStackFilter(); + + string? pointInTimeId = null; + bool retainSnapshot = false; + try + { + cancellationToken.ThrowIfCancellationRequested(); + var results = await _eventRepository.FindAsync(query, SnapshotOptions(Math.Max(1, _options.BatchSize)).TrackTotalHits()); + pointInTimeId = results.GetPointInTimeId(); + cancellationToken.ThrowIfCancellationRequested(); + if (String.IsNullOrEmpty(pointInTimeId)) + return BulkSelectionPreparation.Rejected(SnapshotUnavailable()); + + pinned.EventFilter = query.GetFilterExpression(); + pinned.ExcludedStackIds = query.GetExcludedStacks().ToArray(); + pinned.StackIds = query.GetStacks().ToArray(); + pinned.Total = results.Total; + pinned.IsEmpty = results.Total == 0; + + string[] ids = GetIds(results); + if (results.Total <= ids.Length) + { + // The repository may already have closed a snapshot that fits on one page. Keep its complete, bounded + // membership instead of retaining a server resource for a small operation. + pinned.Ids = ids; + } + else + { + pinned.PointInTimeId = pointInTimeId; + } + + string state = _serializer.SerializeToString(pinned); + retainSnapshot = pinned.PointInTimeId is not null; + return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, results.Total, state); + } + catch (DocumentLimitExceededException ex) + { + return BulkSelectionPreparation.Rejected(BulkOperationCheckResult.Invalid("selection.filter", ex.Message)); + } + finally + { + if (!retainSnapshot) + await CloseSnapshotAsync(pointInTimeId, task.Id); + } + } + + public async Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) + { + var pinned = ReadPinnedSelection(pinnedState); + var position = String.IsNullOrEmpty(cursor) ? new SelectionCursor() : _serializer.Deserialize(cursor) + ?? throw new BulkSelectionRejectedException(SnapshotUnavailable()); + if (position.Position < 0) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + // A final checkpoint can survive a host restart before completion is recorded. There is no remaining action to + // authorize in that case, and a later plan change must not turn already completed work into a failed task. + if (position.Exhausted || pinned.IsEmpty || pinned.Ids is { Length: 0 }) + return new BulkSelectionPage([], null); + + var (appFilter, check) = await GetAppFilterAsync(task); + if (!check.IsOk) + throw new BulkSelectionRejectedException(check); + + if (appFilter is null) + return new BulkSelectionPage([], null); + + cancellationToken.ThrowIfCancellationRequested(); + size = Math.Max(1, size); + var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection, pinned).Include(e => e.Id); + + if (pinned.Ids is not null) + { + // Recheck current access policy, but never re-evaluate mutable user predicates. Search every remaining + // bounded id so a missing or newly excluded prefix does not hide eligible ids on a later page. + string[] remaining = pinned.Ids.Skip((int)Math.Min(position.Position, pinned.Ids.Length)).ToArray(); + if (remaining.Length == 0) + return new BulkSelectionPage([], null); + + var available = await _eventRepository.FindAsync(query.Id(remaining), new CommandOptions().PageLimit(remaining.Length).TrackTotalHits(false)); + var allowed = GetIds(available).ToHashSet(StringComparer.Ordinal); + string[] ids = remaining.Where(allowed.Contains).Take(size).ToArray(); + int next = ids.Length == 0 ? pinned.Ids.Length : Array.IndexOf(pinned.Ids, ids[^1]) + 1; + return new BulkSelectionPage(ids, _serializer.SerializeToString(new SelectionCursor { Position = next, Exhausted = next >= pinned.Ids.Length })); + } + + query.FilterExpression(pinned.EventFilter).SortExpression($"-{EventIndex.Alias.Date}"); + + if (pinned.StackIds.Length > 0) + query.Stack(pinned.StackIds); + + if (pinned.ExcludedStackIds.Length > 0) + query.ExcludeStack(pinned.ExcludedStackIds); + + // Always supply the saved snapshot id. Missing or expired state must never open a new live selection. + string pointInTimeId = position.PointInTimeId ?? pinned.PointInTimeId!; + if (String.IsNullOrEmpty(pointInTimeId) || (position.Position > 0 && String.IsNullOrEmpty(position.After))) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + var options = SnapshotOptions(size).PointInTimeId(pointInTimeId).SearchAfterToken(position.After, _serializer).TrackTotalHits(false); + var results = await _eventRepository.FindAsync(query, options); + // Foundatio represents a missing PIT (404) as an empty result. A valid PIT response includes its id even when + // current retention policy has narrowed the selection to no matches. + string? nextPointInTimeId = results.GetPointInTimeId(); + if (String.IsNullOrEmpty(nextPointInTimeId)) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + string[] pageIds = GetIds(results); + long nextPosition = position.Position + pageIds.Length; + string? after = results.Hits.LastOrDefault()?.GetSortToken(_serializer); + bool exhausted = !results.HasMore || nextPosition >= pinned.Total; + if (!exhausted && String.IsNullOrEmpty(after)) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + // Retain the latest PIT id even on the last page, so terminal cleanup and a restart after the final checkpoint + // can close the correct resource without replaying the selection. + return new BulkSelectionPage(pageIds, _serializer.SerializeToString(new SelectionCursor + { + After = after, + PointInTimeId = nextPointInTimeId, + Position = nextPosition, + Exhausted = exhausted + })); + } + + public async Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor) + { + if (pinnedState is null) + return; + + try + { + var pinned = _serializer.Deserialize(pinnedState); + if (String.IsNullOrEmpty(pinned?.PointInTimeId)) + return; + + var position = String.IsNullOrEmpty(cursor) ? null : _serializer.Deserialize(cursor); + await CloseSnapshotAsync(position?.PointInTimeId ?? pinned.PointInTimeId, task.Id); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not release the event selection for task {TaskId}: {Message}", task.Id, ex.Message); + } + } + + private async Task CloseSnapshotAsync(string? pointInTimeId, string taskId) + { + if (String.IsNullOrEmpty(pointInTimeId) || _eventRepository is not ISupportPointInTime repository) + return; + + try + { + await repository.ClosePointInTimeAsync(pointInTimeId); + } + catch (Exception ex) + { + // Expiry is the bounded fallback if best-effort cleanup cannot reach Elasticsearch. + _logger.LogWarning(ex, "Could not close the event snapshot for task {TaskId}: {Message}", taskId, ex.Message); + } + } + + private static CommandOptions SnapshotOptions(int size) => new CommandOptions() + .SearchAfterPaging(SearchAfterPagingMode.PointInTime) + // A queued task is stale after this idle period. Each page renews the same lease for active work and retries. + .SnapshotPagingLifetime(BackgroundTaskService.StaleAfter) + .PageLimit(size); + + private static string[] GetIds(FindResults results) => results.Hits + .Select(hit => hit.Id ?? throw new BulkSelectionRejectedException(SnapshotUnavailable())).ToArray(); + + private static BulkOperationCheckResult SnapshotUnavailable() => BulkOperationCheckResult.Invalid("selection", SnapshotUnavailableMessage); + + private PinnedEventSelection ReadPinnedSelection(string? state) + { + var pinned = state is null ? null : _serializer.Deserialize(state); + // Records created before snapshots were introduced contain only filters and stack ids. Replaying them would + // broaden the confirmed membership, so those tasks must be restarted instead of silently upgrading in place. + if (pinned is null || pinned.Version != 1 || (!pinned.IsEmpty && pinned.Ids is null && String.IsNullOrEmpty(pinned.PointInTimeId))) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + return pinned; + } + + private PinnedEventSelection CreatePinnedSelection(AppFilter appFilter) + { + var organization = appFilter.Organizations.Single(); + var project = appFilter.Projects?.SingleOrDefault(); + var timeProvider = _configuration.TimeProvider; + var utcNow = timeProvider.GetUtcNow().UtcDateTime; + var retentionDate = organization.GetRetentionUtcCutoff(_appOptions.MaximumRetentionDays, + project?.CreatedUtc.SafeSubtract(TimeSpan.FromDays(3)), timeProvider); + // Keep AppFilterQueryBuilder's date-math rounding, including the inclusive upper day's rounding. Pinning only + // the submitted time range could admit older events after a retention upgrade or project-scope change. + double retentionDays = Math.Max(Math.Round(Math.Abs(utcNow.Subtract(retentionDate).TotalDays), MidpointRounding.AwayFromZero), 1); + string anchor = utcNow.ToString("yyyy-MM-ddTHH:mm:ss.fff'Z'", CultureInfo.InvariantCulture) + "||"; + return new PinnedEventSelection + { + Version = 1, + RetentionStart = FormattableString.Invariant($"{anchor}/d-{(int)retentionDays}d"), + RetentionEnd = $"{anchor}/d+1d" + }; + } + + private static RepositoryQuery ApplyScope(RepositoryQuery query, AppFilter appFilter, BulkOperationSelection selection, PinnedEventSelection pinned) + { + query.AppFilter(appFilter); + if (pinned.RetentionStart is not null && pinned.RetentionEnd is not null) + query.ElasticFilter(new DateRangeQuery { Field = EventIndex.Alias.Date, Gte = pinned.RetentionStart, Lte = pinned.RetentionEnd }); + + if (selection.UtcStart.HasValue || selection.UtcEnd.HasValue) + { + query.DateRange(selection.UtcStart, selection.UtcEnd, EventIndex.Alias.Date) + .Index(selection.UtcStart, selection.UtcEnd); + } + + // The event date is chosen by the client, so an old-dated event can still arrive after the task was created. + if (selection.UtcCreatedBefore.HasValue) + query.DateRange(null, selection.UtcCreatedBefore, CreatedUtcField); + + return query; + } + + private async Task<(AppFilter? Filter, BulkOperationCheckResult Check)> GetAppFilterAsync(BackgroundTask task) + { + var organization = await _organizationRepository.GetByIdAsync(task.OrganizationId, o => o.Cache()); + if (organization is null) + return (null, BulkOperationCheckResult.Ok); + + var validation = await _validator.ValidateQueryAsync(task.Bulk!.Selection.Filter); + AppFilter appFilter; + if (!String.IsNullOrEmpty(task.ProjectId)) + { + var project = await _projectRepository.GetByIdAsync(task.ProjectId, o => o.Cache()); + if (project is null || !String.Equals(project.OrganizationId, organization.Id, StringComparison.Ordinal)) + return (null, BulkOperationCheckResult.Ok); + + appFilter = new AppFilter(project, organization); + } + else + { + appFilter = new AppFilter(organization); + } + + // AppFilter turns an unavailable premium search into no matches. Report the plan change explicitly so the + // worker cannot mistake it for an exhausted selection and claim the deletion completed. + if (validation.UsesPremiumFeatures && !organization.HasPremiumFeatures) + return (null, BulkOperationCheckResult.PlanLimit(PremiumSearchUpgradeMessage)); + + appFilter.UsesPremiumFeatures = validation.UsesPremiumFeatures; + return (appFilter, BulkOperationCheckResult.Ok); + } + + /// A bounded id set or the query and snapshot that fix the originally confirmed membership. + private sealed class PinnedEventSelection + { + public int Version { get; set; } + // Storage serialization omits empty arrays, so an intentionally empty membership needs its own marker. + public bool IsEmpty { get; set; } + public string[]? Ids { get; set; } + public string? PointInTimeId { get; set; } + public long Total { get; set; } + public string? RetentionStart { get; set; } + public string? RetentionEnd { get; set; } + public string? EventFilter { get; set; } + public string[] ExcludedStackIds { get; set; } = []; + public string[] StackIds { get; set; } = []; + } + + private sealed class SelectionCursor + { + public string? After { get; set; } + public string? PointInTimeId { get; set; } + public long Position { get; set; } + public bool Exhausted { get; set; } + } +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs new file mode 100644 index 0000000000..0cfd1f64eb --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs @@ -0,0 +1,110 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; + +namespace Exceptionless.Core.Services.BulkOperations; + +/// +/// One bulk action, e.g. events.delete. Adding a new bulk action means adding one implementation of this +/// interface; handlers are discovered and registered automatically. +/// +public interface IBulkOperationHandler +{ + /// The unique key that identifies the action in API requests and stored operations. + string Type { get; } + + /// The kind of item the action targets. Selects the that pages ids. + BulkOperationEntity Entity { get; } + + /// Destructive actions are limited to one active task per project. + bool IsDestructive { get; } + + /// Checks that the actor may run this action against the organization and project. + Task AuthorizeAsync(BulkOperationRequest request); + + /// Checks the action specific parameters and the selection shape. + Task ValidateAsync(BulkOperationRequest request); + + /// A short, human readable description shown while the task runs. + string Describe(BackgroundTask task); + + /// + /// Executes one batch of ids through the same code path as the single-item action, so side effects stay identical. + /// Ids that cannot be processed (missing, or outside the task's organization and project) must be reported as failed. + /// + /// + /// Handlers must tolerate at-least-once execution: a process can stop after the action but before its checkpoint is + /// saved, so the same ids may be delivered again. Missing ids must still be reported as failed. + /// + Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken); +} + +/// Pages the ids that match a filter selection. One implementation per . +public interface IBulkSelectionResolver +{ + BulkOperationEntity Entity { get; } + + /// Checks that the filter of a selection can be run for the organization (syntax, plan limits). + Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection); + + /// + /// Runs once when the task is created. Counts what matches and pins every part of the selection that could + /// otherwise change while the task runs, so pages never expand beyond what the user confirmed. Current access + /// policy may still narrow that membership. The pinned state is stored with the task for its lifetime. + /// + Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken); + + /// + /// Returns the next page of ids after . The cursor is a value based position, so + /// paging stays correct while earlier items are being deleted or changed. Paging must not count totals. + /// + Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken); + + /// + /// Best-effort release of selection resources when creation is rejected or a task is terminal. Active retries and + /// host shutdown keep their resources so they can resume. Implementations must tolerate duplicate cleanup. + /// + Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor) => Task.CompletedTask; +} + +public record BulkSelectionPreparation(BulkOperationCheckResult Check, long Total = 0, string? PinnedState = null) +{ + public static BulkSelectionPreparation Rejected(BulkOperationCheckResult check) => new(check); +} + +public record BulkSelectionPage(IReadOnlyList Ids, string? NextCursor); + +/// A previously accepted selection can no longer run with its saved membership or current access policy. +public sealed class BulkSelectionRejectedException(BulkOperationCheckResult check) : InvalidOperationException(check.Message) +{ + public BulkOperationCheckResult Check { get; } = check; +} + +public record BulkBatchResult(int Succeeded, int Failed, string? Message = null); + +/// The authenticated caller, independent of the transport the request arrived on. +public record BulkOperationActor(string UserId, bool IsGlobalAdmin, IReadOnlyCollection OrganizationIds) +{ + public bool CanAccessOrganization(string organizationId) => IsGlobalAdmin || OrganizationIds.Contains(organizationId, StringComparer.Ordinal); +} + +public record BulkOperationRequest(BulkOperationActor Actor, Organization Organization, Project? Project, string Type, BulkOperationSelection Selection, IReadOnlyDictionary Parameters); + +public enum BulkOperationCheckStatus +{ + Ok, + Denied, + Invalid, + PlanLimit +} + +public record BulkOperationCheckResult(BulkOperationCheckStatus Status, string? Field = null, string? Message = null) +{ + public static readonly BulkOperationCheckResult Ok = new(BulkOperationCheckStatus.Ok); + + public bool IsOk => Status is BulkOperationCheckStatus.Ok; + + public static BulkOperationCheckResult Denied(string message) => new(BulkOperationCheckStatus.Denied, null, message); + public static BulkOperationCheckResult Invalid(string field, string message) => new(BulkOperationCheckStatus.Invalid, field, message); + public static BulkOperationCheckResult PlanLimit(string message) => new(BulkOperationCheckStatus.PlanLimit, null, message); +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs new file mode 100644 index 0000000000..efe88962de --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs @@ -0,0 +1,141 @@ +using System.Globalization; +using Exceptionless.Core.Repositories.Queries; +using Foundatio.Parsers.LuceneQueries; +using Foundatio.Parsers.LuceneQueries.Nodes; +using Foundatio.Parsers.LuceneQueries.Visitors; + +namespace Exceptionless.Core.Services.BulkOperations; + +/// +/// Pins relative dates in a filter (for example data.date:[now/d-4d TO now/d+1d}) to a fixed time, so a bulk +/// operation keeps selecting the same events no matter when each batch runs. Only date fields are changed: a string range +/// such as environment:[now TO z] is left alone. +/// +public static class RelativeDateFreezer +{ + /// The filter to pin. + /// The time that now stands for. + /// Whether a built-in (non custom data) field is mapped as a date. + public static async Task FreezeAsync(string? filter, DateTime utcNow, Func> isDateFieldAsync) + { + if (String.IsNullOrWhiteSpace(filter) || filter.IndexOf("now", StringComparison.OrdinalIgnoreCase) < 0) + return filter; + + var result = await new LuceneQueryParser().ParseAsync(filter); + if (result is null) + return filter; + + await new FreezeRelativeDatesVisitor(utcNow, isDateFieldAsync).AcceptAsync(result, new QueryVisitorContext()); + return result.ToString(); + } + + private sealed class FreezeRelativeDatesVisitor(DateTime utcNow, Func> isDateFieldAsync) : ChainableQueryVisitor + { + // Elasticsearch date math anchors an expression on a date with ||, for example 2026-11-03T04:00:00.000Z||/d-4d. + private readonly string _anchor = utcNow.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ", CultureInfo.InvariantCulture) + "||"; + + public override async Task VisitAsync(GroupNode node, IQueryVisitorContext context) + { + // A custom field on a group is stored by the type of the group's terms, as the events index decides it. When that + // is a date, name the stored date field so the frozen bounds below keep the date type. + string? storedField = EventFieldsQueryVisitor.GetCustomFieldName(node.Field, EventFieldsQueryVisitor.GetGroupChildTerms(node)); + if (IsStoredDateField(storedField)) + node.Field = storedField; + + foreach (var child in node.Children) + await child.AcceptAsync(this, context); + } + + public override async Task VisitAsync(TermRangeNode node, IQueryVisitorContext context) + { + if (!IsRelative(node.Min) && !IsRelative(node.Max)) + return; + + if (!await EnsureDateFieldAsync(node, [node.Min, node.Max])) + return; + + if (IsRelative(node.Min)) + { + node.Min = _anchor + node.Min![3..]; + node.IsMinQuotedTerm = true; + } + + if (IsRelative(node.Max)) + { + node.Max = _anchor + node.Max![3..]; + node.IsMaxQuotedTerm = true; + } + } + + public override async Task VisitAsync(TermNode node, IQueryVisitorContext context) + { + if (node.IsRegexTerm || node.Proximity is not null || !IsRelative(node.Term) || node.Term!.IndexOfAny(['*', '?']) >= 0 + || !await EnsureDateFieldAsync(node, [node.Term])) + return; + + // Elasticsearch treats a date term such as now/d as an inclusive date range, so it needs the same anchor. + node.Term = _anchor + node.Term![3..]; + node.IsQuotedTerm = true; + } + + private async ValueTask EnsureDateFieldAsync(IFieldQueryNode node, string?[] terms) + { + // An unfielded term or range inside a group takes its field from the group. + string? field = node.Field ?? GetGroupField(node); + if (String.IsNullOrEmpty(field)) + return false; + + if (node.Field is null) + { + // The group already decided the stored field: a custom field left unchanged there is not a date. + if (!IsStoredDateField(field) && (IsCustomField(field) || !await isDateFieldAsync(field))) + return false; + } + else + { + // Custom data fields are stored by type. Freezing their original terms would hide the date type, + // so name the stored date field directly. + string? storedField = EventFieldsQueryVisitor.GetCustomFieldName(field, terms); + if (IsCustomField(field)) + { + if (!IsStoredDateField(storedField)) + return false; + + node.Field = storedField; + } + else if (!IsStoredDateField(field) && !await isDateFieldAsync(field)) + { + return false; + } + } + + return true; + } + + private static bool IsCustomField(string field) + { + return !String.Equals(EventFieldsQueryVisitor.GetCustomFieldName(field, []), field, StringComparison.Ordinal); + } + + private static bool IsStoredDateField(string? field) + { + return field is not null && field.StartsWith("idx.", StringComparison.OrdinalIgnoreCase) && field.EndsWith("-d", StringComparison.Ordinal); + } + + private static string? GetGroupField(IQueryNode node) + { + for (var parent = node.Parent; parent is not null; parent = parent.Parent) + { + if (parent is GroupNode { Field: { Length: > 0 } groupField }) + return groupField; + } + + return null; + } + + private static bool IsRelative(string? value) + { + return value is not null && value.StartsWith("now", StringComparison.Ordinal) && (value.Length == 3 || value[3] is '+' or '-' or '/'); + } + } +} diff --git a/src/Exceptionless.Core/Services/EventDeletionService.cs b/src/Exceptionless.Core/Services/EventDeletionService.cs new file mode 100644 index 0000000000..382cb91db9 --- /dev/null +++ b/src/Exceptionless.Core/Services/EventDeletionService.cs @@ -0,0 +1,46 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Foundatio.Repositories; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services; + +/// +/// Removes events and records the deleted usage. Shared by the single-item delete endpoint and the bulk delete +/// operation so both have identical side effects. +/// +public class EventDeletionService +{ + private readonly IEventRepository _eventRepository; + private readonly UsageService _usageService; + private readonly ILogger _logger; + + public EventDeletionService(IEventRepository eventRepository, UsageService usageService, ILogger logger) + { + _eventRepository = eventRepository; + _usageService = usageService; + _logger = logger; + } + + /// Deletes events the caller has already verified access to. + public async Task DeleteAsync(IReadOnlyCollection events) + { + if (events.Count == 0) + return; + + await _eventRepository.RemoveAsync(events); + + foreach (var projectGroup in events.GroupBy(ev => new { ev.OrganizationId, ev.ProjectId })) + { + try + { + await _usageService.IncrementDeletedAsync(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, projectGroup.Count()); + } + catch (Exception ex) + { + // Accounting has no caller cancellation token. A transport timeout must not replay a completed delete. + _logger.LogWarning(ex, "Failed to increment deleted usage metrics for organization {OrganizationId} project {ProjectId}: {Message}", projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, ex.Message); + } + } + } +} diff --git a/src/Exceptionless.Core/Utility/SampleDataService.cs b/src/Exceptionless.Core/Utility/SampleDataService.cs index 859540416c..7028801175 100644 --- a/src/Exceptionless.Core/Utility/SampleDataService.cs +++ b/src/Exceptionless.Core/Utility/SampleDataService.cs @@ -2,8 +2,10 @@ using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services.BackgroundTasks; using Foundatio.Jobs; using Foundatio.Queues; using Foundatio.Repositories; @@ -21,6 +23,7 @@ public class SampleDataService private readonly BillingPlans _billingPlans; private readonly TimeProvider _timeProvider; private readonly IUserRepository _userRepository; + private readonly BackgroundTaskService _backgroundTaskService; private readonly ILogger _logger; public const string TEST_USER_EMAIL = "admin@exceptionless.test"; @@ -48,6 +51,7 @@ public SampleDataService( IUserRepository userRepository, ITokenRepository tokenRepository, IQueue workItemQueue, + BackgroundTaskService backgroundTaskService, BillingManager billingManager, BillingPlans billingPlans, TimeProvider timeProvider, @@ -59,6 +63,7 @@ ILoggerFactory loggerFactory _userRepository = userRepository; _tokenRepository = tokenRepository; _workItemQueue = workItemQueue; + _backgroundTaskService = backgroundTaskService; _billingManager = billingManager; _billingPlans = billingPlans; _timeProvider = timeProvider; @@ -287,17 +292,28 @@ await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem _logger.LogInformation("Enqueued sample event generation: {EventCount} events over {DaysBack} days", eventCount, daysBack); } - public async Task EnqueueSampleEventsAsync(string organizationId, string projectId, int eventCount = 250, int daysBack = 7) + /// + /// Queues sample event generation for one project and returns the id of the background task that tracks it. + /// + public async Task EnqueueSampleEventsAsync(Project project, string createdByUserId, int eventCount = 250, int daysBack = 7) { - string workItemId = await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem + var task = await _backgroundTaskService.CreateAndEnqueueAsync(new BackgroundTask { - OrganizationId = organizationId, - ProjectId = projectId, + OrganizationId = project.OrganizationId, + ProjectId = project.Id, + Type = BackgroundTaskTypes.GenerateProjectSampleData, + Description = $"Generate sample data for {project.Name}", + CreatedByUserId = createdByUserId + }, t => new GenerateSampleEventsWorkItem + { + OrganizationId = project.OrganizationId, + ProjectId = project.Id, EventCount = eventCount, - DaysBack = daysBack + DaysBack = daysBack, + TaskId = t.Id }); - _logger.LogInformation("Enqueued sample event generation for project {ProjectId}: {EventCount} events over {DaysBack} days", projectId, eventCount, daysBack); - return workItemId; + _logger.LogInformation("Enqueued sample event generation for project {ProjectId}: {EventCount} events over {DaysBack} days", project.Id, eventCount, daysBack); + return task.Id; } } diff --git a/src/Exceptionless.Web/Api/ApiEndpoints.cs b/src/Exceptionless.Web/Api/ApiEndpoints.cs index 540b0abfaa..4063b21e59 100644 --- a/src/Exceptionless.Web/Api/ApiEndpoints.cs +++ b/src/Exceptionless.Web/Api/ApiEndpoints.cs @@ -16,6 +16,7 @@ public static WebApplication MapApiEndpoints(this WebApplication app) app.MapWebHookEndpoints(); app.MapStripeEndpoints(); app.MapSavedViewEndpoints(); + app.MapBackgroundTaskEndpoints(); app.MapUserEndpoints(); app.MapProjectEndpoints(); app.MapOrganizationEndpoints(); diff --git a/src/Exceptionless.Web/Api/Endpoints/BackgroundTaskEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/BackgroundTaskEndpoints.cs new file mode 100644 index 0000000000..3af9e51308 --- /dev/null +++ b/src/Exceptionless.Web/Api/Endpoints/BackgroundTaskEndpoints.cs @@ -0,0 +1,102 @@ +using Exceptionless.Core.Authorization; +using Exceptionless.Web.Api.Filters; +using Exceptionless.Web.Api.Results; +using Exceptionless.Web.Models; +using Exceptionless.Web.Utility; +using Exceptionless.Web.Utility.OpenApi; +using Foundatio.Mediator; +using Microsoft.AspNetCore.Mvc; +using TaskMessages = Exceptionless.Web.Api.Messages; +using HttpIResult = Microsoft.AspNetCore.Http.IResult; + +namespace Exceptionless.Web.Api.Endpoints; + +public static class BackgroundTaskEndpoints +{ + public static IEndpointRouteBuilder MapBackgroundTaskEndpoints(this IEndpointRouteBuilder endpoints) + { + var group = endpoints.MapGroup("api/v2") + .RequireAuthorization(AuthorizationRoles.UserPolicy) + .AddEndpointFilter() + .WithTags("BackgroundTask"); + + group.MapPost("bulk-operations", async (IMediator mediator, IMediatorResultMapper resultMapper, [FromBody] NewBulkOperation operation) + => (await mediator.InvokeAsync>>(new TaskMessages.CreateBulkOperation(operation))).ToHttpResult(resultMapper)) + .Accepts("application/json", "application/*+json") + .Produces(StatusCodes.Status202Accepted) + .ProducesProblem(StatusCodes.Status403Forbidden) + .ProducesProblem(StatusCodes.Status404NotFound) + .ProducesProblem(StatusCodes.Status409Conflict) + .ProducesProblem(StatusCodes.Status422UnprocessableEntity) + .ProducesProblem(StatusCodes.Status426UpgradeRequired) + .WithSummary("Create") + .WithDescription(""" + Starts an action, such as `events.delete`, on everything that matches a selection. The selection is either a list of ids + or a filter with a time range. The time range is resolved to absolute UTC dates when the operation is created, and events + stored after that moment are never included. The operation runs in the background and is returned as a task; poll + `GET api/v2/tasks/{id}` or listen for `BackgroundTaskUpdated` messages to follow its progress. + """) + .WithMetadata(new EndpointDocumentation { + RequestBodyDescription = "The bulk operation to start.", + ResponseDescriptions = new() { + ["202"] = "Accepted", + ["403"] = "The action is not permitted for this organization or project.", + ["404"] = "The organization or project could not be found.", + ["409"] = "A destructive operation is already running for this project.", + ["422"] = "The request is not valid.", + ["426"] = "The filter uses a feature that requires a plan upgrade.", + } + }); + + group.MapGet("tasks/{id:objectid}", async (string id, IMediator mediator, IMediatorResultMapper resultMapper) + => (await mediator.InvokeAsync>(new TaskMessages.GetBackgroundTaskById(id))).ToHttpResult(resultMapper)) + .WithName("GetBackgroundTaskById") + .Produces() + .ProducesProblem(StatusCodes.Status404NotFound) + .WithSummary("Get by id") + .WithDescription("Returns the progress of a task, such as a bulk action, a project data reset or sample data generation. Tasks are kept for seven days.") + .WithMetadata(new EndpointDocumentation { + ParameterDescriptions = new() { + ["id"] = "The identifier of the task.", + }, + ResponseDescriptions = new() { + ["404"] = "The task could not be found.", + } + }); + + group.MapGet("organizations/{organizationId:objectid}/tasks", async (string organizationId, IMediator mediator, IMediatorResultMapper resultMapper) + => (await mediator.InvokeAsync>>(new TaskMessages.GetBackgroundTasksByOrganization(organizationId))).ToHttpResult(resultMapper)) + .Produces>() + .ProducesProblem(StatusCodes.Status404NotFound) + .WithSummary("Get by organization") + .WithDescription("Returns the active and recently finished tasks of the organization, newest first. Finished tasks are kept for seven days.") + .WithMetadata(new EndpointDocumentation { + ParameterDescriptions = new() { + ["organizationId"] = "The identifier of the organization.", + }, + ResponseDescriptions = new() { + ["404"] = "The organization could not be found.", + } + }); + + group.MapPost("tasks/{id:objectid}/cancel", async (string id, IMediator mediator, IMediatorResultMapper resultMapper) + => (await mediator.InvokeAsync>>(new TaskMessages.CancelBackgroundTask(id))).ToHttpResult(resultMapper)) + .Produces(StatusCodes.Status202Accepted) + .ProducesProblem(StatusCodes.Status404NotFound) + .ProducesProblem(StatusCodes.Status422UnprocessableEntity) + .WithSummary("Cancel") + .WithDescription("Requests cancellation. A running bulk action stops after the batch it is processing; work already done is not undone. Cancelling a finished task has no effect. Tasks that cannot be stopped, such as a project data reset, are rejected.") + .WithMetadata(new EndpointDocumentation { + ParameterDescriptions = new() { + ["id"] = "The identifier of the task.", + }, + ResponseDescriptions = new() { + ["202"] = "Accepted", + ["404"] = "The task could not be found.", + ["422"] = "The task cannot be cancelled.", + } + }); + + return endpoints; + } +} diff --git a/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs new file mode 100644 index 0000000000..e08c7d7b79 --- /dev/null +++ b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs @@ -0,0 +1,372 @@ +using System.Text.RegularExpressions; +using Exceptionless.Core; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.DateTimeExtensions; +using Exceptionless.DateTimeExtensions.FormatParsers; +using Exceptionless.DateTimeExtensions.FormatParsers.PartParsers; +using Exceptionless.Web.Api.Infrastructure; +using Exceptionless.Web.Api.Messages; +using Exceptionless.Web.Api.Results; +using Exceptionless.Web.Extensions; +using Exceptionless.Web.Models; +using Foundatio.Mediator; +using Foundatio.Repositories; +using Foundatio.Repositories.Utility; + +namespace Exceptionless.Web.Api.Handlers; + +public class BackgroundTaskHandler( + BackgroundTaskService backgroundTaskService, + BulkOperationService bulkOperationService, + IEnumerable resolvers, + IOrganizationRepository organizationRepository, + IProjectRepository projectRepository, + AppOptions appOptions, + TimeProvider timeProvider, + IHttpContextAccessor httpContextAccessor, + ILogger logger) +{ + private const int MaxIds = 1000; + + private HttpContext HttpContext => httpContextAccessor.HttpContext ?? throw new InvalidOperationException("HttpContext is unavailable."); + + public async Task>> Handle(CreateBulkOperation message) + { + var request = HttpContext.Request; + var model = message.Operation; + + var handler = bulkOperationService.GetHandler(model.Type); + if (handler is null) + return Result.Invalid(ValidationError.Create("type", $"Unknown bulk operation type '{model.Type}'.")); + + // Organization and project access mirrors the single-item actions: not accessible looks the same as not found. + if (!request.CanAccessOrganization(model.OrganizationId)) + return Result.NotFound("Organization not found."); + + var organization = await organizationRepository.GetByIdAsync(model.OrganizationId, o => o.Cache()); + if (organization is null) + return Result.NotFound("Organization not found."); + + Project? project = null; + if (!String.IsNullOrEmpty(model.ProjectId)) + { + project = await projectRepository.GetByIdAsync(model.ProjectId, o => o.Cache()); + if (project is null || !String.Equals(project.OrganizationId, organization.Id, StringComparison.Ordinal)) + return Result.NotFound("Project not found."); + } + + var user = request.GetUser(); + var actor = new BulkOperationActor(user.Id, request.IsGlobalAdmin(), request.GetAssociatedOrganizationIds().ToList()); + var parameters = model.Parameters ?? new Dictionary(); + + var selectionResult = ResolveSelection(model.Selection, organization); + if (selectionResult.Result is not null) + return selectionResult.Result; + + var selection = selectionResult.Selection!; + var bulkRequest = new BulkOperationRequest(actor, organization, project, handler.Type, selection, parameters); + + var authorized = await handler.AuthorizeAsync(bulkRequest); + if (!authorized.IsOk) + return ToResult>(authorized); + + var validated = await handler.ValidateAsync(bulkRequest); + if (!validated.IsOk) + return ToResult>(validated); + + var task = new BackgroundTask + { + OrganizationId = organization.Id, + ProjectId = project?.Id, + Type = handler.Type, + CreatedByUserId = user.Id, + Bulk = new BulkOperationDetails + { + Selection = selection, + Parameters = new Dictionary(parameters) + } + }; + + if (selection.Kind is BulkOperationSelectionKind.Filter) + { + var resolver = resolvers.FirstOrDefault(r => r.Entity == handler.Entity); + if (resolver is null) + return Result.Error($"No selection resolver is registered for {handler.Entity}."); + + var selectionCheck = await resolver.ValidateSelectionAsync(organization, selection); + if (!selectionCheck.IsOk) + return ToResult>(selectionCheck); + } + + // Counting and pinning the selection (and describing the operation) happens while it is created. + var created = await bulkOperationService.CreateAsync(task); + if (created.Rejection is not null) + return ToResult>(created.Rejection); + + if (created.Conflict is not null) + return Result.Conflict("Another destructive operation is already running for this project. Wait for it to finish or cancel it first."); + + if (created.Task is null) + return Result.Conflict("Another bulk operation is being started for this organization. Please try again."); + + using var _ = logger.BeginScope(new ExceptionlessState().Organization(organization.Id).Tag("BulkOperation").Identity(user.EmailAddress)); + logger.LogInformation("User {User} started bulk operation {TaskId} {Type} for {Total} items", user.Id, created.Task.Id, created.Task.Type, created.Task.Total); + + return new AcceptedResponse(MapToViewModel(created.Task)); + } + + public async Task> Handle(GetBackgroundTaskById message) + { + var task = await GetAccessibleAsync(message.Id); + if (task is null) + return Result.NotFound("Task not found."); + + return MapToViewModel(task); + } + + public async Task>> Handle(GetBackgroundTasksByOrganization message) + { + if (!HttpContext.Request.CanAccessOrganization(message.OrganizationId)) + return Result.NotFound("Organization not found."); + + var tasks = await backgroundTaskService.GetByOrganizationAsync(message.OrganizationId); + return Result>.Success(tasks.Select(MapToViewModel).ToList()); + } + + public async Task>> Handle(CancelBackgroundTask message) + { + var task = await GetAccessibleAsync(message.Id); + if (task is null) + return Result.NotFound("Task not found."); + + var cancelled = await backgroundTaskService.CancelAsync(task.Id); + if (cancelled.Task is null) + return Result.NotFound("Task not found."); + + if (cancelled.IsNotCancellable) + return Result.Invalid(ValidationError.Create("task", "This task cannot be cancelled.")); + + if (!cancelled.Task.IsActive) + await bulkOperationService.ReleaseSelectionAsync(cancelled.Task); + + logger.LogInformation("User {User} requested cancellation of task {TaskId}", HttpContext.Request.GetUser().Id, cancelled.Task.Id); + return new AcceptedResponse(MapToViewModel(cancelled.Task)); + } + + private async Task GetAccessibleAsync(string id) + { + var task = await backgroundTaskService.GetAsync(id); + if (task is null || !HttpContext.Request.CanAccessOrganization(task.OrganizationId)) + return null; + + return task; + } + + private (BulkOperationSelection? Selection, Result? Result) ResolveSelection(NewBulkOperationSelection model, Organization organization) + { + var utcNow = timeProvider.GetUtcNow().UtcDateTime; + if (model.Kind is BulkOperationSelectionKind.Ids) + { + if (model.Filter is not null || model.Time is not null) + return (null, Result.Invalid(ValidationError.Create("selection", "An id selection cannot include a filter or time."))); + + var ids = (model.Ids ?? []).Distinct(StringComparer.Ordinal).ToList(); + if (ids.Count == 0) + return (null, Result.Invalid(ValidationError.Create("selection.ids", "At least one id is required."))); + + if (ids.Count > MaxIds) + return (null, Result.Invalid(ValidationError.Create("selection.ids", $"At most {MaxIds} ids can be selected. Use a filter selection for more."))); + + if (ids.Any(id => !ObjectId.TryParse(id, out _))) + return (null, Result.Invalid(ValidationError.Create("selection.ids", "Every id must be a valid identifier."))); + + return (new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ids }, null); + } + + // Only an explicit filter selection may resolve a filter. An unknown kind must never fall through to an empty filter, + // which would select everything in scope. + if (model.Kind is not BulkOperationSelectionKind.Filter) + return (null, Result.Invalid(ValidationError.Create("selection.kind", "The selection kind must be ids or filter."))); + + if (model.Ids is { Count: > 0 }) + return (null, Result.Invalid(ValidationError.Create("selection", "A filter selection cannot include ids."))); + + if (!TryParseSelectionOffset(model.Offset, out var utcOffset)) + return (null, Result.Invalid(ValidationError.Create("selection.offset", "The UTC offset must be a valid whole-minute offset between -14h and 14h."))); + + // Validate the requested range before retention and open-end defaults can turn a parse failure into all history. + // Reuse the parsed range so validation and execution resolve relative dates at exactly the same instant. + if (!TryParseSelectionTime(model.Time, new DateTimeOffset(utcNow).ToOffset(utcOffset), out var range)) + return (null, Result.Invalid(ValidationError.Create("selection.time", "The time range is not valid."))); + + var timeInfo = new TimeInfo { Field = EventIndex.Alias.Date, Offset = utcOffset, Range = range! }; + timeInfo.ApplyMinimumUtcStartDate(organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); + timeInfo.AdjustEndTimeIfMaxValue(timeProvider); + var utcStart = timeInfo.Range.UtcStart; + var utcEnd = timeInfo.Range.UtcEnd > utcNow ? utcNow : timeInfo.Range.UtcEnd; + if (utcStart > utcEnd) + return (null, Result.Invalid(ValidationError.Create("selection.time", "The time range does not include any past time."))); + + var selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Filter, + Filter = model.Filter?.Trim() ?? String.Empty, + Time = model.Time, + UtcStart = utcStart, + UtcEnd = utcEnd, + UtcCreatedBefore = utcNow + }; + + return (selection, null); + } + + internal static bool TryParseSelectionTime(string? time, DateTimeOffset relativeBaseTime, out DateTimeRange? range) + { + if (String.IsNullOrWhiteSpace(time)) + { + range = DateTimeRange.Empty; + return true; + } + + string expression = time.Trim(); + if (expression.StartsWith($"{EventIndex.Alias.Date}|", StringComparison.Ordinal)) + expression = expression[(EventIndex.Alias.Date.Length + 1)..]; + + // DateTimeRange.Parse uses the same unbounded sentinel for invalid input and a valid [* TO *] range. + // Its two-part parser also supplies missing bounds, so require both to have been parsed explicitly. + range = null; + try + { + foreach (var parser in DateTimeRange.FormatParsers) + { + if (parser is TwoPartFormatParser twoPartParser) + { + var bounds = new List(2); + var partParsers = twoPartParser.Parsers.Select(partParser => partParser is WildcardPartParser + ? new SelectionTimeWildcardPartParser(bounds) + : new SelectionTimePartParser(partParser, bounds)); + range = new TwoPartFormatParser(partParsers).Parse(expression, relativeBaseTime); + if (range is not null && (bounds.Count != 2 || bounds[0] > bounds[1])) + { + range = null; + return false; + } + } + else + { + range = parser.Parse(expression, relativeBaseTime); + } + + if (range is not null) + return true; + } + } + catch (Exception ex) when (ex is ArgumentException or FormatException or OverflowException) + { + range = null; + } + + return false; + } + + internal static bool TryParseSelectionOffset(string? offset, out TimeSpan utcOffset) + { + utcOffset = TimeSpan.Zero; + if (String.IsNullOrWhiteSpace(offset)) + return true; + + try + { + if (!TimeUnit.TryParse(offset, out var parsedOffset) || !parsedOffset.HasValue + || parsedOffset.Value < TimeSpan.FromHours(-14) || parsedOffset.Value > TimeSpan.FromHours(14) + || parsedOffset.Value.Ticks % TimeSpan.TicksPerMinute != 0) + return false; + + utcOffset = parsedOffset.Value; + return true; + } + catch (Exception ex) when (ex is ArgumentException or FormatException or OverflowException) + { + return false; + } + } + + private sealed class SelectionTimePartParser(IPartParser inner, List bounds) : IPartParser + { + public Regex Regex => inner.Regex; + + public DateTimeOffset? Parse(Match match, DateTimeOffset relativeBaseTime, bool isUpperLimit) + { + var value = inner.Parse(match, relativeBaseTime, isUpperLimit); + if (value.HasValue) + bounds.Add(value.Value); + + return value; + } + } + + // The two-part parser checks this type to keep wildcard bounds open regardless of bracket inclusivity. + private sealed class SelectionTimeWildcardPartParser(List bounds) : WildcardPartParser, IPartParser + { + DateTimeOffset? IPartParser.Parse(Match match, DateTimeOffset relativeBaseTime, bool isUpperLimit) + { + var value = base.Parse(match, relativeBaseTime, isUpperLimit); + if (value.HasValue) + bounds.Add(value.Value); + + return value; + } + } + + private static Result ToResult(BulkOperationCheckResult check) + { + return check.Status switch + { + BulkOperationCheckStatus.Denied => Result.Forbidden(check.Message ?? "Forbidden"), + BulkOperationCheckStatus.PlanLimit => Result.Invalid(ValidationError.Create(ApiValidationErrorIdentifiers.PlanLimit, check.Message ?? "Please upgrade your plan.")), + _ => Result.Invalid(ValidationError.Create(check.Field ?? "operation", check.Message ?? "Invalid request.")) + }; + } + + private static ViewBackgroundTask MapToViewModel(BackgroundTask task) + { + var selection = task.Bulk?.Selection; + return new ViewBackgroundTask + { + Id = task.Id, + OrganizationId = task.OrganizationId, + ProjectId = task.ProjectId, + Type = task.Type, + Description = task.Description, + Status = task.Status, + Progress = task.Progress, + Total = task.Total, + Processed = task.Processed, + Succeeded = task.Succeeded, + Failed = task.Failed, + CanCancel = task.CanCancel, + Selection = selection is null ? null : new ViewBulkOperationSelection + { + Kind = selection.Kind, + IdCount = selection.Kind is BulkOperationSelectionKind.Ids ? selection.Ids?.Count : null, + Filter = selection.Filter, + Time = selection.Time, + UtcStart = selection.UtcStart, + UtcEnd = selection.UtcEnd + }, + CreatedByUserId = task.CreatedByUserId, + CreatedUtc = task.CreatedUtc, + StartedUtc = task.StartedUtc, + CompletedUtc = task.CompletedUtc, + UpdatedUtc = task.UpdatedUtc, + Message = task.Message + }; + } +} diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index 1b9a8f4029..633c62dde5 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -49,6 +49,7 @@ public class EventHandler( AppOptions appOptions, SystemSettingsService systemSettingsService, UsageService usageService, + EventDeletionService eventDeletionService, TimeProvider timeProvider, LinkGenerator linkGenerator, ILoggerFactory loggerFactory) @@ -665,23 +666,7 @@ public async Task> Handle(DeleteEvents message) _logger.LogInformation("User {User} deleted {RemovedCount} events in project ({ProjectId})", currentUser.Id, projectGroup.Count(), ev.ProjectId); } - await eventRepository.RemoveAsync(list); - - foreach (var projectGroup in projectGroups) - { - try - { - await usageService.IncrementDeletedAsync(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, projectGroup.Count()); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to increment deleted usage metrics for org {OrganizationId} project {ProjectId}: {Message}", projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, ex.Message); - } - } + await eventDeletionService.DeleteAsync(list.ToList()); if (results.Failure.Count == 0) return new WorkInProgressResult(); diff --git a/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs b/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs index 1d9d84d45c..e36706dc1c 100644 --- a/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs @@ -3,10 +3,12 @@ using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Queries; using Exceptionless.Core.Services; +using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Core.Utility; using Exceptionless.Web.Api.Infrastructure; using Exceptionless.Web.Api.Messages; @@ -37,6 +39,7 @@ public class ProjectHandler( SlackService slackService, IOAuthProviderClient oauthProviderClient, SampleDataService sampleDataService, + BackgroundTaskService backgroundTaskService, ApiMapper mapper, ITextSerializer serializer, AppOptions options, @@ -228,8 +231,8 @@ public async Task> Handle(GenerateProjectSampleData if (project is null) return Result.NotFound("Project not found."); - string workItemId = await sampleDataService.EnqueueSampleEventsAsync(project.OrganizationId, project.Id); - return new WorkInProgressResult([workItemId]); + string taskId = await sampleDataService.EnqueueSampleEventsAsync(project, GetCurrentUser(message.Context).Id); + return new WorkInProgressResult([taskId]); } public async Task> Handle(ResetProjectData message) @@ -238,13 +241,22 @@ public async Task> Handle(ResetProjectData message) if (project is null) return Result.NotFound("Project not found."); - string workItemId = await workItemQueue.EnqueueAsync(new ResetProjectDataWorkItem + // The returned id is the background task that tracks the reset, so the progress panel can follow it. + var task = await backgroundTaskService.CreateAndEnqueueAsync(new BackgroundTask { OrganizationId = project.OrganizationId, - ProjectId = project.Id + ProjectId = project.Id, + Type = BackgroundTaskTypes.ResetProjectData, + Description = $"Reset data for {project.Name}", + CreatedByUserId = GetCurrentUser(message.Context).Id + }, t => new ResetProjectDataWorkItem + { + OrganizationId = project.OrganizationId, + ProjectId = project.Id, + TaskId = t.Id }); - return new WorkInProgressResult([workItemId]); + return new WorkInProgressResult([task.Id]); } public async Task>> Handle(GetProjectNotificationSettings message) diff --git a/src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs b/src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs new file mode 100644 index 0000000000..26024a5207 --- /dev/null +++ b/src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs @@ -0,0 +1,8 @@ +using Exceptionless.Web.Models; + +namespace Exceptionless.Web.Api.Messages; + +public record CreateBulkOperation(NewBulkOperation Operation); +public record GetBackgroundTaskById(string Id); +public record GetBackgroundTasksByOrganization(string OrganizationId); +public record CancelBackgroundTask(string Id); diff --git a/src/Exceptionless.Web/Api/Results/AcceptedResponse.cs b/src/Exceptionless.Web/Api/Results/AcceptedResponse.cs new file mode 100644 index 0000000000..aeb034f154 --- /dev/null +++ b/src/Exceptionless.Web/Api/Results/AcceptedResponse.cs @@ -0,0 +1,12 @@ +namespace Exceptionless.Web.Api.Results; + +/// Marks a successful handler result that is returned with the 202 Accepted status and a body. +public interface IAcceptedResponse +{ + object? Body { get; } +} + +public record AcceptedResponse(T Value) : IAcceptedResponse +{ + object? IAcceptedResponse.Body => Value; +} diff --git a/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs b/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs index 0a0ac78919..374aee1fa4 100644 --- a/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs +++ b/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs @@ -152,6 +152,9 @@ private static IResult MapSuccess(Foundatio.Mediator.IResult result) if (value is WorkInProgressResult) return HttpResults.Json(value, statusCode: StatusCodes.Status202Accepted); + if (value is IAcceptedResponse accepted) + return HttpResults.Json(accepted.Body, statusCode: StatusCodes.Status202Accepted); + return HttpResults.Ok(value); } diff --git a/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts b/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts index d6493e25c0..758aeb9fd8 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts @@ -38,10 +38,12 @@ export function waitForEventDetailResponses(page: Page, eventId: string, stackId const url = new URL(response.url()); return url.pathname === `/api/v2/events/${eventId}` && url.searchParams.get('expected_stack_id') === stackId; }); - const projectCountResponse = page.waitForResponse((response) => { + return Promise.all([eventResponse, waitForProjectUserCountResponse(page, projectId)]); +} + +export function waitForProjectUserCountResponse(page: Page, projectId: string): Promise { + return page.waitForResponse((response) => { const url = new URL(response.url()); return url.pathname === `/api/v2/projects/${projectId}/events/count` && url.searchParams.get('aggregations') === 'cardinality:user'; }); - - return Promise.all([eventResponse, projectCountResponse]); } diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts new file mode 100644 index 0000000000..0d17da7145 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts @@ -0,0 +1,163 @@ +import { BackgroundTaskStatus, type ViewBackgroundTask } from '#generated/api.ts'; + +import { expect, test } from '../fixtures/e2e-test'; +import { createRepresentativeEvent } from '../support/synthetic-event'; + +const EVENT_COUNT = 5; +const PAGE_SIZE = 2; + +test('many running tasks stay within the viewport and the oldest task can be cancelled after scrolling', async ({ e2eScenario, page }) => { + // Arrange + const now = Date.now(); + const tasks: ViewBackgroundTask[] = Array.from({ length: 12 }, (_, index) => ({ + can_cancel: true, + created_by_user_id: '000000000000000000000001', + created_utc: new Date(now - index * 1000).toISOString(), + description: `Synthetic bulk task ${index + 1}`, + failed: 0, + id: (index + 1).toString(16).padStart(24, '0'), + organization_id: e2eScenario.organizationId, + processed: 25, + progress: 25, + project_id: e2eScenario.projectId, + status: BackgroundTaskStatus.Running, + succeeded: 25, + total: 100, + type: 'events.delete', + updated_utc: new Date(now).toISOString() + })); + const oldest = tasks.at(-1)!; + await page.route( + (url) => url.pathname === `/api/v2/organizations/${e2eScenario.organizationId}/tasks`, + (route) => route.fulfill({ json: tasks }) + ); + await page.route(`**/api/v2/tasks/${oldest.id}/cancel`, async (route) => { + oldest.status = BackgroundTaskStatus.Cancelled; + oldest.updated_utc = new Date().toISOString(); + await route.fulfill({ json: oldest, status: 202 }); + }); + await page.setViewportSize({ height: 400, width: 390 }); + const panel = page.getByRole('region', { name: 'Background tasks' }); + const oldestCancel = panel.getByRole('button', { exact: true, name: `Cancel ${oldest.description}` }); + + // Act + await page.goto(`/event?project=${e2eScenario.projectId}&time=all`); + + // Assert + await expect(panel.getByRole('button', { name: /^Cancel Synthetic bulk task / })).toHaveCount(tasks.length); + const metrics = await panel.evaluate((element) => { + const bounds = element.getBoundingClientRect(); + return { + bottom: bounds.bottom, + clientHeight: element.clientHeight, + scrollHeight: element.scrollHeight, + top: bounds.top, + viewportHeight: window.innerHeight + }; + }); + expect(metrics.top).toBeGreaterThanOrEqual(0); + expect(metrics.bottom).toBeLessThanOrEqual(metrics.viewportHeight); + expect(metrics.scrollHeight).toBeGreaterThan(metrics.clientHeight); + await expect(oldestCancel).not.toBeInViewport(); + + // Act + await oldestCancel.scrollIntoViewIfNeeded(); + + // Assert + await expect(oldestCancel).toBeInViewport(); + expect(await panel.evaluate((element) => element.scrollTop)).toBeGreaterThan(0); + + // Act + await oldestCancel.click(); + + // Assert + await expect(panel.getByRole('button', { exact: true, name: `Dismiss ${oldest.description}` })).toBeVisible(); +}); + +test('deleting everything matching the filter runs in the background and shows progress', async ({ e2eApi, e2eScenario, page }) => { + // Arrange: more events than fit on one page, all in this test's own project. + const references = Array.from({ length: EVENT_COUNT }, (_, index) => `${e2eScenario.referenceId}-bulk-${index}`); + for (const referenceId of references) { + await e2eApi.submitEvent( + e2eScenario.projectId, + e2eScenario.projectToken, + createRepresentativeEvent({ + appUrl: e2eApi.environment.appUrl, + message: `${e2eScenario.message} ${referenceId}`, + referenceId, + runId: e2eApi.environment.runId + }) + ); + } + + for (const referenceId of references) { + await e2eApi.pollForEventByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId); + } + + await page.goto(`/event?project=${e2eScenario.projectId}&time=all&limit=${PAGE_SIZE}`); + const table = page.locator('table:has(thead [role="checkbox"])').first(); + await expect(table.getByRole('checkbox', { name: 'Select row' })).toHaveCount(PAGE_SIZE, { timeout: 30_000 }); + + await test.step('selecting the page offers to select everything matching the filter', async () => { + // Act + await table.getByRole('checkbox', { name: 'Select all rows' }).click(); + + // Assert + await expect(page.getByRole('status').filter({ hasText: `All ${PAGE_SIZE} events on this page are selected.` })).toBeVisible(); + await expect(page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` })).toBeVisible(); + }); + + await test.step('selecting everything matching the filter shows the total', async () => { + // Act + await page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` }).click(); + + // Assert + await expect(page.getByRole('status').filter({ hasText: `All ${EVENT_COUNT} events are selected` })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Clear selection' })).toBeVisible(); + }); + + await test.step('clearing the selection deselects the page', async () => { + // Act + await page.getByRole('button', { name: 'Clear selection' }).click(); + + // Assert + await expect(page.getByRole('button', { name: 'Clear selection' })).toBeHidden(); + await expect(table.getByRole('checkbox', { name: 'Select all rows' })).not.toBeChecked(); + }); + + await test.step('deleting against the filter asks to confirm the exact count', async () => { + // Arrange + await table.getByRole('checkbox', { name: 'Select all rows' }).click(); + await page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` }).click(); + + // Act + await page.getByRole('button', { name: 'Actions' }).click(); + await page.getByRole('menuitem', { name: 'Delete' }).click(); + + // Assert + await expect(page.getByRole('alertdialog')).toContainText(`Delete ${EVENT_COUNT} events matching this filter? This can't be undone.`); + }); + + await test.step('confirming runs in the background and the progress panel reaches completion', async () => { + // Act + await page + .getByRole('alertdialog') + .getByRole('button', { name: `Delete ${EVENT_COUNT} Events` }) + .click(); + + // Assert + const panel = page.getByRole('region', { name: 'Background tasks' }); + await expect(panel).toContainText(/Delete \d+ events matching filter/, { timeout: 30_000 }); + await expect(panel).toContainText(`Completed · ${EVENT_COUNT} processed`, { timeout: 60_000 }); + await expect(table.getByRole('checkbox', { name: 'Select row' })).toHaveCount(0, { timeout: 30_000 }); + }); + + await test.step('every event is gone', async () => { + // Assert + for (const referenceId of references) { + await expect + .poll(async () => (await e2eApi.getEventsByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId)).length, { timeout: 30_000 }) + .toBe(0); + } + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts index 6ecf710fb7..07a85b557e 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts @@ -1,7 +1,7 @@ import type { ConsoleMessage, Page, Request, Response } from '@playwright/test'; import { expect, test } from '../fixtures/e2e-test'; -import { isApplicationApiRequest, trackPendingApiRequests, waitForEventDetailResponses } from '../support/api-request-monitor'; +import { isApplicationApiRequest, trackPendingApiRequests, waitForEventDetailResponses, waitForProjectUserCountResponse } from '../support/api-request-monitor'; import { ExceptionlessE2EJourney } from '../support/exceptionless-journey'; import { createRepresentativeEvent } from '../support/synthetic-event'; import { @@ -206,9 +206,19 @@ test('event list and detail effects stay bounded through paging and background c await measureAction(diagnostics, 'stack detail discovery route remounts', async () => { for (let index = 0; index < 5; index++) { + // Arrange await pendingApiRequests.waitForPendingRequests(); + // Overview can render before the dependent project metric request starts. + const projectCountResponse = waitForProjectUserCountResponse(page, e2eScenario.projectId); + + // Act await page.goto(`/stack/${journey.stackId}`); + + // Assert await expect(page.getByRole('tab', { name: 'Overview' })).toBeVisible(); + const response = await projectCountResponse; + expect(response.ok()).toBe(true); + expect(await response.finished()).toBeNull(); } }); expect(actionSample(diagnostics, 'stack detail discovery route remounts').eventDetails).toBeLessThanOrEqual(10); @@ -217,9 +227,18 @@ test('event list and detail effects stay bounded through paging and background c await measureAction(diagnostics, 'canonical stack event route remounts', async () => { for (let index = 0; index < 5; index++) { + // Arrange await pendingApiRequests.waitForPendingRequests(); + const projectCountResponse = waitForProjectUserCountResponse(page, e2eScenario.projectId); + + // Act await page.goto(`/stack/${journey.stackId}/event/${journey.eventId}`); + + // Assert await expect(page.getByRole('tab', { name: 'Overview' })).toBeVisible(); + const response = await projectCountResponse; + expect(response.ok()).toBe(true); + expect(await response.finished()).toBeNull(); } }); expect(actionSample(diagnostics, 'canonical stack event route remounts').eventDetails).toBeLessThanOrEqual(10); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts index f14e187745..b498e6c4cc 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts @@ -99,7 +99,7 @@ export async function invalidatePersistentEventQueries(queryClient: QueryClient, } export const queryKeys = { - deleteEvent: (ids: string[] | undefined) => [...queryKeys.type, 'delete', ...(ids ?? [])] as const, + deleteEvent: () => [...queryKeys.type, 'delete'] as const, eventsByReference: (referenceId: string | undefined, projectId?: string | undefined, params?: GetEventsByReferenceRequest['params']) => [...queryKeys.type, 'by-ref', referenceId, projectId, params] as const, id: (id: string | undefined) => [...queryKeys.type, id] as const, @@ -127,12 +127,6 @@ export const PERSISTENT_EVENT_DELETE_RECONCILE_RETRY_DELAY = 5000; export const ORGANIZATION_EVENT_QUERY_STALE_TIME_MS = 60 * 1000; export const ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS = 5 * 1000; -export interface DeleteEventsRequest { - route: { - ids: string[] | undefined; - }; -} - export interface EventNavigation { nextId: null | string; previousId: null | string; @@ -335,26 +329,26 @@ export function createEventWithNavigationQueryOptions(request: GetEventRequest, }; } -export function deleteEvent(request: DeleteEventsRequest) { +export function deleteEvent() { const queryClient = useQueryClient(); - return createMutation(() => ({ - enabled: () => !!accessToken.current && !!request.route.ids?.length, - mutationFn: async () => { + return createMutation(() => ({ + enabled: () => !!accessToken.current, + mutationFn: async (ids) => { const client = useFetchClient(); - const response = await client.deleteJSON(`events/${request.route.ids?.join(',')}`); + const response = await client.deleteJSON(`events/${ids.join(',')}`); return response.data!; }, - mutationKey: queryKeys.deleteEvent(request.route.ids), - onError: () => { - request.route.ids?.forEach((id) => + mutationKey: queryKeys.deleteEvent(), + onError: (_, ids) => { + ids.forEach((id) => queryClient.invalidateQueries({ queryKey: queryKeys.id(id) }) ); }, - onSuccess: () => { - request.route.ids?.forEach((id) => + onSuccess: (_, ids) => { + ids.forEach((id) => queryClient.invalidateQueries({ queryKey: queryKeys.id(id) }) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts index 26f88837ff..c692de219c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts @@ -1,6 +1,10 @@ -import { QueryClient } from '@tanstack/svelte-query'; +import type { ProblemDetails } from '@foundatiofx/fetchclient'; + +import { MutationObserver, type MutationObserverOptions, onlineManager, QueryClient } from '@tanstack/svelte-query'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { WorkInProgressResult } from '#shared/models/index.ts'; + import { ChangeType } from '#features/websockets/models.ts'; import type { PersistentEvent } from './models'; @@ -9,6 +13,7 @@ import { queryKeys as stackQueryKeys } from '../stacks/api.svelte'; import { createEventWithNavigationQueryOptions, createOrganizationEventNotificationRefresher, + deleteEvent, type GetEventRequest, invalidatePersistentEventQueries, ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS, @@ -20,10 +25,19 @@ import { schedulePersistentEventDeleteReconciliation } from './api.svelte'; -const fetchClientMocks = vi.hoisted(() => ({ getJSON: vi.fn() })); +const fetchClientMocks = vi.hoisted(() => ({ deleteJSON: vi.fn(), getJSON: vi.fn() })); +const mutationMocks = vi.hoisted(() => ({ + createMutation: vi.fn<(options: () => MutationObserverOptions) => void>(), + useQueryClient: vi.fn() +})); vi.mock('@foundatiofx/fetchclient', () => ({ - useFetchClient: () => ({ getJSON: fetchClientMocks.getJSON }) + useFetchClient: () => fetchClientMocks +})); +vi.mock('@tanstack/svelte-query', async (importOriginal) => ({ + ...(await importOriginal()), + createMutation: mutationMocks.createMutation, + useQueryClient: mutationMocks.useQueryClient })); afterEach(() => { @@ -41,6 +55,46 @@ describe('retainPreviousOrganizationQueryData', () => { }); }); +describe('deleteEvent', () => { + it('keeps the confirmed event IDs when a paused deletion resumes after the selection changes', async () => { + // Arrange + vi.useFakeTimers(); + vi.stubGlobal('document', new EventTarget()); + onlineManager.setOnline(false); + const client = new QueryClient(); + const invalidate = vi.spyOn(client, 'invalidateQueries').mockResolvedValue(undefined); + mutationMocks.useQueryClient.mockReturnValue(client); + fetchClientMocks.deleteJSON.mockResolvedValue({ data: { workers: ['worker-id'] } }); + const selection = ['event-a']; + deleteEvent(); + const mutation = new MutationObserver(client, mutationMocks.createMutation.mock.calls.at(-1)![0]()); + + try { + // Act + const result = mutation.mutate([...selection]); + + // Assert + expect(mutation.getCurrentResult().isPaused).toBe(true); + expect(fetchClientMocks.deleteJSON).not.toHaveBeenCalled(); + + // Act + selection.splice(0, selection.length, 'event-b'); + onlineManager.setOnline(true); + await client.resumePausedMutations(); + await result; + + // Assert + expect(fetchClientMocks.deleteJSON).toHaveBeenCalledExactlyOnceWith('events/event-a'); + expect(invalidate).toHaveBeenCalledWith({ queryKey: queryKeys.id('event-a') }); + expect(invalidate).not.toHaveBeenCalledWith({ queryKey: queryKeys.id('event-b') }); + } finally { + onlineManager.setOnline(true); + client.clear(); + vi.unstubAllGlobals(); + } + }); +}); + describe('createOrganizationEventNotificationRefresher', () => { it('defers removal reconciliation without a leading refetch', async () => { // Arrange diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte index 115a40f3de..e8e2855fe1 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte @@ -5,11 +5,13 @@ interface Props { count?: number; + /** The delete runs against everything matching the current filter instead of the selected rows. */ + matchingFilter?: boolean; open: boolean; remove: () => Promise; } - let { count = 1, open = $bindable(), remove }: Props = $props(); + let { count = 1, matchingFilter = false, open = $bindable(), remove }: Props = $props(); async function onSubmit() { await remove(); @@ -29,12 +31,17 @@ {/if} - Are you sure you want to delete - {#if count === 1} - this event + {#if matchingFilter} + Delete + {count === 1 ? 'event' : 'events'} matching this filter? This can't be undone. {:else} - events - {/if}? + Are you sure you want to delete + {#if count === 1} + this event + {:else} + events + {/if}? + {/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts new file mode 100644 index 0000000000..7800dd442f --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts @@ -0,0 +1,107 @@ +import { createTable, stockFeatures, type StockFeatures, type Table, tableFeatures } from '@tanstack/svelte-table'; +import { describe, expect, it } from 'vitest'; + +import { createAllMatchingSelection, getEventsQueryIdentity } from './all-matching-selection.svelte'; + +describe('createAllMatchingSelection', () => { + it('is active only for the query it was made for', () => { + // Arrange + const identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id', time: '[now-7d TO now]' }); + const selection = createAllMatchingSelection(() => identity); + + // Act + selection.select(1234); + + // Assert + expect(selection.active).toEqual({ count: 1234, identity }); + }); + + it.each([ + ['filter', { filter: 'type:log', organizationId: 'organization-id', time: '[now-7d TO now]' }], + ['time range', { filter: 'type:error', organizationId: 'organization-id', time: '[now-30d TO now]' }], + ['project filter', { filter: 'type:error project:project-b', organizationId: 'organization-id', time: '[now-7d TO now]' }], + ['saved view', { filter: 'type:error', organizationId: 'organization-id', savedViewId: 'view-2', time: '[now-7d TO now]' }], + ['organization', { filter: 'type:error', organizationId: 'another-organization', time: '[now-7d TO now]' }] + ])('is dropped when the %s changes', (_, changed) => { + // Arrange + let identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id', time: '[now-7d TO now]' }); + const selection = createAllMatchingSelection(() => identity); + selection.select(1234); + + // Act + identity = getEventsQueryIdentity(changed); + + // Assert + expect(selection.active).toBeUndefined(); + }); + + it('ignores a count that was fetched for a filter that is no longer showing', () => { + // Arrange + let identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' }); + const selection = createAllMatchingSelection(() => identity); + const selectionWhenCountingStarted = selection.snapshot; + + // Act + identity = getEventsQueryIdentity({ filter: 'type:log', organizationId: 'organization-id' }); + const selected = selection.select(1234, selectionWhenCountingStarted); + + // Assert + expect(selected).toBe(false); + expect(selection.active).toBeUndefined(); + }); + + it('does not come back when the original query is restored after being cleared', () => { + // Arrange + const original = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' }); + let identity = original; + const selection = createAllMatchingSelection(() => identity); + selection.select(10); + + // Act + identity = getEventsQueryIdentity({ filter: 'type:log', organizationId: 'organization-id' }); + selection.clear(); + identity = original; + + // Assert + expect(selection.active).toBeUndefined(); + }); + + it.each([ + ['unchanged', false], + ['cleared and selected again', true] + ])('handles a deferred count when the page selection is %s', async (_, changeSelection) => { + // Arrange + const identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' }); + let table: Table; + const dispose = $effect.root(() => { + table = createTable({ + columns: [{ accessorKey: 'id' }], + data: [{ id: 'event-1' }, { id: 'event-2' }], + features: tableFeatures(stockFeatures), + getRowId: (row) => row.id + }); + }); + table!.toggleAllRowsSelected(true); + const selection = createAllMatchingSelection( + () => identity, + () => table!.store.state.rowSelection + ); + const counting = Promise.withResolvers(); + const selectionWhenCountingStarted = selection.snapshot; + const selecting = counting.promise.then((count) => selection.select(count, selectionWhenCountingStarted)); + + // Act + if (changeSelection) { + table!.resetRowSelection(); + table!.toggleAllRowsSelected(true); + } + counting.resolve(1234); + const selected = await selecting; + dispose(); + + // Assert + expect(table!.getIsAllRowsSelected()).toBe(true); + expect(selected).toBe(!changeSelection); + expect(selection.active).toEqual(changeSelection ? undefined : { count: 1234, identity }); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts new file mode 100644 index 0000000000..3c41f18933 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts @@ -0,0 +1,67 @@ +import type { RowSelectionState } from '@tanstack/svelte-table'; + +export interface AllMatchingSelection { + /** How many events matched when the user chose "select all matching". */ + count: number; + /** The query the selection was made for. */ + identity: string; +} + +export interface AllMatchingSelectionSnapshot { + identity: string; + rowSelection: RowSelectionState | undefined; +} + +/** Everything matching the current filter, as passed to an action menu. */ +export interface AllMatchingTarget { + count: number; + filter?: string; + offset?: string; + organizationId: string; + time?: string; +} + +/** + * Tracks the Gmail-style "everything matching this filter" selection. The selection belongs to the query it was made + * for: as soon as the query identity (organization, filter, time range, saved view) changes it is no longer active, so + * an action can never run against a different set of events than the user confirmed. + */ +export function createAllMatchingSelection(getIdentity: () => string, getRowSelection?: () => RowSelectionState) { + let selection = $state(); + + return { + get active(): AllMatchingSelection | undefined { + return selection && selection.identity === getIdentity() ? selection : undefined; + }, + clear() { + selection = undefined; + }, + /** + * Selects everything matching the current query. A pending count must not replace a newer filter or page + * selection, including clearing and then selecting the same rows again. + */ + select(count: number, expected?: AllMatchingSelectionSnapshot): boolean { + const identity = getIdentity(); + if (expected && (expected.identity !== identity || expected.rowSelection !== getRowSelection?.())) { + return false; + } + + selection = { + count, + identity + }; + return true; + }, + /** Capture the query and page selection before awaiting a count, then pass the snapshot to select. */ + get snapshot(): AllMatchingSelectionSnapshot { + return { + identity: getIdentity(), + rowSelection: getRowSelection?.() + }; + } + }; +} + +export function getEventsQueryIdentity(parts: { filter?: null | string; organizationId?: string; savedViewId?: string; time?: null | string }): string { + return JSON.stringify([parts.organizationId ?? null, parts.filter ?? null, parts.time ?? null, parts.savedViewId ?? null]); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte index fec3c8e401..046a9d8728 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte @@ -3,6 +3,8 @@ @@ -56,10 +122,10 @@ {#snippet child({ props })} + +{:else if canSelectAllMatching} +
+ All {pageCount === 1 ? 'event' : 'events'} on this page {pageCount === 1 ? 'is' : 'are'} selected. + +
+{/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts new file mode 100644 index 0000000000..9ebd9f330c --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts @@ -0,0 +1,56 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import EventsSelectAllBanner from './events-select-all-banner.svelte'; + +function bannerText(): string { + return (screen.getByRole('status').textContent ?? '').replace(/\s+/g, ' ').trim(); +} + +describe('EventsSelectAllBanner', () => { + afterEach(cleanup); + + it('offers to select everything matching the filter once the page is selected', async () => { + // Arrange + const onSelectAllMatching = vi.fn(); + render(EventsSelectAllBanner, { props: { isPageSelected: true, onClear: vi.fn(), onSelectAllMatching, pageCount: 20, totalMatching: 1234 } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Select all 1,234 events matching this filter' })); + + // Assert + expect(bannerText()).toBe('All 20 events on this page are selected. Select all 1,234 events matching this filter'); + expect(onSelectAllMatching).toHaveBeenCalledOnce(); + }); + + it('confirms that everything is selected and offers to clear it', async () => { + // Arrange + const onClear = vi.fn(); + render(EventsSelectAllBanner, { + props: { allMatchingCount: 1234, isPageSelected: true, onClear, onSelectAllMatching: vi.fn(), pageCount: 20, totalMatching: 1234 } + }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Clear selection' })); + + // Assert + expect(bannerText()).toBe('All 1,234 events are selected · Clear selection'); + expect(screen.queryByRole('button', { name: /Select all/ })).toBeNull(); + expect(onClear).toHaveBeenCalledOnce(); + }); + + it.each([ + ['only some rows are selected', { isPageSelected: false, pageCount: 20, totalMatching: 1234 }], + ['the page already shows every match', { isPageSelected: true, pageCount: 20, totalMatching: 20 }], + ['the total is not known yet', { isPageSelected: true, pageCount: 20, totalMatching: undefined }] + ])('shows nothing when %s', (_, selection) => { + // Arrange + const props = { ...selection, onClear: vi.fn(), onSelectAllMatching: vi.fn() }; + + // Act + render(EventsSelectAllBanner, { props }); + + // Assert + expect(screen.queryByRole('status')).toBeNull(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts index 557b9f1f3e..e5eb392f9d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts @@ -62,6 +62,7 @@ export function getColumns renderComponent(Checkbox, { + 'aria-label': 'Select all rows', checked: table.getIsAllRowsSelected(), indeterminate: table.getIsSomeRowsSelected() && !table.getIsAllRowsSelected(), onCheckedChange: (checked: 'indeterminate' | boolean) => diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts index 28a3a0f4c9..fe2492701c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts @@ -9,6 +9,7 @@ import { accessToken } from '#features/auth/index.svelte.ts'; import { queryKeys as eventQueryKeys } from '#features/events/api.svelte.ts'; import { fetchApiJson } from '#features/shared/api/api.svelte.ts'; import { queryKeys as stackQueryKeys } from '#features/stacks/api.svelte.ts'; +import { invalidateTaskQueries } from '#features/tasks/api.svelte.ts'; export async function invalidateProjectQueries(queryClient: QueryClient, message: WebSocketMessageValue<'ProjectChanged'>) { const { id, organization_id } = message; @@ -360,6 +361,8 @@ export function deleteSourceMapMutation(request: SourceMapRequest) { } export function generateSampleData(request: GenerateSampleDataRequest) { + const queryClient = useQueryClient(); + return createMutation(() => ({ enabled: () => !!accessToken.current && !!request.route.id, mutationFn: async () => { @@ -370,7 +373,9 @@ export function generateSampleData(request: GenerateSampleDataRequest) { return response.data!; }, - mutationKey: queryKeys.generateSampleData(request.route.id) + mutationKey: queryKeys.generateSampleData(request.route.id), + // The request starts a background task; show it in the progress panel. + onSuccess: () => invalidateTaskQueries(queryClient) })); } @@ -707,6 +712,8 @@ export function putProjectIntegrationNotificationSettings(request: PutProjectInt } export function resetData(request: ResetDataRequest) { + const queryClient = useQueryClient(); + return createMutation(() => ({ enabled: () => !!accessToken.current && !!request.route.id, mutationFn: async () => { @@ -717,7 +724,9 @@ export function resetData(request: ResetDataRequest) { return response.data!; }, - mutationKey: queryKeys.resetData(request.route.id) + mutationKey: queryKeys.resetData(request.route.id), + // The request starts a background task; show it in the progress panel. + onSuccess: () => invalidateTaskQueries(queryClient) })); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/index.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/index.ts new file mode 100644 index 0000000000..25eee615aa --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/index.ts @@ -0,0 +1,7 @@ +import Root from "./progress.svelte"; + +export { + Root, + // + Root as Progress, +}; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/progress.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/progress.svelte new file mode 100644 index 0000000000..12c78f5030 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/progress.svelte @@ -0,0 +1,27 @@ + + + +
+
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api-polling.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api-polling.svelte.test.ts new file mode 100644 index 0000000000..cddeb214d4 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api-polling.svelte.test.ts @@ -0,0 +1,222 @@ +import type { ProblemDetails } from '@foundatiofx/fetchclient'; + +import { focusManager, QueryClient, QueryObserver, type QueryObserverOptions } from '@tanstack/svelte-query'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { BackgroundTaskStatus, type ViewBackgroundTask } from './models'; + +const mocks = vi.hoisted(() => ({ + createQuery: vi.fn<(options: () => QueryObserverOptions) => void>(), + getJSON: vi.fn(), + useQueryClient: vi.fn() +})); + +vi.mock('#features/auth/index.svelte.ts', () => ({ accessToken: { current: 'access-token' } })); +vi.mock('#features/events/api.svelte.ts', () => ({ queryKeys: { type: ['PersistentEvent'] } })); +vi.mock('#features/stacks/api.svelte.ts', () => ({ queryKeys: { type: ['Stack'] } })); +vi.mock('@foundatiofx/fetchclient', () => ({ useFetchClient: () => ({ getJSON: mocks.getJSON }) })); +vi.mock('@tanstack/svelte-query', async (importOriginal) => ({ + ...(await importOriginal()), + createQuery: mocks.createQuery, + useQueryClient: mocks.useQueryClient +})); + +import { ACTIVE_TASK_POLL_INTERVAL_MS, getTasksQuery, queryKeys, startTaskPolling, updateTaskFromMessage } from './api.svelte'; + +function createTask(status: BackgroundTaskStatus): ViewBackgroundTask { + return { + can_cancel: false, + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Generate sample data', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 0, + progress: 0, + status, + succeeded: 0, + total: 0, + type: 'projects.sample-data', + updated_utc: '2026-01-01T10:00:00Z' + }; +} + +describe('task polling', () => { + beforeEach(() => { + vi.useFakeTimers(); + focusManager.setFocused(true); + }); + + afterEach(() => { + focusManager.setFocused(undefined); + vi.useRealTimers(); + }); + + it.each([false, true])('polls on schedule during continuous progress with a preseeded cache: %s', async (preseeded) => { + // Arrange + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + mocks.useQueryClient.mockReturnValue(client); + const running = createTask(BackgroundTaskStatus.Running); + const unknown = { ...running, id: 'started-elsewhere' }; + mocks.getJSON.mockReset().mockResolvedValue({ data: [running, unknown] }); + if (preseeded) { + client.setQueryData(queryKeys.organization(running.organization_id), [running]); + } else { + mocks.getJSON.mockResolvedValueOnce({ data: [running] }); + } + getTasksQuery({ route: { organizationId: 'organization-id' } }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, { ...options(), refetchOnMount: !preseeded }); + const unsubscribe = observer.subscribe(() => {}); + const stopPolling = startTaskPolling(client, running.organization_id, true); + + try { + await observer.getCurrentQuery().promise; + + // Act + for (let progress = 1; progress <= 5; progress++) { + await vi.advanceTimersByTimeAsync(500); + await updateTaskFromMessage(client, { + ...running, + message: running.message ?? undefined, + processed: progress, + project_id: running.project_id ?? undefined, + updated_utc: new Date().toISOString() + }); + } + await vi.advanceTimersByTimeAsync(500); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(preseeded ? 1 : 2); + expect(observer.getCurrentResult().data?.map((task) => task.id)).toEqual([running.id, unknown.id]); + + // Act + focusManager.setFocused(false); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS * 2); + await updateTaskFromMessage(client, { + ...running, + message: running.message ?? undefined, + processed: 6, + project_id: running.project_id ?? undefined, + updated_utc: new Date().toISOString() + }); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(preseeded ? 1 : 2); + + // Act + focusManager.setFocused(true); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(preseeded ? 2 : 3); + + // Act + stopPolling?.(); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(preseeded ? 2 : 3); + } finally { + stopPolling?.(); + unsubscribe(); + client.clear(); + } + }); + + it('keeps its polling cadence after a failed HTTP attempt', async () => { + // Arrange + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + mocks.useQueryClient.mockReturnValue(client); + const running = createTask(BackgroundTaskStatus.Running); + mocks.getJSON + .mockReset() + .mockResolvedValueOnce({ data: [running] }) + .mockRejectedValueOnce(new Error('Unavailable')) + .mockResolvedValue({ data: [running] }); + getTasksQuery({ route: { organizationId: 'organization-id' } }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, options()); + const unsubscribe = observer.subscribe(() => {}); + const stopPolling = startTaskPolling(client, running.organization_id, true); + + try { + await observer.getCurrentQuery().promise; + + // Act + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS - 1); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + + // Act + await vi.advanceTimersByTimeAsync(1); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(3); + } finally { + stopPolling?.(); + unsubscribe(); + client.clear(); + } + }); + + it.each(['empty', 'failed'] as const)('reconciles a list that is %s at a slower rate only while the tab is visible', async (initialState) => { + // Arrange + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + mocks.useQueryClient.mockReturnValue(client); + const initialTasks = initialState === 'failed' ? [createTask(BackgroundTaskStatus.Failed)] : []; + const running = createTask(BackgroundTaskStatus.Running); + const completed = createTask(BackgroundTaskStatus.Completed); + mocks.getJSON + .mockReset() + .mockResolvedValueOnce({ data: initialTasks }) + .mockResolvedValueOnce({ data: [running] }) + .mockResolvedValue({ data: [completed] }); + getTasksQuery({ route: { organizationId: 'organization-id' } }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, options()); + const unsubscribe = observer.subscribe(() => {}); + let stopPolling = startTaskPolling(client, 'organization-id', false); + + try { + await observer.getCurrentQuery().promise; + + // Act + focusManager.setFocused(false); + await vi.advanceTimersByTimeAsync(60_000); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledOnce(); + + // Act + focusManager.setFocused(true); + await vi.advanceTimersByTimeAsync(60_000 - 1); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledOnce(); + + // Act + await vi.advanceTimersByTimeAsync(1); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + expect(observer.getCurrentResult().data).toEqual([running]); + + // Act + stopPolling?.(); + stopPolling = startTaskPolling(client, 'organization-id', true); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(3); + expect(observer.getCurrentResult().data).toEqual([completed]); + } finally { + stopPolling?.(); + unsubscribe(); + client.clear(); + } + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts new file mode 100644 index 0000000000..06fcaa7b7d --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts @@ -0,0 +1,263 @@ +import type { Query, QueryClient } from '@tanstack/svelte-query'; + +import { type ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; +import { createMutation, createQuery, focusManager, replaceEqualDeep, useQueryClient } from '@tanstack/svelte-query'; + +import type { BackgroundTaskUpdated } from '#features/websockets/models.ts'; + +import { accessToken } from '#features/auth/index.svelte.ts'; +import { queryKeys as eventQueryKeys } from '#features/events/api.svelte.ts'; +import { queryKeys as stackQueryKeys } from '#features/stacks/api.svelte.ts'; + +import { applyTaskUpdate, compareTaskUpdateTimes, mergeTaskList, type NewBulkOperation, type ViewBackgroundTask } from './models'; + +/** While a task is active the list is polled as well, so progress still moves when push messages are unavailable. */ +export const ACTIVE_TASK_POLL_INTERVAL_MS = 3000; +/** Reconcile missed notifications, tasks started elsewhere, and ordinary retries while the tab is visible. */ +export const IDLE_TASK_POLL_INTERVAL_MS = 60_000; + +export const queryKeys = { + cancel: (id: string | undefined) => [...queryKeys.type, id, 'cancel'] as const, + organization: (organizationId: string | undefined) => [...queryKeys.type, 'organization', organizationId] as const, + post: () => [...queryKeys.type, 'post'] as const, + type: ['BackgroundTask'] as const +}; + +export interface GetTasksRequest { + route: { + organizationId: string | undefined; + }; +} + +interface TaskListRead { + baseline: readonly ViewBackgroundTask[]; + result?: ViewBackgroundTask[]; +} + +type TaskQuery = Query; + +// Query identity survives observer/options changes. Each settled read releases its baseline. +const taskListReads = new WeakMap(); + +export function cancelTask() { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current, + mutationFn: async (id: string) => { + const client = useFetchClient(); + const response = await client.postJSON(`tasks/${id}/cancel`); + + return response.data!; + }, + mutationKey: queryKeys.cancel(undefined), + onSuccess: (task) => upsertTask(queryClient, task) + })); +} + +export function getTasksQuery(request: GetTasksRequest) { + const queryClient = useQueryClient(); + + return createQuery(() => { + const organizationId = request.route.organizationId; + const key = queryKeys.organization(organizationId); + const getQuery = () => + queryClient.getQueryCache().find({ + exact: true, + queryKey: key + })!; + + return { + enabled: () => !!accessToken.current && !!organizationId, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const query = getQuery(); + const read: TaskListRead = { + baseline: query.state.data ?? [] + }; + taskListReads.set(query, read); + try { + const client = useFetchClient(); + const response = await client.getJSON(`organizations/${organizationId}/tasks`, { + signal + }); + read.result = response.data ?? []; + return read.result; + } finally { + if (taskListReads.get(query) === read && (signal.aborted || !read.result)) { + taskListReads.delete(query); + } + } + }, + queryKey: key, + // Refresh immediately on return; updates for other organizations are not observed. + staleTime: 0, + structuralSharing: (current, incoming) => { + const query = getQuery(); + const read = taskListReads.get(query); + if (!read || read.result !== incoming) { + return replaceEqualDeep(current, incoming); + } + + // Merge at the synchronous cache commit: a push can arrive after queryFn returns its response. + taskListReads.delete(query); + return replaceEqualDeep(current, mergeTaskList(incoming as ViewBackgroundTask[], current as undefined | ViewBackgroundTask[], read.baseline)); + } + }; + }); +} + +/** Refreshes everything a finished bulk task may have changed. */ +export async function invalidateTaskAffectedQueries(queryClient: QueryClient) { + await Promise.all( + [eventQueryKeys.type, stackQueryKeys.type].map(async (queryKey) => { + // Invalidation alone reuses an uncached initial fetch, which may predate the completed operation. + await queryClient.cancelQueries({ + queryKey + }); + // Simultaneous completions can share the replacement read; a later completion cancels it above. + await queryClient.invalidateQueries( + { + queryKey + }, + { + cancelRefetch: false + } + ); + }) + ); +} + +/** Refetches the task lists, e.g. after a request that starts a task. */ +export async function invalidateTaskQueries(queryClient: QueryClient) { + // Replace a read that began before creation, including an uncached initial fetch. Once invalidated, + // let its reconciliation finish so a burst of notifications cannot keep restarting the request. + await queryClient.cancelQueries( + { + predicate: (query) => !query.state.isInvalidated, + queryKey: queryKeys.type + }, + { + revert: false + } + ); + return queryClient.invalidateQueries( + { + queryKey: queryKeys.type + }, + { + cancelRefetch: false + } + ); +} + +export function postBulkOperation() { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current, + mutationFn: async (task: NewBulkOperation) => { + const client = useFetchClient(); + const response = await client.postJSON('bulk-operations', task); + + return response.data!; + }, + mutationKey: queryKeys.post(), + onSuccess: (task) => upsertTask(queryClient, task) + })); +} + +/** Poll on a fixed cadence owned by the panel, so pushed cache writes cannot postpone reconciliation. */ +export function startTaskPolling(queryClient: QueryClient, organizationId: string | undefined, hasActiveTasks: boolean) { + if (!organizationId) { + return; + } + + const key = queryKeys.organization(organizationId); + const timer = window.setInterval( + () => { + if (!focusManager.isFocused()) { + return; + } + + void queryClient.refetchQueries( + { + exact: true, + fetchStatus: 'idle', + queryKey: key, + type: 'active' + }, + { + cancelRefetch: false + } + ); + }, + hasActiveTasks ? ACTIVE_TASK_POLL_INTERVAL_MS : IDLE_TASK_POLL_INTERVAL_MS + ); + return () => window.clearInterval(timer); +} + +/** Applies a pushed update to the cached list. Unknown tasks (started elsewhere) trigger a refetch instead. */ +export async function updateTaskFromMessage(queryClient: QueryClient, update: BackgroundTaskUpdated) { + const key = queryKeys.organization(update.organization_id); + const current = queryClient.getQueryData(key); + const next = current ? applyTaskUpdate(current, update) : undefined; + if (!next) { + // Replace an earlier read once, then share the reconciliation during a burst of unknown-task messages. + await queryClient.cancelQueries( + { + predicate: (query) => !query.state.isInvalidated, + queryKey: key + }, + { + revert: false + } + ); + void queryClient.invalidateQueries( + { + queryKey: key + }, + { + cancelRefetch: false + } + ); + return; + } + + if (next === current) { + return; + } + + updateCachedTasks(queryClient, key, () => next); +} + +function updateCachedTasks( + queryClient: QueryClient, + key: ReturnType, + update: (current: undefined | ViewBackgroundTask[]) => undefined | ViewBackgroundTask[] +) { + // Cache writes clear invalidation. Keep any required list reconciliation running while applying pushed progress. + const needsReconciliation = queryClient.getQueryState(key)?.isInvalidated; + queryClient.setQueryData(key, update); + if (needsReconciliation) { + void queryClient.invalidateQueries( + { + queryKey: key + }, + { + cancelRefetch: false + } + ); + } +} + +function upsertTask(queryClient: QueryClient, task: ViewBackgroundTask) { + const key = queryKeys.organization(task.organization_id); + updateCachedTasks(queryClient, key, (current) => { + const existing = current?.find((item) => item.id === task.id); + if (existing && compareTaskUpdateTimes(existing.updated_utc, task.updated_utc) > 0) { + return current; + } + + return [task, ...(current ?? []).filter((item) => item.id !== task.id)]; + }); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts new file mode 100644 index 0000000000..fae2f0b23c --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts @@ -0,0 +1,548 @@ +import type { ProblemDetails } from '@foundatiofx/fetchclient'; + +import { QueryClient, QueryObserver, type QueryObserverOptions } from '@tanstack/svelte-query'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from './models'; + +const mocks = vi.hoisted(() => ({ + createMutation: vi.fn<(options: () => { onSuccess: (task: ViewBackgroundTask) => Promise | void }) => void>(), + createQuery: vi.fn<(options: () => QueryObserverOptions) => void>(), + getJSON: vi.fn(), + useQueryClient: vi.fn() +})); + +vi.mock('#features/auth/index.svelte.ts', () => ({ accessToken: { current: 'access-token' } })); +vi.mock('#features/events/api.svelte.ts', () => ({ queryKeys: { type: ['PersistentEvent'] } })); +vi.mock('#features/stacks/api.svelte.ts', () => ({ queryKeys: { type: ['Stack'] } })); +vi.mock('@foundatiofx/fetchclient', () => ({ useFetchClient: () => ({ getJSON: mocks.getJSON }) })); +vi.mock('@tanstack/svelte-query', async (importOriginal) => ({ + ...(await importOriginal()), + createMutation: mocks.createMutation, + createQuery: mocks.createQuery, + useQueryClient: mocks.useQueryClient +})); + +import { + cancelTask, + getTasksQuery, + invalidateTaskAffectedQueries, + invalidateTaskQueries, + postBulkOperation, + queryKeys, + updateTaskFromMessage +} from './api.svelte'; + +function createTask(overrides: Partial = {}): ViewBackgroundTask { + return { + can_cancel: true, + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Delete events matching filter', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 0, + progress: 0, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BackgroundTaskStatus.Running, + succeeded: 0, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:00Z', + ...overrides + }; +} + +function createUpdate(id: string) { + return { + can_cancel: true, + description: 'Delete events matching filter', + failed: 0, + id, + organization_id: 'organization-id', + processed: 75, + progress: 75, + status: BackgroundTaskStatus.Running, + succeeded: 75, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:09Z' + }; +} + +function getTaskQueryOptions() { + getTasksQuery({ route: { organizationId: 'organization-id' } }); + return mocks.createQuery.mock.calls.at(-1)![0](); +} + +describe('getTasksQuery', () => { + it.each([ + ['empty', BackgroundTaskStatus.Running], + ['empty', BackgroundTaskStatus.Completed], + ['finished', BackgroundTaskStatus.Running], + ['finished', BackgroundTaskStatus.Completed] + ] as const)('refreshes a cached %s list after returning to an organization with a new %s task', async (cachedList, newStatus) => { + // Arrange + vi.clearAllMocks(); + const client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: 5 * 60 * 1000 } } }); + mocks.useQueryClient.mockReturnValue(client); + let organizationId = 'organization-id'; + const initialTasks = cachedList === 'finished' ? [createTask({ status: BackgroundTaskStatus.Completed })] : []; + const missedTask = createTask({ id: 'started-elsewhere', status: newStatus }); + mocks.getJSON + .mockReset() + .mockResolvedValueOnce({ data: initialTasks }) + .mockResolvedValueOnce({ data: [] }) + .mockResolvedValueOnce({ data: [...initialTasks, missedTask] }); + getTasksQuery({ + route: { + get organizationId() { + return organizationId; + } + } + }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, options()); + const unsubscribe = observer.subscribe(() => {}); + + try { + await observer.getCurrentQuery().promise; + + // Act + organizationId = 'another-organization'; + observer.setOptions(options()); + await observer.getCurrentQuery().promise; + organizationId = 'organization-id'; + observer.setOptions(options()); + await observer.getCurrentQuery().promise; + + // Assert + expect(mocks.getJSON.mock.calls.map(([url]) => url)).toEqual([ + 'organizations/organization-id/tasks', + 'organizations/another-organization/tasks', + 'organizations/organization-id/tasks' + ]); + expect(observer.getCurrentResult().data).toEqual([...initialTasks, missedTask]); + } finally { + unsubscribe(); + client.clear(); + } + }); +}); + +describe('updateTaskFromMessage', () => { + let queryClient: QueryClient; + + beforeEach(() => { + vi.clearAllMocks(); + queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + vi.spyOn(queryClient, 'invalidateQueries'); + }); + + afterEach(() => queryClient.clear()); + + it('writes the progress of a known task into the cached list', async () => { + // Arrange + queryClient.setQueryData(queryKeys.organization('organization-id'), [createTask()]); + + // Act + await updateTaskFromMessage(queryClient, createUpdate('task-1')); + + // Assert + expect(queryClient.getQueryData(queryKeys.organization('organization-id'))).toMatchObject([{ id: 'task-1', processed: 75 }]); + expect(queryClient.invalidateQueries).not.toHaveBeenCalled(); + }); + + it('reloads the list when the task was started somewhere else', async () => { + // Arrange + const tasks = [createTask()]; + queryClient.setQueryData(queryKeys.organization('organization-id'), tasks); + + // Act + await updateTaskFromMessage(queryClient, createUpdate('started-elsewhere')); + + // Assert + expect(queryClient.getQueryData(queryKeys.organization('organization-id'))).toEqual(tasks); + expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: queryKeys.organization('organization-id') }, { cancelRefetch: false }); + }); +}); + +describe('invalidateTaskAffectedQueries', () => { + it.each([false, true])('replaces affected reads started before task completion with cached data: %s', async (cached) => { + // Arrange + const client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: 60_000 } } }); + const lists = [['PersistentEvent'], ['Stack']].map((queryKey) => { + const pending = Promise.withResolvers(); + const queryFn = vi.fn().mockReturnValueOnce(pending.promise).mockResolvedValue([]); + if (cached) { + client.setQueryData(queryKey, ['deleted-item']); + } + const observer = new QueryObserver(client, { queryFn, queryKey, refetchOnMount: false }); + const unsubscribe = observer.subscribe(() => {}); + const read = observer.refetch(); + return { observer, pending, queryFn, read, unsubscribe }; + }); + + try { + // Act + // With removal pushes unavailable, the task panel learns the delete completed through polling. + const completion = invalidateTaskAffectedQueries(client); + for (const list of lists) { + list.pending.resolve(['deleted-item']); + } + await Promise.all([completion, ...lists.map((list) => list.read)]); + + // Assert + for (const list of lists) { + expect(list.observer.getCurrentResult().data).toEqual([]); + expect(list.queryFn).toHaveBeenCalledTimes(2); + expect(list.observer.getCurrentResult().isStale).toBe(false); + } + } finally { + for (const list of lists) { + list.unsubscribe(); + } + client.clear(); + } + }); + + it.each([false, true])('shares simultaneous completion refreshes and replaces them after a later completion with cached data: %s', async (cached) => { + // Arrange + const client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: 60_000 } } }); + const queryKey = ['PersistentEvent', 'organizations', 'organization-id', 'events']; + const beforeCompletion = Promise.withResolvers(); + const firstRefresh = Promise.withResolvers(); + const firstRefreshStarted = Promise.withResolvers(); + const lastRefresh = Promise.withResolvers(); + const lastRefreshStarted = Promise.withResolvers(); + const queryFn = vi + .fn() + .mockReturnValueOnce(beforeCompletion.promise) + .mockImplementationOnce(() => { + firstRefreshStarted.resolve(); + return firstRefresh.promise; + }) + .mockImplementationOnce(() => { + lastRefreshStarted.resolve(); + return lastRefresh.promise; + }); + if (cached) { + client.setQueryData(queryKey, ['first-event', 'second-event']); + } + const observer = new QueryObserver(client, { queryFn, queryKey, refetchOnMount: false }); + const unsubscribe = observer.subscribe(() => {}); + const initialRead = observer.refetch(); + + try { + // Act + const simultaneous = [invalidateTaskAffectedQueries(client), invalidateTaskAffectedQueries(client)]; + await firstRefreshStarted.promise; + + // Assert + expect(queryFn).toHaveBeenCalledTimes(2); + + // Act + // Another delete finishes after the shared refresh took its snapshot. + const laterCompletion = invalidateTaskAffectedQueries(client); + await lastRefreshStarted.promise; + beforeCompletion.resolve(['first-event', 'second-event']); + firstRefresh.resolve(['second-event']); + lastRefresh.resolve([]); + await Promise.all([initialRead, ...simultaneous, laterCompletion]); + + // Assert + expect(queryFn).toHaveBeenCalledTimes(3); + expect(observer.getCurrentResult().data).toEqual([]); + expect(observer.getCurrentResult().isError).toBe(false); + } finally { + unsubscribe(); + client.clear(); + } + }); +}); + +describe('task cache lifecycle', () => { + let client: QueryClient; + const key = queryKeys.organization('organization-id'); + + beforeEach(() => { + vi.clearAllMocks(); + client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + mocks.useQueryClient.mockReturnValue(client); + }); + + afterEach(() => client.clear()); + + it.each([ + ['pushed update', false], + ['task creation invalidation', false], + ['pushed update', true], + ['task creation invalidation', true] + ] as const)('coalesces unknown-task notifications after a %s replaces a pending list request with cached data: %s', async (source, cached) => { + // Arrange + const previousRead = Promise.withResolvers(); + const nextRead = Promise.withResolvers(); + const nextReadStarted = Promise.withResolvers(); + const completed = createTask({ id: 'started-elsewhere', status: BackgroundTaskStatus.Completed }); + const queued = createTask({ id: 'queued-during-read', status: BackgroundTaskStatus.Queued }); + const signals: AbortSignal[] = []; + mocks.getJSON.mockReset().mockImplementation((_url: string, { signal }: { signal: AbortSignal }) => { + signals.push(signal); + if (signals.length === 1) { + return previousRead.promise.then((data) => ({ data })); + } + + nextReadStarted.resolve(); + return nextRead.promise.then((data) => ({ data })); + }); + if (cached) { + client.setQueryData(key, []); + } + + const observer = new QueryObserver(client, getTaskQueryOptions()); + const unsubscribe = observer.subscribe(() => {}); + + try { + // Act + const reconciliation = source === 'pushed update' ? updateTaskFromMessage(client, createUpdate(completed.id)) : invalidateTaskQueries(client); + await nextReadStarted.promise; + for (let notification = 0; notification < 5; notification++) { + await updateTaskFromMessage(client, createUpdate(completed.id)); + } + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + expect(signals.map((signal) => signal.aborted)).toEqual([true, false]); + + // Act + postBulkOperation(); + await mocks.createMutation.mock.calls[0]![0]().onSuccess(queued); + previousRead.resolve([]); + nextRead.resolve([completed]); + await reconciliation; + await observer.getCurrentQuery().promise; + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + expect(observer.getCurrentResult().data).toEqual([completed, queued]); + } finally { + unsubscribe(); + } + }); + + it('keeps a newly queued task when an earlier list request returns without it', async () => { + // Arrange + const pending = Promise.withResolvers(); + mocks.getJSON.mockReset().mockImplementation(() => pending.promise.then((data) => ({ data }))); + const read = client.fetchQuery(getTaskQueryOptions()); + const task = createTask({ status: BackgroundTaskStatus.Queued }); + postBulkOperation(); + const options = mocks.createMutation.mock.calls[0]![0](); + + // Act + await options.onSuccess(task); + pending.resolve([]); + await read; + + // Assert + expect(client.getQueryData(key)).toEqual([task]); + }); + + it('merges a task inserted after HTTP completion and drops it when a later read omits it unchanged', async () => { + // Arrange + mocks.getJSON.mockReset().mockResolvedValue({ data: [] }); + const task = createTask({ status: BackgroundTaskStatus.Queued }); + const queryOptions = getTaskQueryOptions(); + const queryFn = queryOptions.queryFn; + if (typeof queryFn !== 'function') { + throw new Error('The task list requires a query function.'); + } + postBulkOperation(); + const mutationOptions = mocks.createMutation.mock.calls[0]![0](); + + // Act + await client.fetchQuery({ + ...queryOptions, + queryFn: async (context) => { + const data = await queryFn(context); + await mutationOptions.onSuccess(task); + return data; + } + }); + + // Assert + expect(client.getQueryData(key)).toEqual([task]); + + // Act + await client.fetchQuery(queryOptions); + + // Assert + expect(client.getQueryData(key)).toEqual([]); + }); + + it.each([ + ['2026-01-01T10:00:00Z', '2026-01-01T10:01:00Z'], + ['2026-01-01T10:00:00.1234566Z', '2026-01-01T10:00:00.1234567Z'] + ])('ignores a cancellation snapshot at %s after a pushed completion at %s', async (older, newer) => { + // Arrange + const running = createTask({ updated_utc: older }); + client.setQueryData(key, [running]); + const pending = Promise.withResolvers(); + mocks.getJSON.mockReset().mockImplementation(() => pending.promise.then((data) => ({ data }))); + const read = client.fetchQuery(getTaskQueryOptions()); + const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: newer }); + client.setQueryData(key, [completed]); + cancelTask(); + const options = mocks.createMutation.mock.calls[0]![0](); + + // Act + await options.onSuccess(running); + pending.resolve([running]); + await read; + + // Assert + expect(client.getQueryData(key)).toEqual([completed]); + }); + + it.each([ + [BackgroundTaskStatus.Failed, BackgroundTaskStatus.Running], + [BackgroundTaskStatus.Running, BackgroundTaskStatus.Completed] + ])('preserves a pushed %s -> %s transition when an earlier list request finishes', async (previousStatus, nextStatus) => { + // Arrange + const previous = createTask({ status: previousStatus }); + client.setQueryData(key, [previous]); + const pending = Promise.withResolvers(); + mocks.getJSON.mockReset().mockImplementation(() => pending.promise.then((data) => ({ data }))); + const read = client.fetchQuery(getTaskQueryOptions()); + + // Act + await updateTaskFromMessage(client, { ...createUpdate(previous.id), status: nextStatus }); + pending.resolve([previous]); + await read; + + // Assert + expect(client.getQueryData(key)?.[0]).toMatchObject({ status: nextStatus, updated_utc: '2026-01-01T10:00:09Z' }); + }); + + it('retains concurrent pushed updates for different tasks', async () => { + // Arrange + client.setQueryData(key, [createTask(), createTask({ id: 'task-2' })]); + + // Act + await Promise.all([updateTaskFromMessage(client, createUpdate('task-1')), updateTaskFromMessage(client, createUpdate('task-2'))]); + + // Assert + expect(client.getQueryData(key)?.map((task) => task.processed)).toEqual([75, 75]); + }); + + it('finishes reconciliation during continuous progress without rolling it back or retaining unchanged omitted tasks', async () => { + // Arrange + const running = createTask(); + const expired = createTask({ id: 'expired', status: BackgroundTaskStatus.Completed }); + const unknown = createTask({ id: 'started-elsewhere' }); + const pending = Promise.withResolvers<{ data: ViewBackgroundTask[] }>(); + const signals: AbortSignal[] = []; + mocks.getJSON + .mockReset() + .mockResolvedValueOnce({ data: [running, expired] }) + .mockImplementation((_url: string, { signal }: { signal: AbortSignal }) => { + signals.push(signal); + return pending.promise; + }); + getTasksQuery({ route: { organizationId: 'organization-id' } }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, options()); + const unsubscribe = observer.subscribe(() => {}); + + try { + await observer.getCurrentQuery().promise; + + // Act + await updateTaskFromMessage(client, createUpdate(unknown.id)); + for (let progress = 1; progress <= 5; progress++) { + await updateTaskFromMessage(client, { + ...createUpdate(running.id), + processed: progress, + updated_utc: `2026-01-01T10:00:0${progress}Z` + }); + } + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + expect(signals.map((signal) => signal.aborted)).toEqual([false]); + + // Act + pending.resolve({ data: [running, unknown] }); + await observer.getCurrentQuery().promise; + + // Assert + expect(observer.getCurrentResult().data).toEqual([ + { ...running, message: null, processed: 5, progress: 75, succeeded: 75, updated_utc: '2026-01-01T10:00:05Z' }, + unknown + ]); + } finally { + unsubscribe(); + } + }); + + it.each(['pushed completion', 'cancellation response'] as const)( + 'finishes discovering an unknown task when a %s arrives during the list request', + async (source) => { + // Arrange + const running = createTask(); + const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: '2026-01-01T10:00:09Z' }); + const unknown = createTask({ id: 'started-elsewhere' }); + client.setQueryData(key, [running]); + const pending = Promise.withResolvers(); + mocks.getJSON.mockReset().mockImplementation(() => pending.promise.then((data) => ({ data }))); + const observer = new QueryObserver(client, { ...getTaskQueryOptions(), staleTime: Infinity }); + const unsubscribe = observer.subscribe(() => {}); + + // Act + await updateTaskFromMessage(client, createUpdate(unknown.id)); + if (source === 'pushed completion') { + await updateTaskFromMessage(client, { ...completed, message: undefined, project_id: undefined }); + } else { + cancelTask(); + await mocks.createMutation.mock.calls[0]![0]().onSuccess(completed); + } + + // Assert + expect(client.getQueryData(key)).toMatchObject([completed]); + expect(mocks.getJSON).toHaveBeenCalledOnce(); + + // Act + pending.resolve([completed, unknown]); + await client.getQueryCache().find({ queryKey: key })?.promise; + unsubscribe(); + + // Assert + expect(client.getQueryData(key)).toEqual([completed, unknown]); + } + ); + + it('keeps unknown-task reconciliation pending when the panel unmounts before the list is merged', async () => { + // Arrange + const running = createTask(); + const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: '2026-01-01T10:00:09Z' }); + client.setQueryData(key, [running]); + const pending = Promise.withResolvers(); + const queryFn = vi.fn(({ signal }: { signal: AbortSignal }) => { + void signal; + return pending.promise; + }); + const observer = new QueryObserver(client, { queryFn, queryKey: key, staleTime: Infinity }); + const unsubscribe = observer.subscribe(() => {}); + + // Act + await updateTaskFromMessage(client, createUpdate('started-elsewhere')); + unsubscribe(); + await updateTaskFromMessage(client, { ...completed, message: undefined, project_id: undefined }); + + // Assert + expect(client.getQueryData(key)).toMatchObject([completed]); + expect(client.getQueryState(key)?.isInvalidated).toBe(true); + expect(queryFn).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte new file mode 100644 index 0000000000..aa74ad30f5 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte @@ -0,0 +1,99 @@ + + +
+
+ +
+ {announcement} +

{task.description}

+

+ {#if task.status === BackgroundTaskStatus.Queued} + Waiting to start + {:else if task.status === BackgroundTaskStatus.Running} + {#if counts} + of + {:else} + {percent}%{#if task.message}{` · ${task.message}`}{/if} + {/if} + {:else if task.status === BackgroundTaskStatus.Completed} + {#if counts} + Completed · processed{#if task.failed > 0}, failed{/if} + {:else} + Completed{#if task.message}{` · ${task.message}`}{/if} + {/if} + {:else if task.status === BackgroundTaskStatus.Cancelled} + {#if counts} + Cancelled after of + {:else} + Cancelled + {/if} + {:else if counts} + Failed after of {#if task.message}{` · ${task.message}`}{/if} + {:else} + Failed{#if task.message}{` · ${task.message}`}{/if} + {/if} +

+
+ {#if isActive} + {#if canCancel} + + {/if} + {:else} + + {/if} +
+ {#if isActive || task.status === BackgroundTaskStatus.Completed} + + {/if} +
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts new file mode 100644 index 0000000000..82b4a25bbf --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts @@ -0,0 +1,178 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from '../models'; +import TaskItem from './task-item.svelte'; + +function createTask(overrides: Partial = {}): ViewBackgroundTask { + return { + can_cancel: true, + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Delete 1,234 events matching filter', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 500, + progress: 0, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BackgroundTaskStatus.Running, + succeeded: 500, + total: 1234, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:00Z', + ...overrides + }; +} + +function statusText(): string { + return (document.querySelector('[data-slot="task-progress"]')?.textContent ?? '').replace(/\s+/g, ' ').trim(); +} + +describe('TaskItem', () => { + afterEach(cleanup); + + it('shows the description, x of y progress and a Cancel button while running', async () => { + // Arrange + const onCancel = vi.fn(); + const task = createTask(); + render(TaskItem, { props: { canCancel: true, onCancel, onDismiss: vi.fn(), task } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); + + // Assert + expect(screen.getByText('Delete 1,234 events matching filter')).toBeTruthy(); + expect(statusText()).toBe('500 of 1,234'); + expect(screen.getByRole('progressbar').getAttribute('aria-valuenow')).toBe('41'); + expect(onCancel).toHaveBeenCalledWith(task); + expect(screen.queryByRole('button', { name: /^Dismiss / })).toBeNull(); + }); + + it('names the task on its buttons and announces only state changes, not every progress update', async () => { + // Arrange + const { rerender } = render(TaskItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ processed: 100 }) } }); + const announcement = () => screen.getByRole('status').textContent; + const initial = announcement(); + + // Act + await rerender({ task: createTask({ processed: 900 }) }); + const afterProgress = announcement(); + await rerender({ task: createTask({ processed: 1234, status: BackgroundTaskStatus.Completed }) }); + + // Assert + expect(initial).toBe('Delete 1,234 events matching filter: in progress'); + expect(afterProgress).toBe(initial); + expect(announcement()).toBe('Delete 1,234 events matching filter: completed'); + expect(screen.getByRole('button', { name: 'Dismiss Delete 1,234 events matching filter' })).toBeTruthy(); + }); + + it('labels Cancel with the task it cancels', () => { + // Arrange + const props = { canCancel: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() }; + + // Act + render(TaskItem, { props }); + + // Assert + expect(screen.getByRole('button', { name: 'Cancel Delete 1,234 events matching filter' })).toBeTruthy(); + }); + + it('shows the queued state without a count', () => { + // Arrange + const props = { + canCancel: true, + onCancel: vi.fn(), + onDismiss: vi.fn(), + task: createTask({ processed: 0, status: BackgroundTaskStatus.Queued }) + }; + + // Act + render(TaskItem, { props }); + + // Assert + expect(statusText()).toBe('Waiting to start'); + expect(screen.getByRole('button', { name: /^Cancel / })).toBeTruthy(); + }); + + it('disables Cancel once cancellation was requested', () => { + // Arrange + const props = { canCancel: true, isCancelling: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() }; + + // Act + render(TaskItem, { props }); + + // Assert + expect((screen.getByRole('button', { name: /^Cancel / }) as HTMLButtonElement).disabled).toBe(true); + }); + + it('shows a completed task with its failures and a Dismiss button', async () => { + // Arrange + const onDismiss = vi.fn(); + const task = createTask({ failed: 2, processed: 1234, status: BackgroundTaskStatus.Completed, succeeded: 1232 }); + render(TaskItem, { props: { onCancel: vi.fn(), onDismiss, task } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + + // Assert + expect(statusText()).toBe('Completed · 1,232 processed, 2 failed'); + expect(screen.queryByRole('button', { name: /^Cancel / })).toBeNull(); + expect(onDismiss).toHaveBeenCalledWith(task); + }); + + it('shows how far a cancelled task got', () => { + // Arrange + const props = { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ status: BackgroundTaskStatus.Cancelled }) }; + + // Act + render(TaskItem, { props }); + + // Assert + expect(statusText()).toBe('Cancelled after 500 of 1,234'); + expect(screen.queryByRole('progressbar')).toBeNull(); + }); + + it('shows why a task failed', () => { + // Arrange + const props = { + onCancel: vi.fn(), + onDismiss: vi.fn(), + task: createTask({ message: 'The task stopped because of an unexpected error.', status: BackgroundTaskStatus.Failed }) + }; + + // Act + render(TaskItem, { props }); + + // Assert + expect(statusText()).toBe('Failed after 500 of 1,234 · The task stopped because of an unexpected error.'); + }); + + it('shows the percent and message of a task that has no item counts', () => { + // Arrange + const props = { + onCancel: vi.fn(), + onDismiss: vi.fn(), + task: createTask({ can_cancel: false, message: 'Events removed: 12', processed: 0, progress: 50, total: 0, type: 'projects.reset-data' }) + }; + + // Act + render(TaskItem, { props }); + + // Assert + expect(statusText()).toBe('50% · Events removed: 12'); + expect(screen.getByRole('progressbar').getAttribute('aria-valuenow')).toBe('50'); + }); + + it('does not offer Cancel for a running task when cancelling is not allowed', () => { + // Arrange + const props = { canCancel: false, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ can_cancel: false }) }; + + // Act + render(TaskItem, { props }); + + // Assert + expect(screen.queryByRole('button', { name: /^Cancel / })).toBeNull(); + expect(screen.queryByRole('button', { name: /^Dismiss / })).toBeNull(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte new file mode 100644 index 0000000000..9dc96ec236 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte @@ -0,0 +1,140 @@ + + +{#if visibleTasks.length > 0} +
+ {#each visibleTasks as task (task.id)} + + {/each} +
+{/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts new file mode 100644 index 0000000000..5b729874f9 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts @@ -0,0 +1,434 @@ +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { tick } from 'svelte'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { organization } from '#features/organizations/context.svelte.ts'; + +import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from '../models'; + +const mocks = vi.hoisted(() => ({ + cancelMutateAsync: vi.fn(), + invalidateAffected: vi.fn(), + queryClient: { id: 'query-client' }, + startPolling: vi.fn(), + stopPolling: vi.fn(), + updateFromMessage: vi.fn() +})); + +vi.mock('@tanstack/svelte-query', () => ({ useQueryClient: () => mocks.queryClient })); +vi.mock('svelte-sonner', () => ({ toast: { error: vi.fn(), success: vi.fn() } })); +vi.mock('#features/organizations/context.svelte.ts', async () => { + const { SvelteMap } = await import('svelte/reactivity'); + const state = new SvelteMap([['current', 'organization-id']]); + return { + organization: { + get current() { + return state.get('current'); + }, + set current(value: string | undefined) { + state.set('current', value); + } + } + }; +}); +vi.mock('../api.svelte', async () => { + const { SvelteMap } = await import('svelte/reactivity'); + const state = new SvelteMap(); + + return { + __state: state, + cancelTask: () => ({ mutateAsync: mocks.cancelMutateAsync }), + getTasksQuery: () => ({ + get data() { + return state.get('tasks'); + } + }), + invalidateTaskAffectedQueries: mocks.invalidateAffected, + startTaskPolling: mocks.startPolling, + updateTaskFromMessage: mocks.updateFromMessage + }; +}); + +import * as api from '../api.svelte'; +import TaskPanel from './task-panel.svelte'; + +const state = (api as unknown as { __state: Map }).__state; + +function createTask(overrides: Partial = {}): ViewBackgroundTask { + const now = new Date().toISOString(); + return { + can_cancel: true, + created_by_user_id: 'user-id', + created_utc: now, + description: 'Delete 50 events matching filter', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 20, + progress: 0, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BackgroundTaskStatus.Running, + succeeded: 20, + total: 50, + type: 'events.delete', + updated_utc: now, + ...overrides + }; +} + +describe('TaskPanel', () => { + beforeEach(() => { + state.clear(); + organization.current = 'organization-id'; + mocks.cancelMutateAsync.mockResolvedValue(undefined); + mocks.startPolling.mockReturnValue(mocks.stopPolling); + }); + + afterEach(() => { + cleanup(); + vi.useRealTimers(); + }); + + it('hydrates running tasks and reconciles older outcomes without displaying them', async () => { + // Arrange + state.set('tasks', [createTask(), createTask({ id: 'long-finished', status: BackgroundTaskStatus.Completed, updated_utc: '2020-01-01T00:00:00Z' })]); + + // Act + render(TaskPanel); + await tick(); + + // Assert + expect(screen.getAllByText('Delete 50 events matching filter')).toHaveLength(1); + expect(document.querySelector('[data-slot="task-progress"]')?.textContent?.replace(/\s+/g, ' ').trim()).toBe('20 of 50'); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + + // Act + state.set( + 'tasks', + state.get('tasks')!.map((task) => ({ ...task })) + ); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + }); + + it('keeps polling across progress and replaces it only when activity or organization changes', async () => { + // Arrange + const running = createTask(); + state.set('tasks', [running]); + render(TaskPanel); + await tick(); + + // Act + for (let progress = 1; progress <= 5; progress++) { + state.set('tasks', [{ ...running, processed: progress }]); + await tick(); + } + + // Assert + expect(mocks.startPolling).toHaveBeenCalledExactlyOnceWith(mocks.queryClient, 'organization-id', true); + expect(mocks.stopPolling).not.toHaveBeenCalled(); + + // Act + state.set('tasks', [{ ...running, status: BackgroundTaskStatus.Completed }]); + await tick(); + + // Assert + expect(mocks.startPolling).toHaveBeenLastCalledWith(mocks.queryClient, 'organization-id', false); + expect(mocks.stopPolling).toHaveBeenCalledOnce(); + + // Act + organization.current = 'another-organization'; + state.set('tasks', []); + await tick(); + + // Assert + expect(mocks.startPolling).toHaveBeenLastCalledWith(mocks.queryClient, 'another-organization', false); + expect(mocks.stopPolling).toHaveBeenCalledTimes(2); + + // Act + cleanup(); + + // Assert + expect(mocks.stopPolling).toHaveBeenCalledTimes(3); + }); + + it('applies pushed updates for its organization and ignores other organizations', async () => { + // Arrange + state.set('tasks', [createTask()]); + render(TaskPanel); + const update = { id: 'task-1', organization_id: 'organization-id', processed: 30, status: BackgroundTaskStatus.Running }; + + // Act + document.dispatchEvent(new CustomEvent('BackgroundTaskUpdated', { detail: update })); + document.dispatchEvent(new CustomEvent('BackgroundTaskUpdated', { detail: { ...update, organization_id: 'another-organization' } })); + + // Assert + expect(mocks.updateFromMessage).toHaveBeenCalledTimes(1); + expect(mocks.updateFromMessage).toHaveBeenCalledWith(mocks.queryClient, update); + }); + + it.each([1, 3])('refreshes events and stacks once when %i running tasks finish in one update', async (count) => { + // Arrange + const running = Array.from({ length: count }, (_, index) => createTask({ id: `task-${index}` })); + const completed = running.map((task) => ({ ...task, processed: 50, status: BackgroundTaskStatus.Completed, succeeded: 50 })); + state.set('tasks', running); + render(TaskPanel); + await tick(); + + // Act + state.set('tasks', completed); + await tick(); + state.set( + 'tasks', + completed.map((task) => ({ ...task })) + ); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(1); + expect(mocks.invalidateAffected).toHaveBeenCalledWith(mocks.queryClient); + expect(document.querySelector('[data-slot="task-progress"]')?.textContent).toContain('Completed'); + }); + + it('reconciles an active task omitted from the finished history only once', async () => { + // Arrange + const finished = createTask({ id: 'known-finished', status: BackgroundTaskStatus.Completed, updated_utc: '2020-01-01T00:00:00Z' }); + state.set('tasks', [createTask(), finished]); + + // Act + render(TaskPanel); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + + // Act + // A completed task can fall outside the API's finished-history limit before its terminal update arrives. + state.set('tasks', [finished]); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); + + // Act + state.set('tasks', [{ ...finished }]); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); + }); + + it('requests cancellation of a running task', async () => { + // Arrange + state.set('tasks', [createTask()]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); + + // Assert + await waitFor(() => expect(mocks.cancelMutateAsync).toHaveBeenCalledWith('task-1')); + }); + + it('allows cancellation to be retried after the request finishes while the worker is still running', async () => { + // Arrange + const request = Promise.withResolvers(); + mocks.cancelMutateAsync.mockReturnValueOnce(request.promise); + state.set('tasks', [createTask()]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); + + // Assert + expect(screen.getByRole('button', { name: /^Cancel / })).toBeDisabled(); + + // Act + request.resolve(createTask()); + await tick(); + + // Assert + await waitFor(() => expect(screen.getByRole('button', { name: /^Cancel / })).toBeEnabled()); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); + + // Assert + expect(mocks.cancelMutateAsync).toHaveBeenCalledTimes(2); + }); + + it('removes a finished task when it is dismissed', async () => { + // Arrange + state.set('tasks', [createTask({ status: BackgroundTaskStatus.Completed })]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + + // Assert + expect(screen.queryByText('Delete 50 events matching filter')).toBeNull(); + }); + + it('shows a dismissed failed task again during and after an automatic retry', async () => { + // Arrange + state.set('tasks', [createTask({ status: BackgroundTaskStatus.Failed })]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + + // Assert + expect(screen.queryByText('Delete 50 events matching filter')).toBeNull(); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + + // Act + state.set('tasks', [createTask()]); + await tick(); + + // Assert + expect(screen.getByRole('button', { name: /^Cancel / })).toBeEnabled(); + + // Act + state.set('tasks', [createTask({ processed: 50, status: BackgroundTaskStatus.Completed, succeeded: 50 })]); + await tick(); + + // Assert + expect(document.querySelector('[data-slot="task-progress"]')?.textContent).toContain('Completed'); + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); + }); + + it.each([BackgroundTaskStatus.Completed, BackgroundTaskStatus.Failed])( + 'shows and reconciles a newer %s retry outcome when the running update was missed', + async (status) => { + // Arrange + vi.useFakeTimers(); + vi.setSystemTime('2026-01-01T10:00:00.123Z'); + const failed = createTask({ status: BackgroundTaskStatus.Failed, updated_utc: '2026-01-01T10:00:00.1234566Z' }); + const outcome = { ...failed, status, updated_utc: '2026-01-01T10:00:00.1234567Z' }; + state.set('tasks', [failed]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + state.set('tasks', [outcome]); + await tick(); + + // Assert + expect(screen.getByRole('button', { name: /^Dismiss / })).toBeVisible(); + expect(document.querySelector('[data-slot="task"]')).toHaveAttribute('data-status', status); + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + state.set('tasks', [{ ...outcome }]); + await tick(); + + // Assert + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); + } + ); + + it.each(['unavailable', 'empty'])('reconciles a newly discovered completed task when its initial list was %s', async (initialList) => { + // Arrange + // Event and stack views can finish loading while the first task list is still unavailable. + if (initialList === 'empty') { + state.set('tasks', []); + } + render(TaskPanel); + await tick(); + const completed = createTask({ status: BackgroundTaskStatus.Completed }); + + // Act + state.set('tasks', [completed]); + await tick(); + state.set('tasks', [{ ...completed }]); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + }); + + it('preserves a dismissal across organization changes and shows a retry that finished while away', async () => { + // Arrange + const failed = createTask({ status: BackgroundTaskStatus.Failed }); + const completed = { ...failed, status: BackgroundTaskStatus.Completed, updated_utc: new Date(Date.now() + 1000).toISOString() }; + state.set('tasks', [failed]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + organization.current = 'another-organization'; + state.set('tasks', []); + await tick(); + document.dispatchEvent(new CustomEvent('BackgroundTaskUpdated', { detail: completed })); + organization.current = 'organization-id'; + state.set('tasks', [failed]); + await tick(); + + // Assert + expect(mocks.updateFromMessage).not.toHaveBeenCalled(); + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); + + // Act + state.set('tasks', [completed]); + await tick(); + + // Assert + expect(screen.getByRole('button', { name: /^Dismiss / })).toBeVisible(); + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); + }); + + it('renders nothing when there is nothing to show', () => { + // Arrange + state.clear(); + + // Act + render(TaskPanel); + + // Assert + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); + }); + + it('does not show expired outcomes discovered after the panel was idle', async () => { + // Arrange + vi.useFakeTimers(); + vi.setSystemTime('2026-01-01T10:00:00Z'); + state.set('tasks', []); + render(TaskPanel); + await tick(); + + // Act + await vi.advanceTimersByTimeAsync(10 * 60 * 1000); + state.set('tasks', [createTask({ status: BackgroundTaskStatus.Completed, updated_utc: '2026-01-01T10:05:00Z' })]); + await tick(); + + // Assert + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + }); + + it('expires a finished task while another task keeps reporting progress', async () => { + // Arrange + vi.useFakeTimers(); + vi.setSystemTime('2026-01-01T10:00:00Z'); + const finished = createTask({ description: 'Finished task', id: 'finished', status: BackgroundTaskStatus.Completed }); + state.set('tasks', [createTask(), finished]); + render(TaskPanel); + await tick(); + + // Act + for (let progress = 1; progress <= 25; progress++) { + await vi.advanceTimersByTimeAsync(5000); + state.set('tasks', [createTask({ processed: progress }), finished]); + await tick(); + } + + // Assert + expect(screen.queryByText('Finished task')).toBeNull(); + expect(screen.getByText('Delete 50 events matching filter')).toBeVisible(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts new file mode 100644 index 0000000000..76cd5beb65 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, it } from 'vitest'; + +import { + applyTaskUpdate, + BackgroundTaskStatus, + BulkOperationSelectionKind, + canCancelTask, + FINISHED_TASK_VISIBLE_MS, + getProgressPercent, + mergeTaskList, + selectVisibleTasks, + STALE_TASK_MS, + type ViewBackgroundTask +} from './models'; + +function createTask(overrides: Partial = {}): ViewBackgroundTask { + return { + can_cancel: true, + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Delete events matching filter', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 0, + progress: 0, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BackgroundTaskStatus.Running, + succeeded: 0, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:00Z', + ...overrides + }; +} + +function createUpdate(overrides: Record = {}) { + return { + can_cancel: true, + description: 'Delete events matching filter', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 0, + progress: 0, + status: BackgroundTaskStatus.Running, + succeeded: 0, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:05Z', + ...overrides + }; +} + +describe('applyTaskUpdate', () => { + it('patches the progress of a known task', () => { + // Arrange + const tasks = [createTask(), createTask({ id: 'task-2' })]; + + // Act + const result = applyTaskUpdate(tasks, createUpdate({ failed: 1, processed: 40, succeeded: 39 })); + + // Assert + expect(result?.[0]).toMatchObject({ failed: 1, processed: 40, succeeded: 39 }); + expect(result?.[1]).toBe(tasks[1]); + }); + + it.each([ + ['2026-01-01T10:00:05Z', '2026-01-01T10:00:10Z'], + ['2026-01-01T10:00:00.1234566Z', '2026-01-01T10:00:00.1234567Z'], + ['2026-01-01T10:00:00Z', '2026-01-01T10:00:00.0000001Z'] + ])('ignores a message at %s when a newer completion at %s is already known', (older, newer) => { + // Arrange + const tasks = [createTask({ processed: 100, status: BackgroundTaskStatus.Completed, updated_utc: newer })]; + + // Act + const result = applyTaskUpdate(tasks, createUpdate({ processed: 20, updated_utc: older })); + + // Assert + expect(result).toBe(tasks); + }); + + it('accepts a newer retry within the same millisecond', () => { + // Arrange + const tasks = [createTask({ status: BackgroundTaskStatus.Failed, updated_utc: '2026-01-01T10:00:00.1234566Z' })]; + + // Act + const result = applyTaskUpdate(tasks, createUpdate({ updated_utc: '2026-01-01T10:00:00.1234567Z' })); + + // Assert + expect(result?.[0]).toMatchObject({ status: BackgroundTaskStatus.Running, updated_utc: '2026-01-01T10:00:00.1234567Z' }); + }); + + it('reports an unknown task so the list can be reloaded', () => { + // Arrange + const tasks = [createTask()]; + + // Act + const result = applyTaskUpdate(tasks, createUpdate({ id: 'started-elsewhere', status: BackgroundTaskStatus.Queued, total: 5 })); + + // Assert + expect(result).toBeUndefined(); + }); +}); + +describe('mergeTaskList', () => { + it('keeps newer cached revisions and accepts complete server metadata on equal revisions', () => { + // Arrange + const older = createTask({ updated_utc: '2026-01-01T10:00:00.1234566Z' }); + const newer = { ...older, processed: 50, updated_utc: '2026-01-01T10:00:00.1234567Z' }; + const cached = createTask({ id: 'complete-metadata', status: BackgroundTaskStatus.Completed }); + const fromServer = { ...cached, completed_utc: '2026-01-01T10:00:00Z' }; + + // Act + const merged = mergeTaskList([older, fromServer], [newer, cached], [older, cached]); + + // Assert + expect(merged).toEqual([newer, fromServer]); + }); + + it('preserves only omitted tasks added or advanced during the read', () => { + // Arrange + const unchanged = createTask({ id: 'unchanged' }); + const previous = createTask({ id: 'advanced', updated_utc: '2026-01-01T10:00:00.1234566Z' }); + const advanced = { ...previous, updated_utc: '2026-01-01T10:00:00.1234567Z' }; + const inserted = createTask({ id: 'inserted' }); + + // Act + const merged = mergeTaskList([], [unchanged, advanced, inserted], [unchanged, previous]); + + // Assert + expect(merged).toEqual([advanced, inserted]); + }); +}); + +describe('getProgressPercent', () => { + it.each([ + [{ processed: 0, progress: 0, status: BackgroundTaskStatus.Queued, total: 0 }, 0], + [{ processed: 50, progress: 0, status: BackgroundTaskStatus.Running, total: 200 }, 25], + [{ processed: 5, progress: 0, status: BackgroundTaskStatus.Running, total: 3 }, 100], + [{ processed: 3, progress: 0, status: BackgroundTaskStatus.Completed, total: 0 }, 100], + [{ processed: 0, progress: 42, status: BackgroundTaskStatus.Running, total: 0 }, 42] + ])('maps %j to %i percent', (overrides, expected) => { + // Arrange + const task = createTask(overrides); + + // Act + const percent = getProgressPercent(task); + + // Assert + expect(percent).toBe(expected); + }); +}); + +describe('canCancelTask', () => { + const updated = new Date('2026-01-01T10:00:00Z').getTime(); + + it.each([ + ['a cancellable running task', { can_cancel: true }, updated, true], + ['a running task that cannot be cancelled', { can_cancel: false }, updated + STALE_TASK_MS - 1, false], + ['an abandoned task that cannot be cancelled', { can_cancel: false }, updated + STALE_TASK_MS, true], + ['a finished task', { can_cancel: true, status: BackgroundTaskStatus.Completed }, updated, false] + ])('%s: %j', (_name, overrides, now, expected) => { + // Arrange + const task = createTask(overrides); + + // Act + const result = canCancelTask(task, now); + + // Assert + expect(result).toBe(expected); + }); +}); + +describe('selectVisibleTasks', () => { + const now = new Date('2026-01-01T10:30:00Z').getTime(); + + it('keeps active tasks and recently finished ones, newest first', () => { + // Arrange + const active = createTask({ created_utc: '2026-01-01T09:00:00Z', id: 'active', updated_utc: '2026-01-01T09:00:00Z' }); + const recent = createTask({ + created_utc: '2026-01-01T10:20:00Z', + id: 'recent', + status: BackgroundTaskStatus.Completed, + updated_utc: new Date(now - FINISHED_TASK_VISIBLE_MS + 1000).toISOString() + }); + const old = createTask({ + id: 'old', + status: BackgroundTaskStatus.Completed, + updated_utc: new Date(now - FINISHED_TASK_VISIBLE_MS - 1000).toISOString() + }); + + // Act + const result = selectVisibleTasks([active, old, recent], new Map(), now); + + // Assert + expect(result.map((task) => task.id)).toEqual(['recent', 'active']); + }); + + it('hides dismissed finished tasks', () => { + // Arrange + const task = createTask({ id: 'dismissed', status: BackgroundTaskStatus.Failed }); + + // Act + const result = selectVisibleTasks([task], new Map([['dismissed', task.updated_utc]]), now); + + // Assert + expect(result).toEqual([]); + }); + + it.each([BackgroundTaskStatus.Queued, BackgroundTaskStatus.Running])('shows a dismissed failed task when it is retried as %s', (status) => { + // Arrange + const task = createTask({ id: 'retried', status }); + + // Act + const result = selectVisibleTasks([task], new Map([['retried', task.updated_utc]]), now); + + // Assert + expect(result).toEqual([task]); + }); + + it.each([ + ['2026-01-01T10:29:59.1234566Z', false], + ['2026-01-01T10:29:59.1234567Z', false], + ['2026-01-01T10:29:59.1234568Z', true] + ] as const)('only shows a terminal outcome at %s when it is newer than the dismissed revision', (updatedUtc, visible) => { + // Arrange + const task = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: updatedUtc }); + const dismissed = new Map([[task.id, '2026-01-01T10:29:59.1234567Z']]); + + // Act + const result = selectVisibleTasks([task], dismissed, now); + + // Assert + expect(result).toEqual(visible ? [task] : []); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts new file mode 100644 index 0000000000..59958a2414 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts @@ -0,0 +1,126 @@ +import type { BackgroundTaskUpdated } from '#features/websockets/models.ts'; + +import { BackgroundTaskStatus, BulkOperationSelectionKind, type NewBulkOperation, type ViewBackgroundTask } from '#generated/api.ts'; + +export { BackgroundTaskStatus, BulkOperationSelectionKind, type NewBulkOperation, type ViewBackgroundTask }; + +export const EVENTS_DELETE_OPERATION = 'events.delete'; + +/** Finished tasks stay visible in the panel for this long, unless dismissed. */ +export const FINISHED_TASK_VISIBLE_MS = 2 * 60 * 1000; + +export function applyTaskUpdate(tasks: ViewBackgroundTask[], update: BackgroundTaskUpdated): undefined | ViewBackgroundTask[] { + const index = tasks.findIndex((task) => task.id === update.id); + if (index < 0) { + return undefined; + } + + const current = tasks[index]!; + if (compareTaskUpdateTimes(update.updated_utc, current.updated_utc) < 0) { + // Messages can arrive out of order; never move a task backwards. + return tasks; + } + + const next = [...tasks]; + next[index] = { + ...current, + can_cancel: update.can_cancel, + description: update.description, + failed: update.failed, + message: update.message ?? null, + processed: update.processed, + progress: update.progress, + status: update.status, + succeeded: update.succeeded, + total: update.total, + updated_utc: update.updated_utc + }; + + return next; +} + +/** Date.parse truncates the four submillisecond digits of .NET timestamps; retain them when ordering updates. */ +export function compareTaskUpdateTimes(left: string, right: string): number { + const milliseconds = Date.parse(left) - Date.parse(right); + if (milliseconds !== 0) { + return milliseconds; + } + + const remainingTicks = (value: string) => Number((value.match(/\.(\d+)/)?.[1] ?? '').padEnd(7, '0').slice(3, 7)); + return remainingTicks(left) - remainingTicks(right); +} + +/** Tasks that report item counts show those; tasks that only report a percent (total of zero) show the percent. */ +export function getProgressPercent(task: Pick): number { + if (task.status === BackgroundTaskStatus.Completed) { + return 100; + } + + if (task.total <= 0) { + return Math.min(100, Math.max(0, task.progress ?? 0)); + } + + return Math.min(100, Math.round((task.processed / task.total) * 100)); +} + +export function hasCounts(task: Pick): boolean { + return task.total > 0; +} + +/** Reconciles a list read without discarding task changes received while that read was in flight. */ +export function mergeTaskList( + tasks: ViewBackgroundTask[], + current: readonly ViewBackgroundTask[] | undefined, + baseline: readonly ViewBackgroundTask[] +): ViewBackgroundTask[] { + const remaining = new Map((current ?? []).map((task) => [task.id, task])); + const merged = tasks.map((task) => { + const cached = remaining.get(task.id); + remaining.delete(task.id); + return cached && compareTaskUpdateTimes(cached.updated_utc, task.updated_utc) > 0 ? cached : task; + }); + + const initialUpdates = new Map(baseline.map((task) => [task.id, task.updated_utc])); + for (const task of remaining.values()) { + const initialUpdatedUtc = initialUpdates.get(task.id); + if (!initialUpdatedUtc || compareTaskUpdateTimes(task.updated_utc, initialUpdatedUtc) > 0) { + merged.push(task); + } + } + + return merged; +} + +/** An active task that has not reported progress for this long is abandoned, and the server lets anyone clear it. */ +export const STALE_TASK_MS = 2 * 60 * 60 * 1000; + +/** Whether the panel offers Cancel: the task can be stopped, or it is abandoned and only needs clearing. */ +export function canCancelTask(task: Pick, now: number): boolean { + if (!isTaskActive(task.status)) { + return false; + } + + return task.can_cancel || now - new Date(task.updated_utc).getTime() >= STALE_TASK_MS; +} + +export function isTaskActive(status: BackgroundTaskStatus): boolean { + return status === BackgroundTaskStatus.Queued || status === BackgroundTaskStatus.Running; +} + +/** Active tasks plus recently finished ones the user has not dismissed, newest first. */ +export function selectVisibleTasks(tasks: ViewBackgroundTask[], dismissedUpdates: ReadonlyMap, now: number): ViewBackgroundTask[] { + return tasks + .filter((task) => { + if (isTaskActive(task.status)) { + return true; + } + + const dismissedUtc = dismissedUpdates.get(task.id); + if (dismissedUtc && compareTaskUpdateTimes(task.updated_utc, dismissedUtc) <= 0) { + return false; + } + + return now - new Date(task.updated_utc).getTime() < FINISHED_TASK_VISIBLE_MS; + }) + .sort((a, b) => new Date(b.created_utc).getTime() - new Date(a.created_utc).getTime()); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts index 461805e466..787564c9c8 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts @@ -1,9 +1,28 @@ +import type { BackgroundTaskStatus } from '#generated/api.ts'; + export enum ChangeType { Added = 0, Saved = 1, Removed = 2 } +export interface BackgroundTaskUpdated { + can_cancel: boolean; + description: string; + failed: number; + id: string; + message?: string; + organization_id: string; + processed: number; + progress: number; + project_id?: string; + status: BackgroundTaskStatus; + succeeded: number; + total: number; + type: string; + updated_utc: string; +} + export interface EntityChanged { change_type: ChangeType; data: Record; @@ -49,19 +68,22 @@ export interface WebSocketMessage { type: T; } -export type WebSocketMessageType = 'PlanChanged' | 'PlanOverage' | 'ReleaseNotification' | 'SystemNotification' | 'UserMembershipChanged' | `${string}Changed`; +export type WebSocketMessageType = + 'BackgroundTaskUpdated' | 'PlanChanged' | 'PlanOverage' | 'ReleaseNotification' | 'SystemNotification' | 'UserMembershipChanged' | `${string}Changed`; -export type WebSocketMessageValue = T extends 'PlanChanged' - ? PlanChanged - : T extends 'PlanOverage' - ? PlanOverage - : T extends 'UserMembershipChanged' - ? UserMembershipChanged - : T extends 'ReleaseNotification' - ? ReleaseNotification - : T extends 'SystemNotification' - ? SystemNotification - : EntityChanged; +export type WebSocketMessageValue = T extends 'BackgroundTaskUpdated' + ? BackgroundTaskUpdated + : T extends 'PlanChanged' + ? PlanChanged + : T extends 'PlanOverage' + ? PlanOverage + : T extends 'UserMembershipChanged' + ? UserMembershipChanged + : T extends 'ReleaseNotification' + ? ReleaseNotification + : T extends 'SystemNotification' + ? SystemNotification + : EntityChanged; export function isEntityChangedType(message: { message: unknown; type: WebSocketMessageType }): message is WebSocketMessage<`${string}Changed`> { return message.type !== 'PlanChanged' && message.type !== 'UserMembershipChanged' && message.type.endsWith('Changed'); @@ -73,7 +95,7 @@ export function isPlanOverageType(message: { message: unknown; type: WebSocketMe export function isWebSocketMessageType(type: string): type is WebSocketMessageType { return ( - (['PlanChanged', 'PlanOverage', 'UserMembershipChanged', 'ReleaseNotification', 'SystemNotification'] as const).includes( + (['BackgroundTaskUpdated', 'PlanChanged', 'PlanOverage', 'UserMembershipChanged', 'ReleaseNotification', 'SystemNotification'] as const).includes( type as Exclude ) || type.endsWith('Changed') ); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts index 6f4deae439..4b527ec299 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts @@ -7,6 +7,11 @@ export enum StackStatus { Discarded = "discarded", } +export enum BulkOperationSelectionKind { + Ids = "Ids", + Filter = "Filter", +} + export enum BillingStatus { Trialing = 0, Active = 1, @@ -15,6 +20,14 @@ export enum BillingStatus { Unpaid = 4, } +export enum BackgroundTaskStatus { + Queued = "Queued", + Running = "Running", + Completed = "Completed", + Failed = "Failed", + Cancelled = "Cancelled", +} + export interface AdminAssistantOrganizationUsage { organization_id: string; organization_name: string; @@ -242,6 +255,36 @@ export interface Login { invite_token?: null | string; } +export interface NewBulkOperation { + /** The action to run, e.g. events.delete. */ + type: string; + /** @pattern ^[a-fA-F0-9]{24}$ */ + organization_id: string; + /** + * Optionally limits the operation to one project of the organization. + * @pattern ^[a-fA-F0-9]{24}$ + */ + project_id?: null | string; + selection: NewBulkOperationSelection; + /** Action specific parameters. Actions without parameters reject any value here. */ + parameters?: null | Record; +} + +export interface NewBulkOperationSelection { + kind: BulkOperationSelectionKind; + /** The ids to process when Kind is ids. */ + ids?: string[] | null; + /** The search filter when Kind is filter. An empty filter matches everything in the time window. */ + filter?: null | string; + /** + * The time range of the filter, e.g. [now-7d TO now]. It is resolved to absolute UTC dates when the + * operation is created. + */ + time?: null | string; + /** The UTC offset used to resolve Time, e.g. -5h. */ + offset?: null | string; +} + export interface NewOrganization { name: string; } @@ -834,6 +877,72 @@ export interface VerifyEmailAddressResult { email_address: string; } +export interface ViewBackgroundTask { + /** @pattern ^[a-fA-F0-9]{24}$ */ + id: string; + /** @pattern ^[a-fA-F0-9]{24}$ */ + organization_id: string; + /** @pattern ^[a-fA-F0-9]{24}$ */ + project_id?: null | string; + /** What the task does, e.g. events.delete, projects.reset-data or projects.sample-data. */ + type: string; + description: string; + status: BackgroundTaskStatus; + /** + * Percent complete, 0 to 100. + * @format int32 + */ + progress: number; + /** + * The item counts. All zero for tasks that only report a percent. + * @format int64 + */ + total: number; + /** @format int64 */ + processed: number; + /** @format int64 */ + succeeded: number; + /** @format int64 */ + failed: number; + /** Whether the task can be stopped with the cancel endpoint. */ + can_cancel: boolean; + /** What a bulk action applies to. Only set for bulk actions. */ + selection?: null | ViewBulkOperationSelection; + /** @pattern ^[a-fA-F0-9]{24}$ */ + created_by_user_id: string; + /** @format date-time */ + created_utc: string; + /** @format date-time */ + started_utc?: null | string; + /** @format date-time */ + completed_utc?: null | string; + /** @format date-time */ + updated_utc: string; + message?: null | string; +} + +/** What a bulk action applies to. Only set for bulk actions. */ +export interface ViewBulkOperationSelection { + kind: BulkOperationSelectionKind; + /** + * The number of ids when the selection is an id list. + * @format int32 + */ + id_count?: null | number; + filter?: null | string; + time?: null | string; + /** + * The absolute start of the resolved time window. + * @format date-time + */ + utc_start?: null | string; + /** + * The absolute end of the resolved time window, never later than when the task was created. + * @format date-time + */ + utc_end?: null | string; +} + export interface ViewCurrentUser { hash?: null | string; has_local_account: boolean; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts index f005035a45..cebee6ff1d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts @@ -27,6 +27,7 @@ export const StackStatusSchema = zodEnum([ "ignored", "discarded", ]); +export const BulkOperationSelectionKindSchema = zodEnum(["Ids", "Filter"]); export const BillingStatusSchema = union([ literal(0), literal(1), @@ -34,6 +35,13 @@ export const BillingStatusSchema = union([ literal(3), literal(4), ]); +export const BackgroundTaskStatusSchema = zodEnum([ + "Queued", + "Running", + "Completed", + "Failed", + "Cancelled", +]); export const AdminAssistantOrganizationUsageSchema = object({ organization_id: string().min(1, "Organization id is required"), @@ -295,6 +303,46 @@ export const LoginSchema = object({ }); export type LoginFormData = Infer; +export const NewBulkOperationSchema = object({ + type: string() + .min(1, "Type is required") + .max(100, "Type must be at most 100 characters"), + organization_id: string() + .length(24, "Organization id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Organization id has invalid format"), + project_id: string() + .length(24, "Project id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Project id has invalid format") + .nullable() + .optional(), + selection: lazy(() => NewBulkOperationSelectionSchema), + parameters: record(string(), string()).nullable().optional(), +}); +export type NewBulkOperationFormData = Infer; + +export const NewBulkOperationSelectionSchema = object({ + kind: BulkOperationSelectionKindSchema, + ids: array(string()).nullable().optional(), + filter: string() + .min(1, "Filter is required") + .max(2000, "Filter must be at most 2000 characters") + .nullable() + .optional(), + time: string() + .min(1, "Time is required") + .max(100, "Time must be at most 100 characters") + .nullable() + .optional(), + offset: string() + .min(1, "Offset is required") + .max(20, "Offset must be at most 20 characters") + .nullable() + .optional(), +}); +export type NewBulkOperationSelectionFormData = Infer< + typeof NewBulkOperationSelectionSchema +>; + export const NewOrganizationSchema = object({ name: string().min(1, "Name is required"), }); @@ -976,6 +1024,51 @@ export type VerifyEmailAddressResultFormData = Infer< typeof VerifyEmailAddressResultSchema >; +export const ViewBackgroundTaskSchema = object({ + id: string() + .length(24, "Id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Id has invalid format"), + organization_id: string() + .length(24, "Organization id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Organization id has invalid format"), + project_id: string() + .length(24, "Project id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Project id has invalid format") + .nullable() + .optional(), + type: string().min(1, "Type is required"), + description: string().min(1, "Description is required"), + status: BackgroundTaskStatusSchema, + progress: int32(), + total: int(), + processed: int(), + succeeded: int(), + failed: int(), + can_cancel: boolean(), + selection: lazy(() => ViewBulkOperationSelectionSchema).optional(), + created_by_user_id: string() + .length(24, "Created by user id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Created by user id has invalid format"), + created_utc: iso.datetime(), + started_utc: iso.datetime().nullable().optional(), + completed_utc: iso.datetime().nullable().optional(), + updated_utc: iso.datetime(), + message: string().min(1, "Message is required").nullable().optional(), +}); +export type ViewBackgroundTaskFormData = Infer; + +export const ViewBulkOperationSelectionSchema = object({ + kind: BulkOperationSelectionKindSchema, + id_count: int32().nullable().optional(), + filter: string().min(1, "Filter is required").nullable().optional(), + time: string().min(1, "Time is required").nullable().optional(), + utc_start: iso.datetime().nullable().optional(), + utc_end: iso.datetime().nullable().optional(), +}); +export type ViewBulkOperationSelectionFormData = Infer< + typeof ViewBulkOperationSelectionSchema +>; + export const ViewCurrentUserSchema = object({ hash: string().min(1, "Hash is required").nullable().optional(), has_local_account: boolean(), diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index 677236305e..c74d254fbd 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -62,6 +62,7 @@ import { savedViewHref } from '#features/saved-views/slugs.ts'; import { appKeyboardShortcuts, isKeyboardShortcut } from '#features/shared/keyboard-shortcuts.ts'; import { createProjectStackNotificationRefresher, invalidateStackQueries, type ProjectStackNotificationRefresher } from '#features/stacks/api.svelte.ts'; + import TaskPanel from '#features/tasks/components/task-panel.svelte'; import { invalidateTokenQueries } from '#features/tokens/api.svelte.ts'; import { getMeQuery, invalidateUserQueries } from '#features/users/api.svelte.ts'; import { getGravatarFromCurrentUser } from '#features/users/gravatar.svelte.ts'; @@ -1012,6 +1013,7 @@ + {/if} + getEventsQueryIdentity({ + filter: eventsQueryParameters.filter, + organizationId: organization.current, + savedViewId: savedViewsState.activeSavedView?.id, + time: eventsQueryParameters.time + }), + () => table.store.state.rowSelection + ); + const pageRowCount = $derived(table.getRowModel().rows.length); + const isPageSelected = $derived(pageRowCount > 0 && table.getIsAllRowsSelected()); + + // The same request the list makes, asking only for the total, so the count always agrees with the list. + const matchingTotalQuery = getOrganizationEventsQuery({ + enabled: () => !isSavedViewRoutePending && isPageSelected, + get params() { + return { + filter: eventsQueryParameters.filter, + include: 'total' as const, + limit: 1, + mode: 'summary' as const, + offset: DEFAULT_OFFSET, + time: eventsQueryParameters.time + }; + }, + route: { + get organizationId() { + return organization.current; + } + } + }); + const totalMatching = $derived((matchingTotalQuery.data?.meta.total ?? eventsQuery.data?.meta.total) as number | undefined); + const allMatchingTarget = $derived.by(() => { + const selection = allMatching.active; + if (!selection || !organization.current) { + return undefined; + } + + return { + count: selection.count, + filter: eventsQueryParameters.filter ?? '', + offset: DEFAULT_OFFSET, + organizationId: organization.current, + time: eventsQueryParameters.time + }; + }); + + $effect(() => { + if (!allMatching.active || !isPageSelected) { + untrack(() => allMatching.clear()); + } + }); + async function handleRefresh() { table.resetRowSelection(); await eventsQuery.refetch(); @@ -822,6 +880,17 @@ table.setPageIndex(0); } + async function selectAllMatching() { + // Count again right now, so the number the user confirms is current. The result may only expand the query + // and page selection that started this request, even if the same rows are cleared and selected again. + const selection = allMatching.snapshot; + const result = await matchingTotalQuery.refetch(); + const count = (result.data?.meta.total ?? totalMatching) as number | undefined; + if (count !== undefined) { + allMatching.select(count, selection); + } + } + let reconcileTotalRequested = false; const debouncedRefetch = debounce(1500, async () => { const shouldReconcileTotal = reconcileTotalRequested; @@ -1038,6 +1107,18 @@ /> {/if} + { + allMatching.clear(); + table.resetRowSelection(); + }} + onSelectAllMatching={selectAllMatching} + pageCount={pageRowCount} + {totalMatching} + /> +
{#snippet footerChildren()}
- + allMatching.clear()} {table} />
diff --git a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs index fe059935db..ebf136094a 100644 --- a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs +++ b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs @@ -38,6 +38,7 @@ await Task.WhenAll( _subscriber.SubscribeAsync(OnEntityChangedAsync, shutdownToken), _subscriber.SubscribeAsync(OnPlanChangedAsync, shutdownToken), _subscriber.SubscribeAsync(OnPlanOverageAsync, shutdownToken), + _subscriber.SubscribeAsync(OnBackgroundTaskUpdatedAsync, shutdownToken), _subscriber.SubscribeAsync(OnUserMembershipChangedAsync, shutdownToken), _subscriber.SubscribeAsync(OnReleaseNotificationAsync, shutdownToken), _subscriber.SubscribeAsync(OnSystemNotificationAsync, shutdownToken) @@ -158,6 +159,17 @@ private Task OnPlanOverageAsync(PlanOverage planOverage, CancellationToken cance return Task.CompletedTask; } + internal Task OnBackgroundTaskUpdatedAsync(BackgroundTaskUpdated backgroundTaskUpdated, CancellationToken cancellationToken = default) + { + if (backgroundTaskUpdated is not null) + { + _logger.LogTrace("Sending background task {TaskId} update to organization: {Organization}", backgroundTaskUpdated.Id, backgroundTaskUpdated.OrganizationId); + return GroupSendAsync(backgroundTaskUpdated.OrganizationId, backgroundTaskUpdated); + } + + return Task.CompletedTask; + } + private Task OnPlanChangedAsync(PlanChanged planChanged, CancellationToken cancellationToken = default) { if (planChanged is not null) diff --git a/src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs b/src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs new file mode 100644 index 0000000000..b309fef872 --- /dev/null +++ b/src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs @@ -0,0 +1,64 @@ +using Exceptionless.Core.Attributes; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; + +namespace Exceptionless.Web.Models; + +public record ViewBackgroundTask +{ + [ObjectId] + public string Id { get; set; } = null!; + + [ObjectId] + public string OrganizationId { get; set; } = null!; + + [ObjectId] + public string? ProjectId { get; set; } + + /// What the task does, e.g. events.delete, projects.reset-data or projects.sample-data. + public string Type { get; set; } = null!; + + public string Description { get; set; } = null!; + public BackgroundTaskStatus Status { get; set; } + + /// Percent complete, 0 to 100. + public int Progress { get; set; } + + /// The item counts. All zero for tasks that only report a percent. + public long Total { get; set; } + public long Processed { get; set; } + public long Succeeded { get; set; } + public long Failed { get; set; } + + /// Whether the task can be stopped with the cancel endpoint. + public bool CanCancel { get; set; } + + /// What a bulk action applies to. Only set for bulk actions. + public ViewBulkOperationSelection? Selection { get; set; } + + [ObjectId] + public string CreatedByUserId { get; set; } = null!; + + public DateTime CreatedUtc { get; set; } + public DateTime? StartedUtc { get; set; } + public DateTime? CompletedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } + public string? Message { get; set; } +} + +public record ViewBulkOperationSelection +{ + public BulkOperationSelectionKind Kind { get; set; } + + /// The number of ids when the selection is an id list. + public int? IdCount { get; set; } + + public string? Filter { get; set; } + public string? Time { get; set; } + + /// The absolute start of the resolved time window. + public DateTime? UtcStart { get; set; } + + /// The absolute end of the resolved time window, never later than when the task was created. + public DateTime? UtcEnd { get; set; } +} diff --git a/src/Exceptionless.Web/Models/BulkOperation/NewBulkOperation.cs b/src/Exceptionless.Web/Models/BulkOperation/NewBulkOperation.cs new file mode 100644 index 0000000000..5dc51debbf --- /dev/null +++ b/src/Exceptionless.Web/Models/BulkOperation/NewBulkOperation.cs @@ -0,0 +1,51 @@ +using System.ComponentModel.DataAnnotations; +using Exceptionless.Core.Attributes; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BulkOperations; + +namespace Exceptionless.Web.Models; + +public record NewBulkOperation : IOwnedByOrganization +{ + /// The action to run, e.g. events.delete. + [Required] + [MaxLength(100)] + public string Type { get; set; } = null!; + + [ObjectId] + public string OrganizationId { get; set; } = null!; + + /// Optionally limits the operation to one project of the organization. + [ObjectId] + public string? ProjectId { get; set; } + + [Required] + public NewBulkOperationSelection Selection { get; set; } = null!; + + /// Action specific parameters. Actions without parameters reject any value here. + public Dictionary? Parameters { get; set; } +} + +public record NewBulkOperationSelection +{ + public BulkOperationSelectionKind Kind { get; set; } + + /// The ids to process when is ids. + [MaxLength(1000)] + public List? Ids { get; set; } + + /// The search filter when is filter. An empty filter matches everything in the time window. + [MaxLength(2000)] + public string? Filter { get; set; } + + /// + /// The time range of the filter, e.g. [now-7d TO now]. It is resolved to absolute UTC dates when the + /// operation is created. + /// + [MaxLength(100)] + public string? Time { get; set; } + + /// The UTC offset used to resolve , e.g. -5h. + [MaxLength(20)] + public string? Offset { get; set; } +} diff --git a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json index 850bdf6dea..92a14c65f5 100644 --- a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json +++ b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json @@ -873,6 +873,20 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "POST", + "route": "/api/v2/bulk-operations", + "displayName": "HTTP: POST api/v2/bulk-operations", + "tags": [ + "BackgroundTask" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "POST", "route": "/api/v2/contact", @@ -1683,6 +1697,20 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "GET", + "route": "/api/v2/organizations/{organizationId:objectid}/tasks", + "displayName": "HTTP: GET api/v2/organizations/{organizationId:objectid}/tasks", + "tags": [ + "BackgroundTask" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "GET", "route": "/api/v2/organizations/{organizationId:objectid}/tokens", @@ -2662,6 +2690,34 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "GET", + "route": "/api/v2/tasks/{id:objectid}", + "displayName": "HTTP: GET api/v2/tasks/{id:objectid}", + "tags": [ + "BackgroundTask" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, + { + "method": "POST", + "route": "/api/v2/tasks/{id:objectid}/cancel", + "displayName": "HTTP: POST api/v2/tasks/{id:objectid}/cancel", + "tags": [ + "BackgroundTask" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "POST", "route": "/api/v2/tokens", diff --git a/tests/Exceptionless.Tests/Api/Data/openapi.json b/tests/Exceptionless.Tests/Api/Data/openapi.json index 0efc9792e8..e8d872b136 100644 --- a/tests/Exceptionless.Tests/Api/Data/openapi.json +++ b/tests/Exceptionless.Tests/Api/Data/openapi.json @@ -4067,6 +4067,236 @@ } } }, + "/api/v2/bulk-operations": { + "post": { + "tags": [ + "BackgroundTask" + ], + "summary": "Create", + "description": "Starts an action, such as \u0060events.delete\u0060, on everything that matches a selection. The selection is either a list of ids\nor a filter with a time range. The time range is resolved to absolute UTC dates when the operation is created, and events\nstored after that moment are never included. The operation runs in the background and is returned as a task; poll\n\u0060GET api/v2/tasks/{id}\u0060 or listen for \u0060BackgroundTaskUpdated\u0060 messages to follow its progress.", + "requestBody": { + "description": "The bulk operation to start.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewBulkOperation" + } + }, + "application/*\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/NewBulkOperation" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "Accepted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ViewBackgroundTask" + } + } + } + }, + "403": { + "description": "The action is not permitted for this organization or project.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "404": { + "description": "The organization or project could not be found.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "A destructive operation is already running for this project.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "422": { + "description": "The request is not valid.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "426": { + "description": "The filter uses a feature that requires a plan upgrade.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/api/v2/tasks/{id}": { + "get": { + "tags": [ + "BackgroundTask" + ], + "summary": "Get by id", + "description": "Returns the progress of a task, such as a bulk action, a project data reset or sample data generation. Tasks are kept for seven days.", + "operationId": "GetBackgroundTaskById", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "The identifier of the task.", + "required": true, + "schema": { + "pattern": "^[a-zA-Z\\d]{24,36}$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ViewBackgroundTask" + } + } + } + }, + "404": { + "description": "The task could not be found.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/api/v2/organizations/{organizationId}/tasks": { + "get": { + "tags": [ + "BackgroundTask" + ], + "summary": "Get by organization", + "description": "Returns the active and recently finished tasks of the organization, newest first. Finished tasks are kept for seven days.", + "parameters": [ + { + "name": "organizationId", + "in": "path", + "description": "The identifier of the organization.", + "required": true, + "schema": { + "pattern": "^[a-zA-Z\\d]{24,36}$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ViewBackgroundTask" + } + } + } + } + }, + "404": { + "description": "The organization could not be found.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/api/v2/tasks/{id}/cancel": { + "post": { + "tags": [ + "BackgroundTask" + ], + "summary": "Cancel", + "description": "Requests cancellation. A running bulk action stops after the batch it is processing; work already done is not undone. Cancelling a finished task has no effect. Tasks that cannot be stopped, such as a project data reset, are rejected.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "The identifier of the task.", + "required": true, + "schema": { + "pattern": "^[a-zA-Z\\d]{24,36}$", + "type": "string" + } + } + ], + "responses": { + "202": { + "description": "Accepted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ViewBackgroundTask" + } + } + } + }, + "404": { + "description": "The task could not be found.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "422": { + "description": "The task cannot be cancelled.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, "/api/v2/users/me": { "get": { "tags": [ @@ -12151,6 +12381,22 @@ } } }, + "BackgroundTaskStatus": { + "enum": [ + "Queued", + "Running", + "Completed", + "Failed", + "Cancelled" + ], + "x-enumNames": [ + "Queued", + "Running", + "Completed", + "Failed", + "Cancelled" + ] + }, "BillingPlan": { "required": [ "id", @@ -12220,6 +12466,16 @@ "Unpaid" ] }, + "BulkOperationSelectionKind": { + "enum": [ + "Ids", + "Filter" + ], + "x-enumNames": [ + "Ids", + "Filter" + ] + }, "ChangePasswordModel": { "required": [ "current_password", @@ -12588,6 +12844,96 @@ } } }, + "NewBulkOperation": { + "required": [ + "type", + "selection", + "organization_id" + ], + "type": "object", + "properties": { + "type": { + "maxLength": 100, + "type": "string", + "description": "The action to run, e.g. events.delete." + }, + "organization_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": "string" + }, + "project_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": [ + "null", + "string" + ], + "description": "Optionally limits the operation to one project of the organization." + }, + "selection": { + "$ref": "#/components/schemas/NewBulkOperationSelection" + }, + "parameters": { + "type": [ + "null", + "object" + ], + "additionalProperties": { + "type": "string" + }, + "description": "Action specific parameters. Actions without parameters reject any value here." + } + } + }, + "NewBulkOperationSelection": { + "required": [ + "kind" + ], + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/BulkOperationSelectionKind" + }, + "ids": { + "maxItems": 1000, + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + }, + "description": "The ids to process when Kind is ids." + }, + "filter": { + "maxLength": 2000, + "type": [ + "null", + "string" + ], + "description": "The search filter when Kind is filter. An empty filter matches everything in the time window." + }, + "time": { + "maxLength": 100, + "type": [ + "null", + "string" + ], + "description": "The time range of the filter, e.g. [now-7d TO now]. It is resolved to absolute UTC dates when the\noperation is created." + }, + "offset": { + "maxLength": 20, + "type": [ + "null", + "string" + ], + "description": "The UTC offset used to resolve Time, e.g. -5h." + } + } + }, "NewOrganization": { "required": [ "name" @@ -14497,6 +14843,177 @@ }, "description": "The account whose email address the link verified, which may not be the account that\u0027s signed in." }, + "ViewBackgroundTask": { + "required": [ + "id", + "organization_id", + "type", + "description", + "status", + "progress", + "total", + "processed", + "succeeded", + "failed", + "can_cancel", + "created_by_user_id", + "created_utc", + "updated_utc" + ], + "type": "object", + "properties": { + "id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": "string" + }, + "organization_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": "string" + }, + "project_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": [ + "null", + "string" + ] + }, + "type": { + "type": "string", + "description": "What the task does, e.g. events.delete, projects.reset-data or projects.sample-data." + }, + "description": { + "type": "string" + }, + "status": { + "$ref": "#/components/schemas/BackgroundTaskStatus" + }, + "progress": { + "type": "integer", + "description": "Percent complete, 0 to 100.", + "format": "int32" + }, + "total": { + "type": "integer", + "description": "The item counts. All zero for tasks that only report a percent.", + "format": "int64" + }, + "processed": { + "type": "integer", + "format": "int64" + }, + "succeeded": { + "type": "integer", + "format": "int64" + }, + "failed": { + "type": "integer", + "format": "int64" + }, + "can_cancel": { + "type": "boolean", + "description": "Whether the task can be stopped with the cancel endpoint." + }, + "selection": { + "oneOf": [ + { + "type": "null" + }, + { + "description": "What a bulk action applies to. Only set for bulk actions.", + "$ref": "#/components/schemas/ViewBulkOperationSelection" + } + ] + }, + "created_by_user_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": "string" + }, + "created_utc": { + "type": "string", + "format": "date-time" + }, + "started_utc": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "completed_utc": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated_utc": { + "type": "string", + "format": "date-time" + }, + "message": { + "type": [ + "null", + "string" + ] + } + } + }, + "ViewBulkOperationSelection": { + "required": [ + "kind" + ], + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/BulkOperationSelectionKind" + }, + "id_count": { + "type": [ + "null", + "integer" + ], + "description": "The number of ids when the selection is an id list.", + "format": "int32" + }, + "filter": { + "type": [ + "null", + "string" + ] + }, + "time": { + "type": [ + "null", + "string" + ] + }, + "utc_start": { + "type": [ + "null", + "string" + ], + "description": "The absolute start of the resolved time window.", + "format": "date-time" + }, + "utc_end": { + "type": [ + "null", + "string" + ], + "description": "The absolute end of the resolved time window, never later than when the task was created.", + "format": "date-time" + } + }, + "description": "What a bulk action applies to. Only set for bulk actions." + }, "ViewCurrentUser": { "required": [ "has_local_account", @@ -15368,6 +15885,9 @@ { "name": "Auth" }, + { + "name": "BackgroundTask" + }, { "name": "Event" }, diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs new file mode 100644 index 0000000000..f51edceb9c --- /dev/null +++ b/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs @@ -0,0 +1,351 @@ +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Hubs; +using Exceptionless.Tests.Utility; +using Exceptionless.Web.Api.Results; +using Exceptionless.Web.Hubs; +using Exceptionless.Web.Models; +using Foundatio.Jobs; +using Foundatio.Repositories; +using Foundatio.Repositories.Utility; +using Xunit; + +namespace Exceptionless.Tests.Api.Endpoints; + +public class BackgroundTaskEndpointTests : IntegrationTestsBase +{ + private readonly BackgroundTaskService _taskService; + private readonly WorkItemJob _workItemJob; + + public BackgroundTaskEndpointTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _taskService = GetService(); + _workItemJob = GetService(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + await GetService().CreateDataAsync(); + } + + [Fact] + public async Task Get_Task_ReturnsItsProgressAndOnlyBulkTasksHaveASelection() + { + // Arrange + var plain = await AddTaskAsync(canCancel: false, progress: 40); + var bulk = await AddTaskAsync(canCancel: true, bulk: true); + + // Act + var plainView = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", plain.Id).StatusCodeShouldBeOk()); + var bulkView = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", bulk.Id).StatusCodeShouldBeOk()); + + // Assert + Assert.Equal(plain.Id, plainView!.Id); + Assert.Equal(BackgroundTaskTypes.ResetProjectData, plainView.Type); + Assert.Equal(BackgroundTaskStatus.Queued, plainView.Status); + Assert.Equal(40, plainView.Progress); + Assert.False(plainView.CanCancel); + Assert.Null(plainView.Selection); + Assert.True(bulkView!.CanCancel); + Assert.Equal(BulkOperationSelectionKind.Ids, bulkView.Selection!.Kind); + Assert.Equal(2, bulkView.Selection.IdCount); + } + + [Fact] + public async Task GetByOrganization_MixedTaskTypes_AreListedNewestFirst() + { + // Arrange + var first = await AddTaskAsync(canCancel: false); + TimeProvider.Advance(TimeSpan.FromSeconds(5)); + var second = await AddTaskAsync(canCancel: true, bulk: true); + await AddTaskAsync(canCancel: false, organizationId: SampleDataService.FREE_ORG_ID); + + // Act + var list = await SendRequestAsAsync>(r => r.AsTestOrganizationUser() + .AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "tasks").StatusCodeShouldBeOk()); + + // Assert + Assert.Equal([second.Id, first.Id], list!.Select(t => t.Id).ToArray()); + } + + [Fact] + public async Task Get_TaskOfAnotherOrganization_IsNotFoundForGetListAndCancelButVisibleToGlobalAdmins() + { + // Arrange + var task = await AddTaskAsync(canCancel: true); + + // Act + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("tasks", task.Id).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "tasks").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", ObjectId.GenerateNewId().ToString()).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", ObjectId.GenerateNewId().ToString(), "cancel").StatusCodeShouldBeNotFound()); + var asAdmin = await SendRequestAsAsync(r => r.AsGlobalAdminUser().AppendPaths("tasks", task.Id).StatusCodeShouldBeOk()); + + // Assert + Assert.Equal(task.Id, asAdmin!.Id); + Assert.Equal(BackgroundTaskStatus.Queued, (await _taskService.GetAsync(task.Id))!.Status); + } + + [Fact] + public async Task Tasks_WithoutAUserSession_AreRejected() + { + // Arrange + var task = await AddTaskAsync(canCancel: true); + + // Act + await SendRequestAsync(r => r.AppendPaths("tasks", task.Id).StatusCodeShouldBeUnauthorized()); + await SendRequestAsync(r => r.AsTestOrganizationClientUser().AppendPaths("tasks", task.Id).StatusCodeShouldBeForbidden()); + await SendRequestAsync(r => r.Post().AsTestOrganizationClientUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeForbidden()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Queued, (await _taskService.GetAsync(task.Id))!.Status); + } + + [Fact] + public async Task Cancel_QueuedCancellableTask_IsAcceptedAndCancelledImmediately() + { + // Arrange + var task = await AddTaskAsync(canCancel: true); + + // Act + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeAccepted()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, response!.Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, (await _taskService.GetAsync(task.Id))!.Status); + } + + [Fact] + public async Task Cancel_ActiveTaskThatCannotBeCancelled_IsUnprocessableAndKeepsRunning() + { + // Arrange + var task = await AddTaskAsync(canCancel: false); + await _taskService.UpdateAsync(task.Id, current => + { + current.Start(TimeProvider.GetUtcNow().UtcDateTime); + return true; + }); + + // Act + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeUnprocessableEntity()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Running, (await _taskService.GetAsync(task.Id))!.Status); + Assert.False(await _taskService.IsCancelRequestedAsync(task.Id)); + } + + [Fact] + public async Task Cancel_FinishedTaskThatCannotBeCancelled_IsAcceptedAndChangesNothing() + { + // Arrange + var task = await AddTaskAsync(canCancel: false); + await _taskService.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", TimeProvider.GetUtcNow().UtcDateTime); + return true; + }); + + // Act + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeAccepted()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, response!.Status); + } + + [Fact] + public async Task ResetData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => d.Event().TestProject().Date(now.AddHours(-1))); + + // Act + var accepted = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "reset-data").StatusCodeShouldBeAccepted()); + + // Assert + string taskId = Assert.Single(accepted!.Workers); + + // Act + var queued = await GetTaskAsync(taskId); + + // Assert + Assert.Equal(BackgroundTaskTypes.ResetProjectData, queued.Type); + Assert.Equal(BackgroundTaskStatus.Queued, queued.Status); + Assert.False(queued.CanCancel); + Assert.Equal(SampleDataService.TEST_PROJECT_ID, queued.ProjectId); + Assert.StartsWith("Reset data for ", queued.Description); + + // Act + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var completed = await GetTaskAsync(taskId); + await RefreshDataAsync(); + var remaining = await GetService().GetByIdsAsync(events.Select(e => e.Id).ToArray(), o => o.Cache(false)); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(100, completed.Progress); + Assert.NotNull(completed.StartedUtc); + Assert.NotNull(completed.CompletedUtc); + Assert.Empty(remaining); + } + + [Fact] + public async Task ResetData_CancelledBeforeDelayedDelivery_PreservesTheProjectEvents() + { + // Arrange + var (_, events) = await CreateDataAsync(d => d.Event().TestProject().Date(TimeProvider.GetUtcNow().AddHours(-1))); + var accepted = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "reset-data").StatusCodeShouldBeAccepted()); + string taskId = accepted!.Workers.Single(); + TimeProvider.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromSeconds(1)); + + // Act + var cancelled = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() + .AppendPaths("tasks", taskId, "cancel").StatusCodeShouldBeAccepted()); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var stored = await GetTaskAsync(taskId); + await RefreshDataAsync(); + var remaining = await GetService().GetByIdsAsync(events.Select(e => e.Id).ToArray(), o => o.Cache(false)); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, cancelled!.Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, stored.Status); + Assert.Null(stored.StartedUtc); + Assert.Equal(events.Select(e => e.Id), remaining.Select(e => e.Id)); + } + + [Fact] + public async Task SampleData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() + { + // Arrange + string projectId = SampleDataService.TEST_PROJECT_ID; + + // Act + var accepted = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() + .AppendPaths("projects", projectId, "sample-data").StatusCodeShouldBeAccepted()); + + // Assert + string taskId = Assert.Single(accepted!.Workers); + + // Act + var queued = await GetTaskAsync(taskId); + + // Assert + Assert.Equal(BackgroundTaskTypes.GenerateProjectSampleData, queued.Type); + Assert.Equal(BackgroundTaskStatus.Queued, queued.Status); + Assert.False(queued.CanCancel); + Assert.StartsWith("Generate sample data for ", queued.Description); + + // Act + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var completed = await GetTaskAsync(taskId); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(100, completed.Progress); + } + + [Fact] + public async Task ResetAndSampleData_ProjectOfAnotherOrganization_IsNotFoundAndTracksNothing() + { + // Arrange + string projectId = SampleDataService.TEST_PROJECT_ID; + + // Act + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("projects", projectId, "reset-data").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("projects", projectId, "sample-data").StatusCodeShouldBeNotFound()); + + // Assert + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); + } + + [Fact] + public async Task OnBackgroundTaskUpdatedAsync_SendsOnlyToTheTasksOrganization() + { + // Arrange + var broker = GetService(); + var connectionMapping = GetService(); + var connectionManager = GetService(); + var ownSocket = new TestWebSocket(); + var otherSocket = new TestWebSocket(); + string ownConnection = connectionManager.AddWebSocket(ownSocket); + string otherConnection = connectionManager.AddWebSocket(otherSocket); + await connectionMapping.GroupAddAsync(SampleDataService.TEST_ORG_ID, ownConnection); + await connectionMapping.GroupAddAsync(SampleDataService.FREE_ORG_ID, otherConnection); + + try + { + // Act + await broker.OnBackgroundTaskUpdatedAsync(new BackgroundTaskUpdated + { + Id = "537650f3b77efe23a47914f9", + OrganizationId = SampleDataService.TEST_ORG_ID, + Type = BackgroundTaskTypes.ResetProjectData, + Description = "Reset data for Test", + Status = BackgroundTaskStatus.Running, + Progress = 40, + CanCancel = false + }, TestCancellationToken); + + await WaitForAsync(() => ownSocket.SentMessages.Count > 0); + + // Assert + string sent = Assert.Single(ownSocket.SentMessages); + Assert.Contains("\"type\":\"BackgroundTaskUpdated\"", sent); + Assert.Contains("\"progress\":40", sent); + Assert.Contains("\"can_cancel\":false", sent); + Assert.Empty(otherSocket.SentMessages); + } + finally + { + await connectionMapping.GroupRemoveAsync(SampleDataService.TEST_ORG_ID, ownConnection); + await connectionMapping.GroupRemoveAsync(SampleDataService.FREE_ORG_ID, otherConnection); + await connectionManager.RemoveWebSocketAsync(ownConnection); + await connectionManager.RemoveWebSocketAsync(otherConnection); + } + } + + private async Task AddTaskAsync(bool canCancel, int progress = 0, bool bulk = false, string organizationId = SampleDataService.TEST_ORG_ID) + { + var task = new BackgroundTask + { + OrganizationId = organizationId, + ProjectId = organizationId == SampleDataService.TEST_ORG_ID ? SampleDataService.TEST_PROJECT_ID : SampleDataService.FREE_PROJECT_ID, + Type = bulk ? "events.delete" : BackgroundTaskTypes.ResetProjectData, + Description = bulk ? "Delete 2 selected events" : "Reset data for Test", + Progress = progress, + CanCancel = canCancel, + CreatedByUserId = "537650f3b77efe23a47914f8", + Bulk = bulk ? new BulkOperationDetails + { + Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString()] } + } : null + }; + + await _taskService.AddAsync(task); + return task; + } + + private async Task GetTaskAsync(string id) + { + var task = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", id).StatusCodeShouldBeOk()); + Assert.NotNull(task); + return task; + } + + private static async Task WaitForAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(10); + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(25); + } +} diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs new file mode 100644 index 0000000000..93ea1a0782 --- /dev/null +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -0,0 +1,493 @@ +using System.Collections.Concurrent; +using Exceptionless.Core; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Base; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Web.Models; +using Foundatio.Jobs; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; +using Foundatio.Repositories.Utility; +using Foundatio.Serializer; +using Foundatio.Caching; +using Foundatio.Lock; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Api.Endpoints; + +public partial class BulkOperationEndpointTests +{ + [Fact] + public async Task GetPageAsync_Paging_DoesNotCountTotalHits() + { + // Arrange + ICommandOptions? capturedOptions = null; + var repository = new Mock(); + repository.Setup(r => r.FindAsync(It.IsAny(), It.IsAny())) + .Callback((_, options) => capturedOptions = options) + .ReturnsAsync(new FindResults(data: new Dictionary { [ElasticDataKeys.PointInTimeId] = "snapshot" })); + var resolver = CreateResolver(repository.Object); + + // Act + await resolver.GetPageAsync(NewRecordingOperation("test.record"), "{\"version\":1,\"point_in_time_id\":\"snapshot\",\"total\":1}", null, 10, TestCancellationToken); + + // Assert + Assert.NotNull(capturedOptions); + Assert.False(capturedOptions.ShouldTrackTotalHits()); + } + + [Fact] + public async Task Create_FilterMatchingTooManyStacks_IsRejectedWithTheRealMessage() + { + // Arrange + const string message = "Event filter resulted in too many matching stacks (30000 of 20000 max). Please limit your search criteria."; + var repository = new Mock(); + repository.Setup(r => r.FindAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new DocumentLimitExceededException(message)); + var service = CreateService([GetService()], [CreateResolver(repository.Object)]); + + // Act + var result = await service.CreateAsync(NewRecordingOperation("events.delete")); + + // Assert + Assert.Null(result.Task); + Assert.NotNull(result.Rejection); + Assert.Equal(BulkOperationCheckStatus.Invalid, result.Rejection.Status); + Assert.Equal("selection.filter", result.Rejection.Field); + Assert.Equal(message, result.Rejection.Message); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + } + + [Fact] + public async Task Execute_StackStatusChangesAfterCreation_StillDeletesTheEventsThatWereConfirmed() + { + // Arrange + var (stacks, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("open 1").Status(StackStatus.Open); + d.Event().TestProject().Message("open 2").Status(StackStatus.Open); + d.Event().TestProject().Message("fixed").Status(StackStatus.Fixed); + }); + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "status:open")); + var openStacks = stacks.Where(s => s.Status == StackStatus.Open).ToList(); + + // Act + foreach (var stack in openStacks) + stack.Status = StackStatus.Fixed; + + await GetService().SaveAsync(openStacks, o => o.ImmediateConsistency()); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + string? pinned = await _bulkOperationService.GetPinnedSelectionAsync(created.Id); + Assert.Equal(2, created.Total); + Assert.NotNull(pinned); + Assert.All(openStacks, stack => Assert.Contains(stack.Id, pinned)); + Assert.Equal(["fixed"], (await GetRemainingMessagesAsync(events)).ToArray()); + } + + [Fact] + public async Task Execute_PremiumFeaturesRemovedAfterCreation_FailsWithoutDeletingOrLosingTheSelectionCount() + { + // Arrange + var (_, events) = await CreateDataAsync(data => data.Event().TestProject().Message("selected")); + var organizations = GetService(); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "message:selected")); + + // Assert + Assert.Equal(1, created.Total); + + // Arrange + var organization = await organizations.GetByIdAsync(SampleDataService.TEST_ORG_ID); + organization!.HasPremiumFeatures = false; + await organizations.SaveAsync(organization, options => options.ImmediateConsistency()); + + // Act + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var finished = await GetOperationAsync(created.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Failed, finished.Status); + Assert.Equal(1, finished.Total); + Assert.Equal(0, finished.Processed); + Assert.Equal(0, finished.Succeeded); + Assert.Equal("Please upgrade your plan to use premium search features.", finished.Message); + Assert.Equal(["selected"], (await GetRemainingMessagesAsync(events)).ToArray()); + } + + [Fact] + public async Task Create_MoreThanFiveHundredOperations_StillSeesTheNewestActiveOneForTheConflictCheck() + { + // Arrange + var service = CreateService(new RecordingHandler(), new RecordingHandler("test.destructive", true)); + for (int index = 0; index < 500; index++) + Assert.NotNull((await service.CreateAsync(NewIdsOperation("test.record"))).Task); + + // Act + var newest = await service.CreateAsync(NewIdsOperation("test.destructive")); + var blocked = await service.CreateAsync(NewIdsOperation("test.destructive")); + var list = await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID); + + // Assert + Assert.NotNull(newest.Task); + Assert.Null(blocked.Task); + Assert.Equal(newest.Task.Id, blocked.Conflict?.Id); + Assert.Equal(501, list.Count); + Assert.Equal(newest.Task.Id, list[0].Id); + } + + [Fact] + public async Task Execute_RecordMissingWhenTheJobRuns_PublishesAFinalFailedUpdate() + { + // Arrange + var messages = new ConcurrentQueue(); + await GetService().SubscribeAsync(m => messages.Enqueue(m), TestCancellationToken); + var workItem = new BulkOperationWorkItem + { + Description = "Delete 7 events matching filter", + TaskId = ObjectId.GenerateNewId().ToString(), + OrganizationId = SampleDataService.TEST_ORG_ID, + Type = "events.delete" + }; + + // Act + await RunWorkItemAsync(_bulkOperationService, workItem, 500, TestCancellationToken); + await WaitForAsync(() => !messages.IsEmpty); + + // Assert + var update = Assert.Single(messages); + Assert.Equal(workItem.TaskId, update.Id); + Assert.Equal(SampleDataService.TEST_ORG_ID, update.OrganizationId); + Assert.Equal("Delete 7 events matching filter", update.Description); + Assert.Equal(BackgroundTaskStatus.Failed, update.Status); + Assert.False(String.IsNullOrEmpty(update.Message)); + } + + [Fact] + public async Task Execute_TemporaryErrorMidOperation_IsRetriedFromTheSavedCursor() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var recorder = new RecordingHandler { Failure = call => call == 2 ? new TimeoutException("Elasticsearch timed out") : null }; + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + + long enqueuedBefore = await GetEnqueuedWorkItemCountAsync(); + + // Act + var thrown = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + long continuations = await GetEnqueuedWorkItemCountAsync() - enqueuedBefore; + var interrupted = await _taskService.GetAsync(created.Id); + await RunOperationAsync(service, created.Id, 2, TestCancellationToken); + var finished = await _taskService.GetAsync(created.Id); + + // Assert + Assert.Null(thrown); + Assert.Equal(1, continuations); + Assert.NotNull(interrupted); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); + Assert.Equal(2, interrupted.Processed); + Assert.Equal(1, interrupted.Bulk!.RetryCount); + Assert.NotNull(finished); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(0, finished.Bulk!.RetryCount); + Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); + } + + [Fact] + public async Task Execute_TemporaryErrorsKeepHappening_FailsAfterTheRetriesAreUsedUp() + { + // Arrange + await CreateDataAsync(d => d.Event().TestProject()); + var recorder = new RecordingHandler { Failure = _ => new ApplicationException("Elasticsearch unavailable") }; + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + + // Act + var first = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + var second = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + var third = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + var finished = await _taskService.GetAsync(created.Id); + + // Assert + Assert.Null(first); + Assert.Null(second); + Assert.Null(third); + Assert.NotNull(finished); + Assert.Equal(BackgroundTaskStatus.Failed, finished.Status); + Assert.Equal(3, finished.Bulk!.RetryCount); + } + + [Fact] + public async Task Execute_TemporaryErrorsBetweenProgress_ContinueInNewRunsUntilDone() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 6; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + // A temporary error after every completed batch: more errors in total than the queue's retries for one work item. + var recorder = new RecordingHandler { Failure = call => call % 2 == 0 ? new TimeoutException("Elasticsearch timed out") : null }; + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + long enqueuedBefore = await GetEnqueuedWorkItemCountAsync(); + var thrown = new List(); + + // Act + for (int run = 0; run < 10 && (await _taskService.GetAsync(created.Id))!.IsActive; run++) + thrown.Add(await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 1, TestCancellationToken))); + + long continuations = await GetEnqueuedWorkItemCountAsync() - enqueuedBefore; + var finished = await _taskService.GetAsync(created.Id); + + // Assert + Assert.All(thrown, Assert.Null); + Assert.Equal(thrown.Count - 1, continuations); + Assert.NotNull(finished); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); + } + + [Fact] + public async Task Execute_RestartAfterTheLastIdBatchBeforeCompletion_DoesNotStartOver() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var recorder = new RecordingHandler(); + var service = CreateService(recorder); + var operation = NewRecordingOperation("test.record"); + operation.Bulk!.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = events.Select(e => e.Id).ToList() }; + var result = await service.CreateAsync(operation); + var created = result.Task; + Assert.NotNull(created); + await RunOperationAsync(service, created.Id, 2, TestCancellationToken); + int batchesBefore = recorder.Batches.Count; + // The worker stopped after saving the last batch but before saving the completed status. + await _taskService.UpdateAsync(created.Id, current => + { + current.Status = BackgroundTaskStatus.Running; + current.CompletedUtc = null; + return true; + }); + + // Act + await RunOperationAsync(service, created.Id, 2, TestCancellationToken); + var finished = await _taskService.GetAsync(created.Id); + + // Assert + Assert.Equal(batchesBefore, recorder.Batches.Count); + Assert.NotNull(finished); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(5, finished.Processed); + } + + [Fact] + public async Task Create_WhenTheQueueRejectsTheWorkItem_FailsTheOperationSoItDoesNotBlockTheProject() + { + // Arrange + await CreateDataAsync(d => d.Event().TestProject()); + var unavailableQueue = new Mock>(); + unavailableQueue.SetupGet(q => q.Serializer).Returns(GetService>().Serializer); + unavailableQueue.Setup(q => q.EnqueueAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new TimeoutException("Queue unavailable")); + var failingTasks = new BackgroundTaskService(GetService(), GetService(), GetService(), unavailableQueue.Object, + GetService(), new BackgroundTaskOptions(), TimeProvider, GetService>()); + var recorder = new RecordingHandler("test.destroy", isDestructive: true); + var failingService = CreateService([recorder], [GetService()], failingTasks); + var service = CreateService([recorder], [GetService()]); + + // Act + var thrown = await Record.ExceptionAsync(async () => await failingService.CreateAsync(NewRecordingOperation("test.destroy"))); + var orphan = (await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)).Single(); + var retry = await service.CreateAsync(NewRecordingOperation("test.destroy")); + + // Assert + Assert.IsType(thrown); + Assert.Equal(BackgroundTaskStatus.Failed, orphan.Status); + Assert.NotNull(retry.Task); + Assert.Equal(BackgroundTaskStatus.Queued, retry.Task.Status); + } + + [Fact] + public void Handler_RenewsTheQueueEntryAndLockOnEveryBatch() + { + // Arrange + var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(CreateService(new RecordingHandler()), _taskService, [GetService()], + new BulkOperationOptions(), GetService(), TimeProvider, Log); + + // Act + bool renews = handler.AutoRenewLockOnProgress; + + // Assert + Assert.True(renews); + } + + private async Task GetEnqueuedWorkItemCountAsync() + { + return (await GetService>().GetQueueStatsAsync()).Enqueued; + } + + [Fact] + public async Task Execute_ErrorThatRetryingCannotFix_FailsImmediately() + { + // Arrange + await CreateDataAsync(d => d.Event().TestProject()); + var recorder = new RecordingHandler { Failure = _ => new InvalidOperationException("Bad request") }; + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + + // Act + var thrown = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + var finished = await _taskService.GetAsync(created.Id); + + // Assert + Assert.Null(thrown); + Assert.NotNull(finished); + Assert.Equal(BackgroundTaskStatus.Failed, finished.Status); + Assert.Equal(0, finished.Bulk!.RetryCount); + } + + [Fact] + public async Task Execute_JobRestartsMidDelete_ResumesAndDeletesEachEventExactlyOnce() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Message($"event {i}").Date(now.AddMinutes(-i - 1)); + }); + var deleteHandler = new InterceptingHandler(GetService()); + var service = CreateService(deleteHandler); + var result = await service.CreateAsync(NewRecordingOperation("events.delete")); + Assert.NotNull(result.Task); + using var shutdown = new CancellationTokenSource(); + deleteHandler.AfterBatch = () => shutdown.Cancel(); + + // Act + await RunOperationAsync(service, result.Task.Id, 2, shutdown.Token); + var interrupted = await _taskService.GetAsync(result.Task.Id); + var remainingWhileInterrupted = await GetRemainingMessagesAsync(events); + deleteHandler.AfterBatch = null; + await RunOperationAsync(service, result.Task.Id, 2, TestCancellationToken); + var finished = await _taskService.GetAsync(result.Task.Id); + + // Assert + Assert.NotNull(interrupted); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); + Assert.Equal(2, interrupted.Processed); + Assert.Equal(3, remainingWhileInterrupted.Count); + Assert.NotNull(finished); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(5, finished.Processed); + Assert.Equal(5, finished.Succeeded); + Assert.Equal(0, finished.Failed); + Assert.Empty(await GetRemainingMessagesAsync(events)); + } + + [Fact] + public async Task Cancel_RunningOperationThatStoppedReportingProgress_CancelsImmediately() + { + // Arrange + var service = CreateService(new RecordingHandler()); + var created = await CreateRecordingOperationAsync(service); + await _taskService.UpdateAsync(created.Id, operation => + { + operation.Status = BackgroundTaskStatus.Running; + return true; + }); + TimeProvider.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var cancelled = (await _taskService.CancelAsync(created.Id)).Task; + + // Assert + Assert.NotNull(cancelled); + Assert.Equal(BackgroundTaskStatus.Cancelled, cancelled.Status); + Assert.NotNull(cancelled.CompletedUtc); + } + + [Fact] + public async Task Cancel_FinishedOperation_ChangesNothing() + { + // Arrange + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Act + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", created.Id, "cancel").StatusCodeShouldBeAccepted()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, response?.Status); + Assert.Equal(BackgroundTaskStatus.Completed, (await GetOperationAsync(created.Id)).Status); + Assert.False(await _taskService.IsCancelRequestedAsync(created.Id)); + } + + private EventBulkSelectionResolver CreateResolver(IEventRepository eventRepository) + { + var organizations = GetService(); + var projects = GetService(); + var validator = GetService(); + var serializer = GetService(); + var configuration = GetService(); + var appOptions = GetService(); + var options = GetService(); + return new EventBulkSelectionResolver(eventRepository, organizations, projects, validator, serializer, configuration, appOptions, options); + } + + private BackgroundTask NewIdsOperation(string type) + { + var operation = NewRecordingOperation(type); + operation.Bulk!.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [SampleDataService.TEST_ORG_ID] }; + return operation; + } + + /// Runs the real handler, with a hook after each batch to simulate the host stopping. + private sealed class InterceptingHandler(IBulkOperationHandler inner) : IBulkOperationHandler + { + public Action? AfterBatch { get; set; } + + public string Type => inner.Type; + public BulkOperationEntity Entity => inner.Entity; + public bool IsDestructive => inner.IsDestructive; + + public Task AuthorizeAsync(BulkOperationRequest request) => inner.AuthorizeAsync(request); + public Task ValidateAsync(BulkOperationRequest request) => inner.ValidateAsync(request); + public string Describe(BackgroundTask operation) => inner.Describe(operation); + + public async Task ExecuteBatchAsync(BackgroundTask operation, IReadOnlyCollection ids, CancellationToken cancellationToken) + { + var result = await inner.ExecuteBatchAsync(operation, ids, cancellationToken); + AfterBatch?.Invoke(); + return result; + } + } +} diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs new file mode 100644 index 0000000000..b24c5b9a0d --- /dev/null +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -0,0 +1,759 @@ +using System.Net; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Hubs; +using Exceptionless.Tests.Utility; +using Exceptionless.Web.Hubs; +using Exceptionless.Web.Models; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Repositories; +using Foundatio.Repositories.Utility; +using Foundatio.Utility; +using Microsoft.AspNetCore.Mvc; +using Xunit; + +namespace Exceptionless.Tests.Api.Endpoints; + +public partial class BulkOperationEndpointTests : IntegrationTestsBase +{ + private readonly IEventRepository _eventRepository; + private readonly BulkOperationService _bulkOperationService; + private readonly BackgroundTaskService _taskService; + private readonly WorkItemJob _workItemJob; + + public BulkOperationEndpointTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _eventRepository = GetService(); + _bulkOperationService = GetService(); + _taskService = GetService(); + _workItemJob = GetService(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + await GetService().CreateDataAsync(); + } + + [Fact] + public async Task Create_FilterSelection_DeletesEverythingMatchingThroughTheQueuedJob() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Type(Event.KnownTypes.Log).Date(now.AddHours(-1)); + d.Event().TestProject().Type(Event.KnownTypes.Log).Date(now.AddDays(-2)); + d.Event().TestProject().Project(SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID).Type(Event.KnownTypes.Log).Date(now.AddHours(-2)); + }); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "type:log")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var finished = await GetOperationAsync(created.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Queued, created.Status); + Assert.Equal("events.delete", created.Type); + Assert.Equal(3, created.Total); + Assert.Equal("Delete 3 events matching filter", created.Description); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(3, finished.Processed); + Assert.Equal(3, finished.Succeeded); + Assert.Equal(0, finished.Failed); + Assert.Empty(await GetRemainingMessagesAsync(events)); + } + + [Fact] + public async Task Create_FilterWithRelativeDates_PinsThemAndDeletesOnlyTheMatches() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("inside").Date(now.AddHours(-1)); + d.Event().TestProject().Message("outside").Date(now.AddHours(-3)); + }); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "date:[now-2h TO now]")); + string? pinned = await _bulkOperationService.GetPinnedSelectionAsync(created.Id); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var finished = await GetOperationAsync(created.Id); + + // Assert + Assert.NotNull(pinned); + Assert.DoesNotContain("now", pinned, StringComparison.OrdinalIgnoreCase); + Assert.Contains("||-2h", pinned); + Assert.Equal(1, created.Total); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(1, finished.Succeeded); + Assert.Equal(["outside"], (await GetRemainingMessagesAsync(events)).ToArray()); + } + + [Fact] + public async Task Create_FilterSelection_OnlyDeletesMatchesInsideScopeFilterAndWindow() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Type(Event.KnownTypes.Log).Message("match").Date(now.AddHours(-1)); + d.Event().TestProject().Type(Event.KnownTypes.Error).Message("other type").Date(now.AddHours(-1)); + d.Event().TestProject().Type(Event.KnownTypes.Log).Message("outside window").Date(now.AddDays(-2)); + d.Event().TestProject().Project(SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID).Type(Event.KnownTypes.Log).Message("other project").Date(now.AddHours(-1)); + d.Event().FreeProject().Type(Event.KnownTypes.Log).Message("other organization").Date(now.AddHours(-1)); + }); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "type:log", "[now-1d TO now]")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + Assert.Equal(1, created.Total); + var remaining = await GetRemainingMessagesAsync(events); + Assert.DoesNotContain("match", remaining); + Assert.Contains("other type", remaining); + Assert.Contains("outside window", remaining); + Assert.Contains("other project", remaining); + Assert.Contains("other organization", remaining); + } + + [Fact] + public async Task Create_FilterNamingAnotherOrganization_CannotReachItsEvents() + { + // Arrange + var (_, events) = await CreateDataAsync(d => d.Event().FreeProject()); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, $"organization:{SampleDataService.FREE_ORG_ID}")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + Assert.Equal(0, created.Total); + Assert.NotNull(await _eventRepository.GetByIdAsync(events.Single().Id)); + } + + [Fact] + public async Task Create_FilterSelection_ResolvesTimeWindowToAbsoluteDatesCappedAtCreation() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("past").Date(now.AddHours(-1)); + d.Event().TestProject().Message("future dated").Date(now.AddMinutes(10)); + }); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "[now-7d TO now+7d]")); + TimeProvider.Advance(TimeSpan.FromHours(3)); + var reloaded = await GetOperationAsync(created.Id); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + Assert.NotNull(created.Selection!.UtcEnd); + Assert.True(created.Selection!.UtcEnd <= created.CreatedUtc, "The window end must not be later than the moment the operation was created."); + Assert.True(created.Selection!.UtcStart < created.Selection!.UtcEnd); + Assert.Equal(created.Selection!.UtcStart, reloaded.Selection!.UtcStart); + Assert.Equal(created.Selection.UtcEnd, reloaded.Selection.UtcEnd); + Assert.Equal(["future dated"], (await GetRemainingMessagesAsync(events)).ToArray()); + } + + [Fact] + public async Task Create_FilterSelection_DoesNotIncludeEventsStoredAfterCreation() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + await CreateDataAsync(d => d.Event().TestProject().Message("before").Date(now.AddHours(-1))); + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + + // Act + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + var (_, lateEvents) = await CreateDataAsync(d => d.Event().TestProject().Message("late arrival").Date(now.AddHours(-1)).CreatedDate(TimeProvider.GetUtcNow().UtcDateTime)); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var finished = await GetOperationAsync(created.Id); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(1, finished.Succeeded); + Assert.NotNull(await _eventRepository.GetByIdAsync(lateEvents.Single().Id)); + } + + [Fact] + public async Task Create_IdSelection_DeletesTheIdsThroughTheSameSideEffectsAsTheSingleAction() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Date(now.AddHours(-1)).Message("First selected event") + .Mutate(e => e.Data = new() { ["payload"] = new string('x', 4096) }); + d.Event().TestProject().Date(now.AddDays(-2)).Message("Second selected event") + .Mutate(e => e.Data = new() { ["payload"] = new string('y', 4096) }); + d.Event().TestProject(); + }); + var selected = events.Take(2).ToDictionary(e => e.Id); + var removed = new List(); + using var removing = _eventRepository.DocumentsRemoving.AddHandler((_, args) => + { + removed.AddRange(args.Documents); + return Task.CompletedTask; + }); + var usageService = GetService(); + var request = new NewBulkOperation + { + Type = "events.delete", + OrganizationId = SampleDataService.TEST_ORG_ID, + Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = selected.Keys.ToList() } + }; + + // Act + var created = await CreateOperationAsync(request); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var finished = await GetOperationAsync(created.Id); + // Missing ids fall back to a search across daily indexes, which must observe the completed deletions. + await RefreshDataAsync(); + var remaining = await _eventRepository.GetByIdsAsync(events.Select(e => e.Id).ToArray()); + var usage = await usageService.GetUsageAsync(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(2, finished.Succeeded); + Assert.Equal(events[2].Id, Assert.Single(remaining).Id); + Assert.Equal(2, usage.CurrentUsage.Deleted); + Assert.Equal(selected.Keys.Order(), removed.Select(e => e.Id).Order()); + Assert.All(removed, deleted => + { + var original = selected[deleted.Id]; + Assert.Equal(original.Date, deleted.Date); + Assert.Equal(original.OrganizationId, deleted.OrganizationId); + Assert.Equal(original.ProjectId, deleted.ProjectId); + Assert.Equal(original.StackId, deleted.StackId); + Assert.Null(deleted.Message); + Assert.Empty(deleted.Data!); + }); + } + + [Fact] + public async Task Create_IdSelectionWithAnotherOrganizationsEvent_LeavesThatEventAndCountsItFailed() + { + // Arrange + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject(); + d.Event().FreeProject(); + }); + var own = events.Single(e => e.OrganizationId == SampleDataService.TEST_ORG_ID); + var foreign = events.Single(e => e.OrganizationId == SampleDataService.FREE_ORG_ID); + + // Act + var created = await CreateOperationAsync(new NewBulkOperation + { + Type = "events.delete", + OrganizationId = SampleDataService.TEST_ORG_ID, + Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [own.Id, foreign.Id] } + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var finished = await GetOperationAsync(created.Id); + Assert.Equal(1, finished.Succeeded); + Assert.Equal(1, finished.Failed); + Assert.Null(await _eventRepository.GetByIdAsync(own.Id)); + Assert.NotNull(await _eventRepository.GetByIdAsync(foreign.Id)); + } + + [Fact] + public async Task Create_ProjectScopedIdSelectionWithAnotherProjectsEvent_LeavesThatEvent() + { + // Arrange + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject(); + d.Event().TestProject().Project(SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID); + }); + var otherProject = events.Single(e => e.ProjectId == SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID); + + // Act + var created = await CreateOperationAsync(new NewBulkOperation + { + Type = "events.delete", + OrganizationId = SampleDataService.TEST_ORG_ID, + ProjectId = SampleDataService.TEST_PROJECT_ID, + Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = events.Select(e => e.Id).ToList() } + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var finished = await GetOperationAsync(created.Id); + Assert.Equal(1, finished.Failed); + Assert.NotNull(await _eventRepository.GetByIdAsync(otherProject.Id)); + } + + [Fact] + public async Task Create_OtherOrganizationOrProject_IsNotFoundAndStartsNothing() + { + // Arrange + var freeProjectInTestOrganization = FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.FREE_PROJECT_ID, ""); + + // Act + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations") + .Content(FilterRequest(SampleDataService.FREE_ORG_ID, null, "")).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations") + .Content(freeProjectInTestOrganization).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPath("bulk-operations") + .Content(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")).StatusCodeShouldBeNotFound()); + + // Assert + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); + } + + [Fact] + public async Task Create_WithoutAUserSession_IsRejected() + { + // Arrange + var request = FilterRequest(SampleDataService.TEST_ORG_ID, null, ""); + + // Act + await SendRequestAsync(r => r.Post().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeUnauthorized()); + await SendRequestAsync(r => r.Post().AsTestOrganizationClientUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeForbidden()); + + // Assert + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + } + + [Fact] + public async Task Create_GlobalAdmin_CanStartOperationForAnyOrganization() + { + // Arrange + var request = FilterRequest(SampleDataService.FREE_ORG_ID, null, ""); + + // Act + var created = await SendRequestAsAsync(r => r.Post().AsGlobalAdminUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeAccepted()); + + // Assert + Assert.NotNull(created); + Assert.Equal(SampleDataService.FREE_ORG_ID, created.OrganizationId); + } + + [Fact] + public async Task Get_OperationOfAnotherOrganization_IsNotFoundForListGetAndCancel() + { + // Arrange + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + + // Act + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("tasks", created.Id).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("tasks", created.Id, "cancel").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "tasks").StatusCodeShouldBeNotFound()); + var asAdmin = await SendRequestAsAsync(r => r.AsGlobalAdminUser().AppendPaths("tasks", created.Id).StatusCodeShouldBeOk()); + + // Assert + Assert.Equal(created.Id, asAdmin?.Id); + Assert.Equal(BackgroundTaskStatus.Queued, (await GetOperationAsync(created.Id)).Status); + } + + [Fact] + public async Task GetByOrganization_ReturnsActiveAndRecentOperationsNewestFirst() + { + // Arrange + var first = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + TimeProvider.Advance(TimeSpan.FromSeconds(5)); + var second = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Act + var list = await SendRequestAsAsync>(r => r.AsTestOrganizationUser() + .AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "tasks").StatusCodeShouldBeOk()); + + // Assert + Assert.NotNull(list); + Assert.Equal([second.Id, first.Id], list.Select(o => o.Id).ToArray()); + Assert.Equal(BackgroundTaskStatus.Queued, list.First().Status); + Assert.Equal(BackgroundTaskStatus.Completed, list.Last().Status); + } + + [Fact] + public async Task Create_WhileADestructiveOperationIsActiveForTheProject_ReturnsConflict() + { + // Arrange + var active = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Act + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations") + .Content(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "type:log")).StatusCodeShouldBeConflict()); + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations") + .Content(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")).StatusCodeShouldBeConflict()); + var otherProject = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID, "")); + var otherOrganization = await CreateOperationAsync(FilterRequest(SampleDataService.FREE_ORG_ID, SampleDataService.FREE_PROJECT_ID, ""), r => r.AsFreeOrganizationUser()); + + // Assert + Assert.NotEqual(active.Id, otherProject.Id); + Assert.Equal(SampleDataService.FREE_ORG_ID, otherOrganization.OrganizationId); + } + + [Fact] + public async Task Create_AfterTheActiveOperationFinishesOrIsCancelled_IsAllowedAgain() + { + // Arrange + var first = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var second = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Act + var cancelled = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", second.Id, "cancel").StatusCodeShouldBeAccepted()); + var third = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, (await GetOperationAsync(first.Id)).Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, cancelled?.Status); + Assert.NotEqual(second.Id, third.Id); + } + + [Fact] + public async Task Create_WhileTheActiveOperationHasStoppedReportingProgress_RetiresItAndIsAllowed() + { + // Arrange + var stale = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Act + TimeProvider.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + var next = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + var retired = await GetOperationAsync(stale.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Queued, next.Status); + // A delayed delivery of the stale operation must not run alongside its replacement. + Assert.Equal(BackgroundTaskStatus.Failed, retired.Status); + } + + [Fact] + public async Task Create_InvalidRequests_AreRejectedWithoutStartingAnything() + { + // Arrange + var ids = Enumerable.Range(0, 1001).Select(_ => ObjectId.GenerateNewId().ToString()).ToList(); + var requests = new[] + { + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Type = "events.explode" }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Parameters = new Dictionary { ["x"] = "y" } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "type:(log"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "[now+1d TO now+2d]"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "not a date"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "date|not a date"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "date|"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "-"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "[ - ]"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "[now/d TO now-1d/d]"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Time = "[now/d TO now/d]", Offset = "not an offset" } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Time = "[now/d TO now/d]", Offset = "2147483647d" } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Ids = ["537650f3b77efe23a47914f3"] } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [] } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ["not-an-id"] } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Filter = "type:log", Ids = ["537650f3b77efe23a47914f3"] } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ids } }, + // An undefined kind must not fall through to an empty filter that selects everything. + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = (BulkOperationSelectionKind)2 } } + }; + + // Act + foreach (var request in requests) + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeUnprocessableEntity()); + + // Assert + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + } + + [Fact] + public async Task Create_PremiumFilterOnFreePlan_RequiresUpgrade() + { + // Arrange + var request = FilterRequest(SampleDataService.FREE_ORG_ID, null, "tag:premium-tag"); + + // Act + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeUpgradeRequired()); + + // Assert + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); + } + + [Fact] + public async Task Cancel_QueuedOperation_CancelsImmediatelyAndTheJobDoesNothing() + { + // Arrange + var (_, events) = await CreateDataAsync(d => d.Event().TestProject()); + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + + // Act + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", created.Id, "cancel").StatusCodeShouldBeAccepted()); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, response?.Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, (await GetOperationAsync(created.Id)).Status); + Assert.NotNull(await _eventRepository.GetByIdAsync(events.Single().Id)); + } + + [Fact] + public async Task Execute_MoreMatchesThanOneBatch_ProcessesEveryBatchAndPublishesProgress() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var messages = new System.Collections.Concurrent.ConcurrentQueue(); + await GetService().SubscribeAsync(m => messages.Enqueue(m), TestCancellationToken); + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + + // Act + await RunOperationAsync(_bulkOperationService, created.Id, batchSize: 2, TestCancellationToken); + await WaitForAsync(() => messages.Any(m => m.Status == BackgroundTaskStatus.Completed)); + + // Assert + var finished = await GetOperationAsync(created.Id); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(5, finished.Processed); + Assert.Equal(5, finished.Succeeded); + Assert.Empty(await GetRemainingMessagesAsync(events)); + + var forOperation = messages.Where(m => m.Id == created.Id).ToList(); + Assert.Contains(forOperation, m => m.Status == BackgroundTaskStatus.Running && m.Processed == 2); + Assert.Contains(forOperation, m => m.Status == BackgroundTaskStatus.Running && m.Processed == 4); + Assert.Contains(forOperation, m => m.Status == BackgroundTaskStatus.Running && m.Processed == 5); + var last = forOperation.OrderBy(m => m.UpdatedUtc).ThenBy(m => m.Processed).Last(); + Assert.Equal(BackgroundTaskStatus.Completed, last.Status); + Assert.All(forOperation, m => Assert.Equal(SampleDataService.TEST_ORG_ID, m.OrganizationId)); + } + + [Fact] + public async Task Execute_JobRestartsMidOperation_ResumesAfterTheSavedCursorWithoutRepeatingBatches() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var recorder = new RecordingHandler(); + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + using var shutdown = new CancellationTokenSource(); + recorder.AfterBatch = () => shutdown.Cancel(); + + // Act + await RunOperationAsync(service, created.Id, batchSize: 2, shutdown.Token); + var interrupted = await _taskService.GetAsync(created.Id); + recorder.AfterBatch = null; + await RunOperationAsync(service, created.Id, batchSize: 2, TestCancellationToken); + var finished = await _taskService.GetAsync(created.Id); + + // Assert + Assert.NotNull(interrupted); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); + Assert.Equal(2, interrupted.Processed); + Assert.NotNull(interrupted.Bulk!.Cursor); + Assert.NotNull(finished); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(5, finished.Processed); + Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); + Assert.Equal([2, 2, 1], recorder.Batches.Select(b => b.Count).ToArray()); + } + + [Fact] + public async Task Execute_CancelRequestedDuringABatch_StopsBeforeTheNextBatch() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var recorder = new RecordingHandler(); + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + recorder.AfterBatch = () => _taskService.CancelAsync(created.Id).GetAwaiter().GetResult(); + + // Act + await RunOperationAsync(service, created.Id, batchSize: 2, TestCancellationToken); + + // Assert + var finished = await _taskService.GetAsync(created.Id); + Assert.NotNull(finished); + Assert.Equal(BackgroundTaskStatus.Cancelled, finished.Status); + Assert.Equal(2, finished.Processed); + Assert.NotNull(finished.CompletedUtc); + Assert.Single(recorder.Batches); + } + + [Fact] + public async Task Execute_NonDestructiveOperationsDoNotBlockEachOtherOrDestructiveOnes() + { + // Arrange + var recorder = new RecordingHandler(); + var service = CreateService(recorder, new RecordingHandler("test.destructive", true)); + + // Act + var first = await service.CreateAsync(NewRecordingOperation("test.record")); + var second = await service.CreateAsync(NewRecordingOperation("test.record")); + var destructive = await service.CreateAsync(NewRecordingOperation("test.destructive")); + var blocked = await service.CreateAsync(NewRecordingOperation("test.destructive")); + + // Assert + Assert.NotNull(first.Task); + Assert.NotNull(second.Task); + Assert.NotNull(destructive.Task); + Assert.Null(blocked.Task); + Assert.Equal(destructive.Task.Id, blocked.Conflict?.Id); + } + + private static NewBulkOperation FilterRequest(string organizationId, string? projectId, string filter, string time = "[now-7d TO now]") + { + return new NewBulkOperation + { + Type = "events.delete", + OrganizationId = organizationId, + ProjectId = projectId, + Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Filter = filter, Time = time } + }; + } + + private async Task CreateOperationAsync(NewBulkOperation request, Action? authorize = null) + { + var created = await SendRequestAsAsync(r => + { + r.Post().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeAccepted(); + if (authorize is null) + r.AsTestOrganizationUser(); + else + authorize(r); + }); + + Assert.NotNull(created); + return created; + } + + private async Task GetOperationAsync(string id) + { + var operation = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", id).StatusCodeShouldBeOk()); + Assert.NotNull(operation); + return operation; + } + + private async Task> GetRemainingMessagesAsync(IEnumerable events) + { + await RefreshDataAsync(); + var remaining = await _eventRepository.GetByIdsAsync(events.Select(e => e.Id).ToArray(), o => o.Cache(false)); + return remaining.Select(e => e.Message ?? String.Empty).OrderBy(m => m, StringComparer.Ordinal).ToList(); + } + + private static async Task WaitForAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(10); + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(25); + } + + private BulkOperationService CreateService(params IBulkOperationHandler[] handlers) + { + return CreateService(handlers, [GetService()]); + } + + private BulkOperationService CreateService(IBulkOperationHandler[] handlers, IBulkSelectionResolver[] resolvers, BackgroundTaskService? taskService = null) + { + return new BulkOperationService(taskService ?? _taskService, GetService(), GetService(), handlers, resolvers, TimeProvider); + } + + private async Task CreateRecordingOperationAsync(BulkOperationService service) + { + var result = await service.CreateAsync(NewRecordingOperation("test.record")); + Assert.NotNull(result.Task); + return result.Task; + } + + private BackgroundTask NewRecordingOperation(string type) + { + var now = TimeProvider.GetUtcNow().UtcDateTime; + return new BackgroundTask + { + OrganizationId = SampleDataService.TEST_ORG_ID, + ProjectId = SampleDataService.TEST_PROJECT_ID, + Type = type, + Description = "Record events", + CreatedByUserId = "537650f3b77efe23a47914f8", + Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Filter, + Filter = String.Empty, + UtcStart = now.AddDays(-7), + UtcEnd = now, + UtcCreatedBefore = now + } + } + }; + } + + private Task RunOperationAsync(BulkOperationService service, string operationId, int batchSize, CancellationToken shutdownToken) + { + return RunWorkItemAsync(service, new BulkOperationWorkItem { TaskId = operationId, OrganizationId = SampleDataService.TEST_ORG_ID }, batchSize, shutdownToken); + } + + private Task RunWorkItemAsync(BulkOperationService service, BulkOperationWorkItem workItem, int batchSize, CancellationToken shutdownToken) + { + var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(service, _taskService, [GetService()], + new BulkOperationOptions { BatchSize = batchSize, RetryDelay = TimeSpan.Zero }, GetService(), TimeProvider, Log); + var context = new WorkItemContext(workItem, "test-job", EmptyLock.Empty, shutdownToken, static (_, _) => Task.CompletedTask); + + return handler.HandleItemAsync(context); + } + + private sealed class RecordingHandler(string type = "test.record", bool isDestructive = false) : IBulkOperationHandler + { + private int _calls; + + public List> Batches { get; } = []; + public Action? AfterBatch { get; set; } + + /// Called with the 1 based batch number before the batch is recorded; return an exception to make it fail. + public Func? Failure { get; set; } + + public string Type => type; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => isDestructive; + + public Task AuthorizeAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task ValidateAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public string Describe(BackgroundTask operation) => "Record events"; + + public Task ExecuteBatchAsync(BackgroundTask operation, IReadOnlyCollection ids, CancellationToken cancellationToken) + { + var failure = Failure?.Invoke(++_calls); + if (failure is not null) + throw failure; + + Batches.Add(ids.ToList()); + AfterBatch?.Invoke(); + return Task.FromResult(new BulkBatchResult(ids.Count, 0)); + } + } +} diff --git a/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs b/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs new file mode 100644 index 0000000000..6e8ee4997c --- /dev/null +++ b/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs @@ -0,0 +1,172 @@ +using System.Globalization; +using System.Security.Claims; +using Exceptionless.Core; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Tests.Utility; +using Exceptionless.Web.Api.Handlers; +using Exceptionless.Web.Api.Messages; +using Exceptionless.Web.Extensions; +using Foundatio.Xunit; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Api.Handlers; + +public class BackgroundTaskHandlerTests : TestWithLoggingBase +{ + private static readonly DateTimeOffset UtcNow = new(2026, 11, 3, 12, 0, 0, TimeSpan.Zero); + + public BackgroundTaskHandlerTests(ITestOutputHelper output) : base(output) { } + + [Theory] + [InlineData(false, BackgroundTaskStatus.Cancelled, 1)] + [InlineData(true, BackgroundTaskStatus.Running, 0)] + public async Task Handle_CancelBulkTask_ReleasesSelectionOnlyAfterItBecomesTerminal(bool running, BackgroundTaskStatus expectedStatus, int expectedReleases) + { + // Arrange + const string organizationId = "537650f3b77efe23a47914f4"; + const string pinnedSelection = "saved-event-selection"; + var time = new FakeTimeProvider(); + using var harness = new BackgroundTaskTestHarness(time, Log); + var user = new User + { + Id = "537650f3b77efe23a47914f8", + EmailAddress = "user@example.test", + OrganizationIds = new HashSet { organizationId } + }; + var task = new BackgroundTask + { + OrganizationId = organizationId, + Type = DeleteEventsBulkOperationHandler.OperationType, + Description = "Delete matching events", + CreatedByUserId = user.Id, + CanCancel = true, + Bulk = new BulkOperationDetails { Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter } } + }; + await harness.Tasks.AddAsync(task); + await harness.Cache.SetAsync(BulkOperationService.GetPinnedSelectionCacheKey(task.Id), pinnedSelection, BackgroundTaskService.TaskLifetime); + if (running) + { + await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Start(time.GetUtcNow().UtcDateTime); + return true; + }); + } + + var operation = new Mock(MockBehavior.Strict); + operation.SetupGet(handler => handler.Type).Returns(task.Type); + operation.SetupGet(handler => handler.Entity).Returns(BulkOperationEntity.Events); + var resolver = new Mock(MockBehavior.Strict); + resolver.SetupGet(selection => selection.Entity).Returns(BulkOperationEntity.Events); + resolver.Setup(selection => selection.ReleaseAsync(It.IsAny(), pinnedSelection, null)).Returns(Task.CompletedTask); + var bulkService = new BulkOperationService(harness.Tasks, harness.Cache, harness.LockProvider, [operation.Object], [resolver.Object], time); + var context = new DefaultHttpContext { User = new ClaimsPrincipal(user.ToIdentity()) }; + context.Request.SetUser(user); + var organizations = new Mock(MockBehavior.Strict); + var projects = new Mock(MockBehavior.Strict); + var handler = new BackgroundTaskHandler(harness.Tasks, bulkService, [resolver.Object], organizations.Object, projects.Object, + new AppOptions(), time, new HttpContextAccessor { HttpContext = context }, Log.CreateLogger()); + + // Act + var result = await handler.Handle(new CancelBackgroundTask(task.Id)); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.True(result.IsSuccess); + Assert.Equal(expectedStatus, result.Value.Value.Status); + Assert.Equal(expectedStatus, stored!.Status); + Assert.True(await harness.Tasks.IsCancelRequestedAsync(task.Id)); + resolver.Verify(selection => selection.ReleaseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(expectedReleases)); + resolver.Verify(selection => selection.ReleaseAsync( + It.Is(current => current.Id == task.Id && current.Status == BackgroundTaskStatus.Cancelled), pinnedSelection, null), Times.Exactly(expectedReleases)); + } + + [Theory] + [InlineData("not a date")] + [InlineData("24h")] + [InlineData("date|not a date")] + [InlineData("date|")] + [InlineData("created_utc|[now-1d TO now]")] + [InlineData("-")] + [InlineData("[ - ]")] + [InlineData("- now")] + [InlineData("now -")] + [InlineData("[now-1d TO now")] + [InlineData("[now/d TO now-1d/d]")] + [InlineData("last 0 days")] + [InlineData("last 999999999999999999 days")] + public void TryParseSelectionTime_InvalidRange_IsRejectedBeforeDefaultsCanBroadenIt(string time) + { + // Arrange + var relativeBaseTime = UtcNow; + + // Act + bool parsed = BackgroundTaskHandler.TryParseSelectionTime(time, relativeBaseTime, out var range); + + // Assert + Assert.False(parsed); + Assert.Null(range); + } + + [Theory] + [InlineData(null, 0, "0001-01-01T00:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData("", 0, "0001-01-01T00:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData("[now-1d TO now]", 0, "2026-11-02T12:00:00Z", "2026-11-03T12:00:00Z")] + [InlineData("date|[now/d TO now/d]", -5, "2026-11-03T05:00:00Z", "2026-11-04T04:59:59.999Z")] + [InlineData("[now-1d TO *]", 0, "2026-11-02T12:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData("[* TO now]", 0, "0001-01-01T00:00:00Z", "2026-11-03T12:00:00Z")] + [InlineData("[* TO *]", 0, "0001-01-01T00:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData("{* TO *}", 0, "0001-01-01T00:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData("(); + BackgroundTask? duringWork = null; + var handler = CreateHandler(async context => + { + await context.ReportProgressAsync(0, "Starting"); + _time.Advance(TimeSpan.FromSeconds(30)); + await context.ReportProgressAsync(50, "Halfway"); + duringWork = await _harness.Tasks.GetAsync(task.Id); + }); + + // Act + await handler.HandleItemAsync(CreateContext(task.Id, (progress, message) => + { + reported.Add((progress, message)); + return Task.CompletedTask; + })); + + // Assert + Assert.Equal([(0, "Starting"), (50, "Halfway")], reported); + Assert.Equal(BackgroundTaskStatus.Running, duringWork!.Status); + Assert.Equal(50, duringWork.Progress); + Assert.Equal("Halfway", duringWork.Message); + Assert.NotNull(duringWork.StartedUtc); + Assert.Contains(_harness.Publisher.Updates, u => u.Status == BackgroundTaskStatus.Running && u.Progress == 50); + } + + [Fact] + public async Task HandleItemAsync_ProgressPublicationWaits_ReportsToTheJobBeforeTrackingFinishes() + { + // Arrange + var task = await AddTaskAsync(); + var publishing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releasePublication = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + bool reported = false; + var handler = CreateHandler(async context => + { + _harness.Publisher.BeforePublish = () => + { + _harness.Publisher.BeforePublish = null; + publishing.SetResult(); + return releasePublication.Task; + }; + await context.ReportProgressAsync(50, "Halfway"); + }); + var context = CreateContext(task.Id, (_, _) => + { + reported = true; + return Task.CompletedTask; + }); + + // Act + var handling = handler.HandleItemAsync(context); + bool reportedWhilePublishing; + try + { + await publishing.Task.WaitAsync(TestCancellationToken); + reportedWhilePublishing = reported; + } + finally + { + releasePublication.SetResult(); + await handling; + } + + // Assert + Assert.True(reportedWhilePublishing); + } + + [Fact] + public async Task HandleItemAsync_ProgressTrackingIsCancelled_StillReportsProgressAndCompletesTheWork() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => new CancellingTaskCacheClient(inner, task => task.Status is BackgroundTaskStatus.Running && task.Progress == 50)); + var task = await AddTaskAsync(harness: harness); + bool continued = false; + var reported = new List<(int Progress, string? Message)>(); + var handler = CreateHandler(async context => + { + await context.ReportProgressAsync(50, "Halfway"); + continued = true; + }, harness); + var context = CreateContext(task.Id, (progress, message) => + { + reported.Add((progress, message)); + return Task.CompletedTask; + }); + + // Act + var exception = await Record.ExceptionAsync(() => handler.HandleItemAsync(context)); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Null(exception); + Assert.True(continued); + Assert.Equal([(50, "Halfway")], reported); + Assert.Equal(BackgroundTaskStatus.Completed, stored!.Status); + } + + [Fact] + public async Task HandleItemAsync_JobProgressCallbackIsCancelled_RethrowsCancellation() + { + // Arrange + var task = await AddTaskAsync(); + var failure = new TaskCanceledException("The job progress callback was cancelled"); + var handler = CreateHandler(context => context.ReportProgressAsync(50, "Halfway")); + var context = CreateContext(task.Id, (_, _) => Task.FromException(failure)); + + // Act + var thrown = await Record.ExceptionAsync(() => handler.HandleItemAsync(context)); + + // Assert + Assert.Same(failure, thrown); + } + + [Fact] + public async Task HandleItemAsync_FailureTrackingIsCancelled_RethrowsTheOriginalHandlerException() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => new CancellingTaskCacheClient(inner, task => task.Status is BackgroundTaskStatus.Failed)); + var task = await AddTaskAsync(harness: harness); + var failure = new InvalidOperationException("The work failed"); + var handler = CreateHandler(_ => Task.FromException(failure), harness); + + // Act + var thrown = await Record.ExceptionAsync(() => handler.HandleItemAsync(CreateContext(task.Id))); + + // Assert + Assert.Same(failure, thrown); + } + + [Fact] + public async Task HandleItemAsync_WorkSucceeds_CompletesTheTaskAtOneHundredPercentWithTheLastMessage() + { + // Arrange + var task = await AddTaskAsync(); + var handler = CreateHandler(async context => + { + await context.ReportProgressAsync(50, "Events removed: 3"); + await context.ReportProgressAsync(100, "Events removed: 3, stacks removed: 1"); + }); + + // Act + await handler.HandleItemAsync(CreateContext(task.Id)); + var stored = await _harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, stored!.Status); + Assert.Equal(100, stored.Progress); + Assert.Equal("Events removed: 3, stacks removed: 1", stored.Message); + Assert.NotNull(stored.CompletedUtc); + Assert.Empty(await _harness.Tasks.GetActiveIdsAsync(OrganizationId)); + Assert.Equal(BackgroundTaskStatus.Completed, _harness.Publisher.Updates[^1].Status); + } + + [Fact] + public async Task HandleItemAsync_WorkThrows_FailsTheTaskAndRethrowsThenARetryBringsItBackToRunningAndCompletes() + { + // Arrange + var task = await AddTaskAsync(); + int calls = 0; + BackgroundTask? duringRetry = null; + IReadOnlyList? activeDuringRetry = null; + var handler = CreateHandler(async context => + { + if (++calls == 1) + throw new InvalidOperationException("Elasticsearch unavailable"); + + duringRetry = await _harness.Tasks.GetAsync(task.Id); + activeDuringRetry = await _harness.Tasks.GetActiveIdsAsync(OrganizationId); + await context.ReportProgressAsync(100, "Done"); + }); + + // Act + var thrown = await Record.ExceptionAsync(() => handler.HandleItemAsync(CreateContext(task.Id))); + var failed = await _harness.Tasks.GetAsync(task.Id); + var activeAfterFailure = await _harness.Tasks.GetActiveIdsAsync(OrganizationId); + await handler.HandleItemAsync(CreateContext(task.Id)); + var completed = await _harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.IsType(thrown); + Assert.Equal(BackgroundTaskStatus.Failed, failed!.Status); + Assert.DoesNotContain("Elasticsearch unavailable", failed.Message); + Assert.Empty(activeAfterFailure); + Assert.Equal(BackgroundTaskStatus.Running, duringRetry!.Status); + Assert.Null(duringRetry.CompletedUtc); + Assert.Equal([task.Id], activeDuringRetry!.ToArray()); + Assert.Equal(BackgroundTaskStatus.Completed, completed!.Status); + Assert.Empty(await _harness.Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Fact] + public async Task HandleItemAsync_ResultIsNotASuccess_PassesItToTheJobAndKeepsTheTaskRunning() + { + // Arrange + var task = await AddTaskAsync(); + var handler = CreateHandler(context => + { + context.Result = JobResult.CancelledWithMessage("Shutting down"); + return Task.CompletedTask; + }); + var context = CreateContext(task.Id); + + // Act + await handler.HandleItemAsync(context); + var stored = await _harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.False(context.Result.IsSuccess); + Assert.Equal("Shutting down", context.Result.Message); + Assert.Equal(BackgroundTaskStatus.Running, stored!.Status); + } + + [Fact] + public async Task HandleItemAsync_WorkItemWithoutATaskId_PassesStraightThroughWithoutTouchingAnyTask() + { + // Arrange + var other = await AddTaskAsync(); + WorkItemContext? received = null; + var handler = CreateHandler(async context => + { + received = context; + await context.ReportProgressAsync(10, "Working"); + }); + var context = new WorkItemContext(new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = ProjectId }, "job", EmptyLock.Empty, TestCancellationToken, (_, _) => Task.CompletedTask); + + // Act + await handler.HandleItemAsync(context); + var untouched = await _harness.Tasks.GetAsync(other.Id); + + // Assert + Assert.Same(context, received); + Assert.Equal(BackgroundTaskStatus.Queued, untouched!.Status); + Assert.Empty(_harness.Publisher.Updates); + } + + [Fact] + public async Task HandleItemAsync_TaskRecordIsGone_StillRunsTheWork() + { + // Arrange + bool ran = false; + var handler = CreateHandler(async context => + { + await context.ReportProgressAsync(100, "Done"); + ran = true; + }); + + // Act + await handler.HandleItemAsync(CreateContext(Foundatio.Repositories.Utility.ObjectId.GenerateNewId().ToString())); + + // Assert + Assert.True(ran); + Assert.Empty(_harness.Publisher.Updates); + } + + [Theory] + [InlineData(BackgroundTaskStatus.Cancelled)] + [InlineData(BackgroundTaskStatus.Completed)] + public async Task HandleItemAsync_TaskIsCancelledOrCompleted_DoesNotRunOrRevive(BackgroundTaskStatus status) + { + // Arrange + var task = await AddTaskAsync(canCancel: true); + await _harness.Tasks.UpdateAsync(task.Id, current => + { + current.Finish(status, "Finished", _time.GetUtcNow().UtcDateTime); + return true; + }); + bool ran = false; + var handler = CreateHandler(_ => + { + ran = true; + return Task.CompletedTask; + }); + var context = CreateContext(task.Id); + + // Act + await handler.HandleItemAsync(context); + + // Assert + Assert.False(ran); + Assert.True(context.Result.IsSuccess); + Assert.Equal(status, (await _harness.Tasks.GetAsync(task.Id))!.Status); + } + + [Theory] + [InlineData(BackgroundTaskStatus.Cancelled)] + [InlineData(BackgroundTaskStatus.Completed)] + public async Task HandleItemAsync_TaskStatusCannotBeChecked_DoesNotRunTheWorker(BackgroundTaskStatus status) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.Zero }, lockAttempts: 1); + var task = await AddTaskAsync(harness: harness); + await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Finish(status, "Finished", _time.GetUtcNow().UtcDateTime); + return true; + }); + await using var held = await harness.LockProvider.AcquireAsync(BackgroundTaskService.GetUpdateLockResource(task.Id), TimeSpan.FromSeconds(30), TestCancellationToken); + bool ran = false; + var inner = new DelegateWorkItemHandler(_ => + { + ran = true; + return Task.CompletedTask; + }); + var handler = new BackgroundTaskTrackingWorkItemHandler(inner, harness.Tasks, _time); + + // Act + var exception = await Record.ExceptionAsync(() => handler.HandleItemAsync(CreateContext(task.Id))); + + // Assert + Assert.IsType(exception); + Assert.False(ran); + Assert.Equal(status, (await harness.Tasks.GetAsync(task.Id))!.Status); + } + + [Fact] + public void Handler_DelegatesTheJobSettingsToTheWrappedHandler() + { + // Arrange + var inner = new DelegateWorkItemHandler(_ => Task.CompletedTask) { AutoRenewLockOnProgress = true }; + var handler = new BackgroundTaskTrackingWorkItemHandler(inner, _harness.Tasks, _time); + + // Act + handler.AutoRenewLockOnProgress = false; + + // Assert + Assert.False(inner.AutoRenewLockOnProgress); + Assert.Same(inner.Log, handler.Log); + } + + private async Task AddTaskAsync(bool canCancel = false, BackgroundTaskTestHarness? harness = null) + { + var task = new BackgroundTask + { + OrganizationId = OrganizationId, + ProjectId = ProjectId, + Type = BackgroundTaskTypes.ResetProjectData, + Description = "Reset data for Test", + CreatedByUserId = "537650f3b77efe23a47914f8", + CanCancel = canCancel + }; + await (harness ?? _harness).Tasks.AddAsync(task); + return task; + } + + private BackgroundTaskTrackingWorkItemHandler CreateHandler(Func work, BackgroundTaskTestHarness? harness = null) + { + return new BackgroundTaskTrackingWorkItemHandler(new DelegateWorkItemHandler(work), (harness ?? _harness).Tasks, _time); + } + + private WorkItemContext CreateContext(string taskId, Func? progress = null) + { + var workItem = new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = ProjectId, TaskId = taskId }; + return new WorkItemContext(workItem, "job", EmptyLock.Empty, TestCancellationToken, progress ?? ((_, _) => Task.CompletedTask)); + } + + private sealed class CancellingTaskCacheClient(ICacheClient inner, Func shouldCancel) : ScopedCacheClient(inner, null), ICacheClient + { + Task ICacheClient.SetAsync(string key, T value, TimeSpan? expiresIn) + { + if (value is BackgroundTask task && shouldCancel(task)) + throw new TaskCanceledException("The task cache timed out"); + + return UnscopedCache.SetAsync(key, value, expiresIn); + } + } +} diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs new file mode 100644 index 0000000000..cac57a9277 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs @@ -0,0 +1,704 @@ +using Exceptionless.Core.Jobs.WorkItemHandlers; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Queues; +using Foundatio.Utility; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Exceptionless.Tests.Jobs.WorkItemHandlers; + +public class BulkOperationWorkItemHandlerTests(ITestOutputHelper output) : TestWithLoggingBase(output) +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private const string ProjectId = "537650f3b77efe23a47914f6"; + private readonly FakeTimeProvider _time = new(); + + [Theory] + [InlineData(1, BackgroundTaskStatus.Completed, 1)] + [InlineData(3, BackgroundTaskStatus.Failed, 0)] + public async Task HandleItemAsync_StartupTemporarilyFails_UsesBoundedContinuationWithoutAbandoningAnActiveTask(int startupFailures, BackgroundTaskStatus expectedStatus, int expectedBatches) + { + // Arrange + var queue = new InMemoryQueue(new InMemoryQueueOptions + { + TimeProvider = _time, + LoggerFactory = Log, + Retries = 2, + RetryDelay = TimeSpan.Zero + }); + var locks = new FaultingLockProvider(); + using var harness = new BackgroundTaskTestHarness(_time, Log, decorateLockProvider: locks.Wrap, queue: queue, lockAttempts: 1); + using var subscription = queue.Dequeued.AddSyncHandler((_, _) => + { + if (startupFailures-- <= 0) + return; + + locks.AcquireFailuresRemaining = 1; + locks.NextAcquireFailure = new TimeoutException("Task startup temporarily unavailable"); + }); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + batches++; + return Task.FromResult(new BulkBatchResult(batch.Count, 0)); + }); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first"])); + var job = CreateJob(harness, service, resolver, batchSize: 1); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + var stats = await queue.GetQueueStatsAsync(); + + // Assert + Assert.NotNull(stored); + Assert.Equal(expectedStatus, stored.Status); + Assert.Equal(expectedBatches, batches); + Assert.Equal(expectedBatches, stored.Processed); + Assert.Equal(expectedBatches, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.Equal(0, stats.Abandoned); + Assert.Equal(0, stats.Deadletter); + Assert.Empty(await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Theory] + [InlineData(BulkOperationSelectionKind.Ids, false)] + [InlineData(BulkOperationSelectionKind.Filter, false)] + [InlineData(BulkOperationSelectionKind.Ids, true)] + [InlineData(BulkOperationSelectionKind.Filter, true)] + public async Task HandleItemAsync_CompletedBatchCheckpointTemporarilyFails_PreservesItsResult(BulkOperationSelectionKind kind, bool checkpointCommitted) + { + // Arrange + string[] ids = ["first", "second", "third"]; + var remaining = ids.ToHashSet(StringComparer.Ordinal); + var locks = new FaultingLockProvider(); + using var harness = new BackgroundTaskTestHarness(_time, Log, decorateLockProvider: locks.Wrap, lockAttempts: 1); + var executedIds = new List(); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + executedIds.AddRange(batch); + int succeeded = batch.Count(remaining.Remove); + if (++batches == 1) + { + if (checkpointCommitted) + locks.NextReleaseFailure = new TimeoutException("Checkpoint saved but lock release failed"); + else + locks.NextAcquireFailure = new TimeoutException("Checkpoint temporarily unavailable"); + } + + return Task.FromResult(new BulkBatchResult(succeeded, batch.Count - succeeded)); + }); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids, kind)); + var job = CreateJob(harness, service, resolver, batchSize: 2); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + + // Assert + Assert.Empty(remaining); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Completed, stored.Status); + Assert.Equal(ids.Length, stored.Processed); + Assert.Equal(ids.Length, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.Equal(ids, executedIds); + Assert.Equal(kind is BulkOperationSelectionKind.Filter ? 1 : 0, resolver.Releases.Count); + Assert.All(resolver.Releases, release => Assert.Equal(stored.Bulk!.Cursor, release.Cursor)); + } + + [Theory] + [InlineData(BulkOperationSelectionKind.Ids)] + [InlineData(BulkOperationSelectionKind.Filter)] + public async Task HandleItemAsync_HostStopsAfterSuccessfulBatch_PersistsItsResultBeforeStopping(BulkOperationSelectionKind kind) + { + // Arrange + string[] ids = ["first", "second", "third"]; + var remaining = ids.ToHashSet(StringComparer.Ordinal); + using var shutdown = new CancellationTokenSource(); + var locks = new FaultingLockProvider(); + using var harness = new BackgroundTaskTestHarness(_time, Log, decorateLockProvider: locks.Wrap, lockAttempts: 1); + int batches = 0; + var executedIds = new List(); + var operationHandler = new DelegateBulkHandler(batch => + { + executedIds.AddRange(batch); + int succeeded = batch.Count(remaining.Remove); + if (++batches == 1) + { + locks.NextAcquireFailure = new TimeoutException("Checkpoint temporarily unavailable"); + shutdown.Cancel(); + } + + return Task.FromResult(new BulkBatchResult(succeeded, batch.Count - succeeded)); + }); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids, kind)); + var handler = CreateHandler(harness, service, resolver, batchSize: 2); + var interruptedContext = CreateContext(created.Task!.Id, shutdown.Token); + + // Act + await handler.HandleItemAsync(interruptedContext); + var interrupted = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.False(interruptedContext.Result.IsSuccess); + Assert.NotNull(interrupted); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); + Assert.Equal(2, interrupted.Processed); + Assert.Equal(2, interrupted.Succeeded); + Assert.Empty(resolver.Releases); + + // Act + await handler.HandleItemAsync(CreateContext(created.Task.Id, TestCancellationToken)); + var completed = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(completed); + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(ids.Length, completed.Succeeded); + Assert.Equal(0, completed.Failed); + Assert.Equal(ids, executedIds); + Assert.Empty(remaining); + Assert.Equal(kind is BulkOperationSelectionKind.Filter ? 1 : 0, resolver.Releases.Count); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandleItemAsync_CompletedBatchCheckpointKeepsFailing_StopsWithoutReplayingOrDiscardingPriorProgress(bool dependencyCancelled) + { + // Arrange + string[] ids = ["first", "second", "third", "fourth", "fifth"]; + var remaining = ids.ToHashSet(StringComparer.Ordinal); + var locks = new FaultingLockProvider(); + using var harness = new BackgroundTaskTestHarness(_time, Log, decorateLockProvider: locks.Wrap, lockAttempts: 1); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + int succeeded = batch.Count(remaining.Remove); + if (++batches == 2) + { + locks.AcquireFailuresRemaining = BulkOperationWorkItemHandler.MaxTransientFailures; + locks.NextAcquireFailure = dependencyCancelled + ? new TaskCanceledException("Checkpoint cancelled by dependency") + : new TimeoutException("Checkpoint unavailable for every attempt"); + } + + return Task.FromResult(new BulkBatchResult(succeeded, batch.Count - succeeded)); + }); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids)); + var job = CreateJob(harness, service, resolver, batchSize: 2); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + var stats = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Equal(ids.Length, stored.Total); + Assert.Equal(2, stored.Processed); + Assert.Equal(2, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.Contains("Some items may have been processed", stored.Message); + Assert.Equal(["fifth"], remaining); + Assert.Equal(2, batches); + Assert.Equal(1, stats.Enqueued); + Assert.Equal(1, stats.Completed); + Assert.Empty(await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Theory] + [InlineData(BatchFailure.None)] + [InlineData(BatchFailure.Temporary)] + [InlineData(BatchFailure.Permanent)] + public async Task HandleItemAsync_TaskRecordDisappearsDuringBatch_PublishesLostAndStops(BatchFailure failure) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var task = NewTask(["first", "second"]); + int batches = 0; + var operationHandler = new DelegateBulkHandler(async batch => + { + batches++; + await harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(task.Id)); + if (failure is BatchFailure.Temporary) + throw new TimeoutException("Batch interrupted after the task record was lost"); + if (failure is BatchFailure.Permanent) + throw new InvalidOperationException("Batch rejected after the task record was lost"); + + return new BulkBatchResult(batch.Count, 0); + }); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + await service.CreateAsync(task); + var job = CreateJob(harness, service, resolver, batchSize: 1); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Null(stored); + Assert.Equal(1, batches); + var lost = Assert.Single(harness.Publisher.Updates, update => update.Status is BackgroundTaskStatus.Failed); + Assert.Equal(task.Id, lost.Id); + Assert.Equal(task.OrganizationId, lost.OrganizationId); + Assert.Equal(task.Type, lost.Type); + Assert.Equal(task.Description, lost.Description); + Assert.Contains("progress was lost", lost.Message); + Assert.Equal(1, (await harness.Queue.GetQueueStatsAsync()).Completed); + } + + [Fact] + public async Task HandleItemAsync_TaskRecordDisappearsAfterFinalCheckpoint_PublishesLostInsteadOfCompletion() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var task = NewTask(["first"]); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + batches++; + return Task.FromResult(new BulkBatchResult(batch.Count, 0)); + }); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + await service.CreateAsync(task); + using var subscription = harness.Queue.LockRenewed.AddHandler(async (_, _) => + { + var checkpoint = await harness.Tasks.GetAsync(task.Id); + if (checkpoint is { Processed: 1 }) + await harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(task.Id)); + }); + var job = CreateJob(harness, service, resolver, batchSize: 1); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Null(stored); + Assert.Equal(1, batches); + Assert.Single(harness.Publisher.Updates, update => update.Status is BackgroundTaskStatus.Failed); + Assert.DoesNotContain(harness.Publisher.Updates, update => update.Status is BackgroundTaskStatus.Completed); + Assert.Equal(1, (await harness.Queue.GetQueueStatsAsync()).Completed); + } + + [Fact] + public async Task HandleItemAsync_ContinuationCannotBeQueued_DoesNotLeaveAnActiveTaskWithoutWork() + { + // Arrange + var queue = new FaultingEnqueueQueue(new InMemoryQueueOptions + { + TimeProvider = _time, + LoggerFactory = Log, + Retries = 2, + RetryDelay = TimeSpan.Zero + }); + using var harness = new BackgroundTaskTestHarness(_time, Log, queue: queue); + int calls = 0; + var operationHandler = new DelegateBulkHandler(batch => ++calls % 2 == 0 + ? Task.FromException(new TimeoutException("Search temporarily unavailable")) + : Task.FromResult(new BulkBatchResult(batch.Count, 0))); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first", "second", "third", "fourth"])); + var job = CreateJob(harness, service, resolver, batchSize: 1); + queue.FailEnqueue = true; + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + var stats = await queue.GetQueueStatsAsync(); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.Equal(0, stats.Queued); + Assert.Equal(0, stats.Working); + Assert.Equal(0, stats.Deadletter); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Equal(4, stored.Total); + Assert.Equal(1, stored.Processed); + Assert.Equal(1, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.DoesNotContain(stored.Id, active); + } + + [Fact] + public async Task HandleItemAsync_HostStopsBeforeContinuation_KeepsTheTaskAvailableForQueueRetry() + { + // Arrange + using var shutdown = new CancellationTokenSource(); + using var harness = new BackgroundTaskTestHarness(_time, Log); + var operationHandler = new DelegateBulkHandler(_ => + { + shutdown.Cancel(); + return Task.FromException(new TimeoutException("Search interrupted during shutdown")); + }); + var resolver = new RemainingIdsResolver(["first"]); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first"], BulkOperationSelectionKind.Filter)); + var handler = CreateHandler(harness, service, resolver, batchSize: 1, retryDelay: TimeSpan.FromSeconds(5)); + var context = CreateContext(created.Task!.Id, shutdown.Token); + + // Act + var thrown = await Record.ExceptionAsync(() => handler.HandleItemAsync(context)); + var stored = await harness.Tasks.GetAsync(created.Task.Id); + var stats = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.Null(thrown); + Assert.False(context.Result.IsSuccess); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Running, stored.Status); + Assert.Null(stored.CompletedUtc); + Assert.Equal(1, stored.Bulk!.RetryCount); + Assert.Contains(stored.Id, await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + Assert.Equal(1, stats.Enqueued); + Assert.Empty(resolver.Releases); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + public async Task HandleItemAsync_SelectionIsRejected_FailsWithoutLosingCompletedCountsOrRetrying(int successfulBatches) + { + // Arrange + const string message = "Please upgrade your plan to use premium search features."; + string[] ids = ["first", "second", "third"]; + var remaining = ids.ToHashSet(StringComparer.Ordinal); + using var harness = new BackgroundTaskTestHarness(_time, Log); + var operationHandler = new DelegateBulkHandler(batch => Task.FromResult(new BulkBatchResult(batch.Count(remaining.Remove), 0))); + var resolver = new RemainingIdsResolver(remaining) + { + Failure = call => call == successfulBatches + 1 ? new BulkSelectionRejectedException(BulkOperationCheckResult.PlanLimit(message)) : null + }; + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids, BulkOperationSelectionKind.Filter)); + var job = CreateJob(harness, service, resolver, batchSize: 1); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + var stats = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Equal(message, stored.Message); + Assert.Equal(ids.Length, stored.Total); + Assert.Equal(successfulBatches, stored.Processed); + Assert.Equal(successfulBatches, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.Equal(0, stored.Bulk!.RetryCount); + Assert.Equal(ids.Length - successfulBatches, remaining.Count); + Assert.Equal(1, stats.Enqueued); + Assert.Equal(1, stats.Completed); + Assert.Empty(await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + var released = Assert.Single(resolver.Releases); + Assert.Equal("{}", released.PinnedState); + Assert.Equal(stored.Bulk.Cursor, released.Cursor); + } + + [Fact] + public async Task HandleItemAsync_TransientFailure_KeepsTheSelectionUntilContinuationCompletes() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var remaining = new HashSet(["first"], StringComparer.Ordinal); + int attempts = 0; + var operationHandler = new DelegateBulkHandler(batch => ++attempts == 1 + ? Task.FromException(new TimeoutException("Search temporarily unavailable")) + : Task.FromResult(new BulkBatchResult(batch.Count(remaining.Remove), 0))); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first"], BulkOperationSelectionKind.Filter)); + var handler = CreateHandler(harness, service, resolver, batchSize: 1); + + // Act + await handler.HandleItemAsync(CreateContext(created.Task!.Id, TestCancellationToken)); + var interrupted = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(interrupted); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); + Assert.Equal(1, interrupted.Bulk!.RetryCount); + Assert.Empty(resolver.Releases); + Assert.Equal(2, (await harness.Queue.GetQueueStatsAsync()).Enqueued); + + // Act + await handler.HandleItemAsync(CreateContext(created.Task.Id, TestCancellationToken)); + var completed = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(completed); + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(1, completed.Succeeded); + Assert.Single(resolver.Releases); + } + + [Fact] + public async Task HandleItemAsync_RetiredBetweenBatches_DoesNotExecuteAnotherBatch() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var executedIds = new List(); + var operationHandler = new DelegateBulkHandler(batch => + { + executedIds.AddRange(batch); + return Task.FromResult(new BulkBatchResult(batch.Count, 0)); + }); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first", "second"])); + var handler = CreateHandler(harness, service, resolver, batchSize: 1); + BulkOperationCreateResult? replacement = null; + int reports = 0; + var context = CreateContext(created.Task!.Id, TestCancellationToken, async (_, _) => + { + if (++reports != 2) + return; + + // The first batch is saved, but this worker has not started the next one. + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + replacement = await service.CreateAsync(NewTask(["replacement"])); + }); + + // Act + await handler.HandleItemAsync(context); + var retired = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(replacement); + Assert.NotNull(replacement.Task); + Assert.NotNull(retired); + Assert.Equal(BackgroundTaskStatus.Failed, retired.Status); + Assert.Equal(1, retired.Processed); + Assert.Equal(["first"], executedIds); + Assert.Equal([replacement.Task.Id], await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Fact] + public async Task HandleItemAsync_ProgressExceedsSelectionLifetime_ResumesAfterShutdown() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + using var shutdown = new CancellationTokenSource(); + int batchesBeforeShutdown = (int)BackgroundTaskService.TaskLifetime.TotalHours + 1; + string[] ids = Enumerable.Range(0, batchesBeforeShutdown + 1).Select(index => index.ToString("D3")).ToArray(); + var remaining = ids.ToHashSet(StringComparer.Ordinal); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + batches++; + return Task.FromResult(new BulkBatchResult(batch.Count(remaining.Remove), 0)); + }); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids, BulkOperationSelectionKind.Filter)); + var handler = CreateHandler(harness, service, resolver, batchSize: 1); + var context = CreateContext(created.Task!.Id, shutdown.Token, (_, _) => + { + if (batches > 0) + _time.Advance(TimeSpan.FromHours(1)); + if (batches == batchesBeforeShutdown) + shutdown.Cancel(); + + return Task.CompletedTask; + }); + + // Act + await handler.HandleItemAsync(context); + var interrupted = await harness.Tasks.GetAsync(created.Task.Id); + string? selection = await service.GetPinnedSelectionAsync(created.Task.Id); + await handler.HandleItemAsync(CreateContext(created.Task.Id, TestCancellationToken)); + var completed = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(interrupted); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); + Assert.Equal(batchesBeforeShutdown, interrupted.Processed); + Assert.NotNull(selection); + Assert.NotNull(completed); + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(ids.Length, completed.Succeeded); + Assert.Equal(0, completed.Failed); + Assert.Empty(remaining); + } + + private BulkOperationService CreateService(BackgroundTaskTestHarness harness, IBulkOperationHandler handler, IBulkSelectionResolver resolver) + { + return new BulkOperationService(harness.Tasks, harness.Cache, harness.LockProvider, [handler], [resolver], _time); + } + + private WorkItemJob CreateJob(BackgroundTaskTestHarness harness, BulkOperationService service, IBulkSelectionResolver resolver, int batchSize) + { + var handler = CreateHandler(harness, service, resolver, batchSize); + var handlers = new Foundatio.Jobs.WorkItemHandlers(); + handlers.Register(() => handler); + return new WorkItemJob(harness.Queue, harness.Publisher, handlers, Log); + } + + private BulkOperationWorkItemHandler CreateHandler(BackgroundTaskTestHarness harness, BulkOperationService service, IBulkSelectionResolver resolver, int batchSize, TimeSpan? retryDelay = null) + { + return new BulkOperationWorkItemHandler(service, harness.Tasks, [resolver], + new BulkOperationOptions { BatchSize = batchSize, RetryDelay = retryDelay ?? TimeSpan.Zero }, harness.LockProvider, _time, Log); + } + + private static WorkItemContext CreateContext(string taskId, CancellationToken cancellationToken, Func? progressCallback = null) + { + var workItem = new BulkOperationWorkItem { TaskId = taskId, OrganizationId = OrganizationId }; + return new WorkItemContext(workItem, "test-job", EmptyLock.Empty, cancellationToken, progressCallback ?? (static (_, _) => Task.CompletedTask)); + } + + private static BackgroundTask NewTask(IReadOnlyCollection ids, BulkOperationSelectionKind kind = BulkOperationSelectionKind.Ids) + { + return new BackgroundTask + { + OrganizationId = OrganizationId, + ProjectId = ProjectId, + Type = "test.delete", + Description = "Delete selected items", + CreatedByUserId = "537650f3b77efe23a47914f8", + Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection { Kind = kind, Ids = kind is BulkOperationSelectionKind.Ids ? ids.ToList() : null } + } + }; + } + + private sealed class DelegateBulkHandler(Func, Task> execute) : IBulkOperationHandler + { + public string Type => "test.delete"; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => true; + public Task AuthorizeAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task ValidateAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public string Describe(BackgroundTask task) => "Delete selected items"; + + public Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) => execute(ids); + } + + private sealed class RemainingIdsResolver(HashSet remaining) : IBulkSelectionResolver + { + private int _calls; + + public Func? Failure { get; init; } + public List<(string? PinnedState, string? Cursor)> Releases { get; } = []; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken) => Task.FromResult(new BulkSelectionPreparation(BulkOperationCheckResult.Ok, remaining.Count, "{}")); + + public Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) + { + if (Failure?.Invoke(++_calls) is { } failure) + return Task.FromException(failure); + + string[] ids = remaining.Order(StringComparer.Ordinal).Take(size).ToArray(); + return Task.FromResult(new BulkSelectionPage(ids, ids.LastOrDefault())); + } + + public Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor) + { + Releases.Add((pinnedState, cursor)); + return Task.CompletedTask; + } + } + + private sealed class FaultingEnqueueQueue(InMemoryQueueOptions options) : InMemoryQueue(options) + { + public bool FailEnqueue { get; set; } + + protected override Task EnqueueImplAsync(WorkItemData data, QueueEntryOptions options) + { + return FailEnqueue + ? Task.FromException(new TimeoutException("Continuation queue unavailable")) + : base.EnqueueImplAsync(data, options); + } + } + + private sealed class FaultingLockProvider : ILockProvider + { + private ILockProvider _inner = null!; + + public Exception? NextAcquireFailure { get; set; } + public Exception? NextReleaseFailure { get; set; } + public int AcquireFailuresRemaining { get; set; } = 1; + + public ILockProvider Wrap(ILockProvider inner) + { + _inner = inner; + return this; + } + + public async Task AcquireAsync(string resource, TimeSpan? timeUntilExpires = null, bool releaseOnDispose = true, CancellationToken cancellationToken = default) + { + if (NextAcquireFailure is { } failure) + { + if (--AcquireFailuresRemaining == 0) + NextAcquireFailure = null; + + throw failure; + } + + var acquired = await _inner.AcquireAsync(resource, timeUntilExpires, releaseOnDispose, cancellationToken); + if (NextReleaseFailure is not { } releaseFailure) + return acquired; + + NextReleaseFailure = null; + return new FaultingReleaseLock(acquired, releaseFailure); + } + + public Task TryAcquireAsync(string resource, TimeSpan? timeUntilExpires = null, bool releaseOnDispose = true, CancellationToken cancellationToken = default) => _inner.TryAcquireAsync(resource, timeUntilExpires, releaseOnDispose, cancellationToken); + public Task IsLockedAsync(string resource) => _inner.IsLockedAsync(resource); + public Task ReleaseAsync(string resource, string lockId) => _inner.ReleaseAsync(resource, lockId); + public Task ReleaseAsync(string resource) => _inner.ReleaseAsync(resource); + public Task RenewAsync(string resource, string lockId, TimeSpan? timeUntilExpires = null) => _inner.RenewAsync(resource, lockId, timeUntilExpires); + } + + private sealed class FaultingReleaseLock(ILock inner, Exception failure) : ILock + { + public string LockId => inner.LockId; + public string Resource => inner.Resource; + public DateTime AcquiredTimeUtc => inner.AcquiredTimeUtc; + public TimeSpan TimeWaitedForLock => inner.TimeWaitedForLock; + public int RenewalCount => inner.RenewalCount; + public Task RenewAsync(TimeSpan? timeUntilExpires = null) => inner.RenewAsync(timeUntilExpires); + public Task ReleaseAsync() => inner.ReleaseAsync(); + + public async ValueTask DisposeAsync() + { + await inner.DisposeAsync(); + throw failure; + } + } + + public enum BatchFailure + { + None, + Temporary, + Permanent + } +} diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandlerTests.cs new file mode 100644 index 0000000000..a2fe5248a8 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandlerTests.cs @@ -0,0 +1,108 @@ +using Exceptionless.Core; +using Exceptionless.Core.Jobs.WorkItemHandlers; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Plugins.EventProcessor; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; +using Foundatio.Utility; +using Foundatio.Xunit; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Jobs.WorkItemHandlers; + +public class GenerateSampleEventsWorkItemHandlerTests(ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Theory] + [InlineData(true, 101, 1, 100, false, BackgroundTaskStatus.Running)] + [InlineData(true, 100, 1, 100, true, BackgroundTaskStatus.Completed)] + [InlineData(false, 101, 1, 100, false, BackgroundTaskStatus.Queued)] + [InlineData(false, 100, 1, 100, true, BackgroundTaskStatus.Queued)] + [InlineData(false, 200, 2, 100, false, BackgroundTaskStatus.Queued)] + [InlineData(false, 1, 2, 1, true, BackgroundTaskStatus.Queued)] + public async Task HandleItemAsync_HostStopsAfterBatch_OnlyCompletesWhenAllWorkFinished( + bool projectScoped, int eventCount, int projectCount, int expectedProcessed, bool expectSuccess, BackgroundTaskStatus expectedTaskStatus) + { + // Arrange + var time = new FakeTimeProvider(); + using var harness = new BackgroundTaskTestHarness(time, Log); + using var services = new ServiceCollection().BuildServiceProvider(); + using var shutdown = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + var organization = new Organization { Id = SampleDataService.TEST_ORG_ID }; + var project = new Project { Id = SampleDataService.TEST_PROJECT_ID, OrganizationId = organization.Id }; + var organizations = new Mock(); + organizations.Setup(repository => repository.GetByIdAsync(organization.Id, It.IsAny())) + .ReturnsAsync(organization); + var projects = new Mock(); + projects.Setup(repository => repository.GetByIdAsync(project.Id, It.IsAny())) + .ReturnsAsync(project); + var hits = Enumerable.Range(0, projectCount) + .Select(index => new Project { Id = index == 0 ? project.Id : SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID, OrganizationId = organization.Id }) + .Select(item => new FindHit(item.Id, item, 0)); + projects.Setup(repository => repository.GetByOrganizationIdAsync(organization.Id, It.IsAny>())) + .ReturnsAsync(new FindResults(hits, projectCount)); + var pipeline = new RecordingEventPipeline(services, Log, shutdown.Cancel); + var inner = new GenerateSampleEventsWorkItemHandler(pipeline, organizations.Object, projects.Object, harness.LockProvider, time, Log); + var handler = new BackgroundTaskTrackingWorkItemHandler(inner, harness.Tasks, time); + var task = new BackgroundTask + { + OrganizationId = organization.Id, + ProjectId = project.Id, + Type = BackgroundTaskTypes.GenerateProjectSampleData, + Description = "Generate sample data", + CreatedByUserId = "537650f3b77efe23a47914f8" + }; + await harness.Tasks.AddAsync(task); + var workItem = new GenerateSampleEventsWorkItem + { + OrganizationId = projectScoped ? organization.Id : null, + ProjectId = projectScoped ? project.Id : null, + TaskId = projectScoped ? task.Id : null, + EventCount = eventCount + }; + var reported = new List(); + var context = new WorkItemContext(workItem, "test-job", EmptyLock.Empty, shutdown.Token, (progress, _) => + { + reported.Add(progress); + return Task.CompletedTask; + }); + + // Act + await handler.HandleItemAsync(context); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Equal(expectedProcessed, pipeline.Processed); + Assert.Equal(expectSuccess, context.Result.IsSuccess); + Assert.Equal(expectSuccess, reported.Contains(100)); + Assert.Equal(expectedTaskStatus, stored!.Status); + Assert.Equal(expectedTaskStatus is BackgroundTaskStatus.Completed, stored.CompletedUtc.HasValue); + } + + private sealed class RecordingEventPipeline(IServiceProvider services, ILoggerFactory loggerFactory, Action afterBatch) + : EventPipeline(services, new AppOptions(), loggerFactory) + { + public int Processed { get; private set; } + + public override Task> RunAsync(ICollection contexts) + { + Processed += contexts.Count; + afterBatch(); + return Task.FromResult(contexts); + } + + protected override IList GetActionTypes() => []; + } +} diff --git a/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs b/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs new file mode 100644 index 0000000000..85cc411d2b --- /dev/null +++ b/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs @@ -0,0 +1,48 @@ +using Exceptionless.Core.Models.BackgroundTasks; +using Foundatio.Xunit; +using Xunit; + +namespace Exceptionless.Tests.Models; + +public class BackgroundTaskTests : TestWithLoggingBase +{ + private static readonly DateTime UtcNow = new(2026, 11, 3, 4, 5, 6, DateTimeKind.Utc); + + public BackgroundTaskTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void Start_CalledTwice_KeepsTheFirstStartTime() + { + // Arrange + var task = new BackgroundTask(); + + // Act + task.Start(UtcNow); + task.Start(UtcNow.AddMinutes(5)); + + // Assert + Assert.Equal(BackgroundTaskStatus.Running, task.Status); + Assert.Equal(UtcNow, task.StartedUtc); + } + + [Theory] + [InlineData(BackgroundTaskStatus.Completed, 100)] + [InlineData(BackgroundTaskStatus.Failed, 40)] + [InlineData(BackgroundTaskStatus.Cancelled, 40)] + public void Finish_AnyOutcome_StopsTheTaskAndOnlyCompletionReachesOneHundredPercent(BackgroundTaskStatus status, int expectedProgress) + { + // Arrange + var task = new BackgroundTask { Progress = 40 }; + task.Start(UtcNow); + + // Act + task.Finish(status, "Done", UtcNow.AddMinutes(1)); + + // Assert + Assert.False(task.IsActive); + Assert.Equal(status, task.Status); + Assert.Equal("Done", task.Message); + Assert.Equal(UtcNow.AddMinutes(1), task.CompletedUtc); + Assert.Equal(expectedProgress, task.Progress); + } +} diff --git a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs new file mode 100644 index 0000000000..77c5eb3606 --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs @@ -0,0 +1,1096 @@ +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Tests.Utility; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Queues; +using Foundatio.Repositories.Utility; +using Foundatio.Serializer; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Services.BackgroundTasks; + +public class BackgroundTaskServiceTests : TestWithLoggingBase, IDisposable +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private const string OtherOrganizationId = "537650f3b77efe23a47914f5"; + private readonly FakeTimeProvider _time = new(new DateTimeOffset(2026, 11, 3, 4, 5, 6, TimeSpan.Zero)); + private readonly BackgroundTaskTestHarness _harness; + + public BackgroundTaskServiceTests(ITestOutputHelper output) : base(output) + { + _harness = new BackgroundTaskTestHarness(_time, Log); + } + + public void Dispose() + { + _harness.Dispose(); + } + + private BackgroundTaskService Tasks => _harness.Tasks; + + [Fact] + public async Task AddAsync_NewTask_StoresItQueuedAndIndexesIt() + { + // Arrange + var task = NewTask(); + + // Act + await Tasks.AddAsync(task); + var stored = await Tasks.GetAsync(task.Id); + var listed = await Tasks.GetByOrganizationAsync(OrganizationId); + var active = await Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.True(ObjectId.TryParse(task.Id, out _)); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Queued, stored.Status); + Assert.Equal(_time.GetUtcNow().UtcDateTime, stored.CreatedUtc); + Assert.Equal(stored.CreatedUtc, stored.UpdatedUtc); + Assert.Equal([task.Id], listed.Select(t => t.Id).ToArray()); + Assert.Equal([task.Id], active.ToArray()); + Assert.Empty(_harness.Publisher.Updates); + } + + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task CreateAndEnqueueAsync_TaskCannotBeStored_DoesNotQueueOrPublish(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, configureCache: options => + { + options.MaxEntrySize = 1; + options.SizeCalculator = value => value is BackgroundTask ? 2 : 1; + options.ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection; + }); + var task = NewTask(); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CreateAndEnqueueAsync(task, + current => new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = task.ProjectId!, TaskId = current.Id })); + var stored = await harness.Tasks.GetAsync(task.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var queued = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.Null(stored); + Assert.Empty(active); + Assert.Equal(0, queued.Enqueued); + Assert.Empty(harness.Publisher.Updates); + } + + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task CreateAndEnqueueAsync_IndexCannotBeStored_DoesNotQueueOrLeaveAnActiveTask(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, configureCache: options => + { + options.MaxEntrySize = 1; + options.SizeCalculator = value => value is IDictionary ? 2 : 1; + options.ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection; + }); + var task = NewTask(); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CreateAndEnqueueAsync(task, + current => new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = task.ProjectId!, TaskId = current.Id })); + var stored = await harness.Tasks.GetAsync(task.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var queued = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Empty(active); + Assert.Equal(0, queued.Enqueued); + } + + [Fact] + public async Task GetByIdsAsync_MissingAndRepeatedIdsAcrossBatches_ReturnsExistingTasksInInputOrder() + { + // Arrange + var tasks = new List(); + for (int index = 0; index <= BackgroundTaskService.ReadBatchSize; index++) + { + var task = NewTask(); + await Tasks.AddAsync(task); + tasks.Add(task); + } + + string[] requested = [.. tasks.Select(task => task.Id).Reverse(), ObjectId.GenerateNewId().ToString(), tasks[0].Id]; + + // Act + var found = await Tasks.GetByIdsAsync(requested); + + // Assert + Assert.Equal(tasks.Select(task => task.Id).Reverse(), found.Select(task => task.Id)); + } + + [Fact] + public async Task GetByOrganizationAsync_FinishedTasks_ReturnsTheNewestFirstUpToTheLimit() + { + // Arrange + var ids = new List(); + for (int index = 0; index < 4; index++) + { + var task = NewTask(); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); + ids.Add(task.Id); + _time.Advance(TimeSpan.FromSeconds(5)); + } + + // Act + var newestTwo = await Tasks.GetByOrganizationAsync(OrganizationId, limit: 2); + + // Assert + Assert.Equal([ids[3], ids[2]], newestTwo.Select(t => t.Id).ToArray()); + } + + [Fact] + public async Task GetByOrganizationAsync_SameSecondHistoryAcrossBatches_ReturnsNewestCreatedTaskWithoutReadingOlderHistory() + { + // Arrange + RecordingReadCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new RecordingReadCacheClient(inner)); + var utcNow = _time.GetUtcNow().UtcDateTime; + var tasks = new List(); + for (int index = 0; index <= BackgroundTaskService.ReadBatchSize; index++) + { + var task = NewTask(); + // Different servers can create ObjectIds whose machine bytes reverse creation order within one second. + task.Id = new ObjectId(utcNow, machine: index + 1, pid: 1, increment: 0).ToString(); + task.CreatedUtc = utcNow.AddMilliseconds(BackgroundTaskService.ReadBatchSize - index); + task.UpdatedUtc = task.CreatedUtc; + task.Finish(BackgroundTaskStatus.Completed, "Done", task.CreatedUtc); + await harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(task.Id), task, BackgroundTaskService.TaskLifetime); + tasks.Add(task); + } + + var older = NewTask(); + older.Id = ObjectId.GenerateNewId(utcNow.AddSeconds(-1)).ToString(); + older.CreatedUtc = utcNow.AddSeconds(-1); + older.UpdatedUtc = older.CreatedUtc; + older.Finish(BackgroundTaskStatus.Completed, "Done", older.CreatedUtc); + await harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(older.Id), older, BackgroundTaskService.TaskLifetime); + await harness.Cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId), tasks.Select(task => task.Id).Append(older.Id), BackgroundTaskService.TaskLifetime); + + // Act + var listed = await harness.Tasks.GetByOrganizationAsync(OrganizationId, limit: 1); + + // Assert + Assert.Equal(tasks[0].Id, Assert.Single(listed).Id); + Assert.Equal(2, cache!.ReadBatches.Count); + Assert.All(cache.ReadBatches, batch => Assert.InRange(batch.Length, 1, BackgroundTaskService.ReadBatchSize)); + Assert.DoesNotContain(BackgroundTaskService.GetTaskCacheKey(older.Id), cache.ReadBatches.SelectMany(batch => batch)); + } + + [Fact] + public async Task GetByOrganizationAsync_HistoryExceedsFormerPageLimit_ReturnsNewestTask() + { + // Arrange + const int historyCount = 50_001; + var utcNow = _time.GetUtcNow().UtcDateTime; + var firstCreatedUtc = utcNow.AddSeconds(1 - historyCount); + var ids = Enumerable.Range(0, historyCount) + .Select(index => ObjectId.GenerateNewId(firstCreatedUtc.AddSeconds(index)).ToString()) + .ToArray(); + var newest = NewTask(); + newest.Id = ids[^1]; + newest.CreatedUtc = utcNow; + newest.UpdatedUtc = utcNow; + newest.Finish(BackgroundTaskStatus.Completed, "Done", utcNow); + await _harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(newest.Id), newest, BackgroundTaskService.TaskLifetime); + await _harness.Cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId), ids, BackgroundTaskService.TaskLifetime); + + // Act + var listed = await Tasks.GetByOrganizationAsync(OrganizationId, limit: 1); + + // Assert + Assert.Equal(newest.Id, Assert.Single(listed).Id); + } + + [Fact] + public async Task GetByOrganizationAsync_ActiveTasks_DoNotConsumeTheFinishedHistoryLimit() + { + // Arrange + var oldest = NewTask(); + await Tasks.AddAsync(oldest); + var finishedIds = new List(); + for (int index = 0; index <= BackgroundTaskService.ListLimit; index++) + { + _time.Advance(TimeSpan.FromSeconds(1)); + var task = NewTask(); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); + finishedIds.Add(task.Id); + } + + _time.Advance(TimeSpan.FromSeconds(1)); + var newest = NewTask(); + await Tasks.AddAsync(newest); + + // Act + var listed = await Tasks.GetByOrganizationAsync(OrganizationId); + + // Assert + Assert.Equal(BackgroundTaskService.ListLimit + 2, listed.Count); + Assert.Equal(newest.Id, listed[0].Id); + Assert.Equal(oldest.Id, listed[^1].Id); + Assert.Equal(finishedIds.Skip(1).Reverse(), listed.Where(task => !task.IsActive).Select(task => task.Id)); + Assert.Equal(listed.Count, listed.Select(task => task.Id).Distinct().Count()); + } + + [Fact] + public async Task GetByOrganizationAsync_ExpiredAndForeignIds_AreSkippedAndExpiredIdsCleanedFromTheIndex() + { + // Arrange + var kept = NewTask(); + var expired = NewTask(); + var foreign = NewTask(OtherOrganizationId); + await Tasks.AddAsync(kept); + _time.Advance(TimeSpan.FromSeconds(1)); + await Tasks.AddAsync(expired); + _time.Advance(TimeSpan.FromSeconds(1)); + await Tasks.AddAsync(foreign); + await _harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(expired.Id)); + // A stray id from another organization must never be listed. + await _harness.Cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId), [foreign.Id]); + + // Act + var listed = await Tasks.GetByOrganizationAsync(OrganizationId); + var indexed = await _harness.Cache.GetListAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId)); + + // Assert + Assert.Equal([kept.Id], listed.Select(t => t.Id).ToArray()); + Assert.DoesNotContain(expired.Id, indexed.Value); + Assert.Contains(kept.Id, indexed.Value); + } + + [Theory] + [InlineData(OrganizationId, BackgroundTaskStatus.Running)] + [InlineData(OtherOrganizationId, BackgroundTaskStatus.Failed)] + public async Task PruneIndexesAsync_ActiveOrForeignTask_ReadsUnderLockWithoutRemovingIndexes(string taskOrganizationId, BackgroundTaskStatus status) + { + // Arrange + ObservingPruneCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingPruneCacheClient(inner)); + var task = NewTask(taskOrganizationId); + await harness.Tasks.AddAsync(task); + task.Status = status; + await harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(task.Id), task, BackgroundTaskService.TaskLifetime); + await harness.Cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId), [task.Id], BackgroundTaskService.TaskLifetime); + await harness.Cache.ListAddAsync(BackgroundTaskService.GetActiveIndexCacheKey(OrganizationId), [task.Id], BackgroundTaskService.TaskLifetime); + var readsUnderLock = new List(); + cache!.BeforeTaskRead = async () => + { + readsUnderLock.Add(await harness.LockProvider.IsLockedAsync(BackgroundTaskService.GetUpdateLockResource(task.Id))); + }; + + // Act + await harness.Tasks.PruneIndexesAsync(OrganizationId, [task.Id]); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var history = await harness.Cache.GetListAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId)); + + // Assert + Assert.Equal([true], readsUnderLock); + Assert.Equal(0, cache.IndexRemovals); + Assert.Contains(task.Id, active); + Assert.Contains(task.Id, history.Value); + } + + [Fact] + public async Task PruneIndexesAsync_TaskUpdateLockIsBusy_SkipsItUntilALaterRead() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, + new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.Zero }, lockAttempts: 1); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + await harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(task.Id)); + await using (var held = await harness.LockProvider.AcquireAsync(BackgroundTaskService.GetUpdateLockResource(task.Id), TimeSpan.FromSeconds(30), TestCancellationToken)) + { + // Act + await harness.Tasks.PruneIndexesAsync(OrganizationId, [task.Id]).WaitAsync(TestCancellationToken); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var history = await harness.Cache.GetListAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId)); + + // Assert + Assert.Contains(task.Id, active); + Assert.Contains(task.Id, history.Value); + } + + // Act + await harness.Tasks.PruneIndexesAsync(OrganizationId, [task.Id]); + var remainingActive = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var remainingHistory = await harness.Cache.GetListAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId)); + + // Assert + Assert.Empty(remainingActive); + Assert.False(remainingHistory.HasValue && remainingHistory.Value.Contains(task.Id)); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task UpdateAsync_NotificationWaits_PersistsOrderedUpdatesWithoutHoldingTheLock(int clockAdjustmentSeconds) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.Zero }, lockAttempts: 1); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + var publishing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releasePublication = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.BeforePublish = () => + { + harness.Publisher.BeforePublish = null; + publishing.SetResult(); + return releasePublication.Task; + }; + + // Act + var firstUpdate = harness.Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.Progress = 20; + return true; + }); + Exception? exception; + try + { + await publishing.Task.WaitAsync(TestCancellationToken); + _time.AdjustTime(_time.GetUtcNow().AddSeconds(clockAdjustmentSeconds)); + exception = await Record.ExceptionAsync(async () => await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Progress = 40; + return true; + })); + } + finally + { + releasePublication.SetResult(); + await firstUpdate; + } + + var stored = await harness.Tasks.GetAsync(task.Id); + var updates = harness.Publisher.Updates; + + // Assert + Assert.Null(exception); + Assert.NotNull(stored); + Assert.Equal(40, stored.Progress); + Assert.Equal([40, 20], updates.Select(update => update.Progress)); + Assert.Equal(stored.UpdatedUtc, updates[0].UpdatedUtc); + Assert.True(updates[0].UpdatedUtc > updates[1].UpdatedUtc); + } + + [Fact] + public async Task UpdateAsync_UpdateReturnsFalse_SavesAndPublishesNothing() + { + // Arrange + var task = NewTask(); + await Tasks.AddAsync(task); + _time.Advance(TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.UpdateAsync(task.Id, current => + { + current.Progress = 50; + return false; + }); + var stored = await Tasks.GetAsync(task.Id); + + // Assert + Assert.NotNull(result); + Assert.Equal(0, stored!.Progress); + Assert.Equal(stored.Progress, result.Progress); + Assert.Equal(stored.CreatedUtc, stored.UpdatedUtc); + Assert.Empty(_harness.Publisher.Updates); + } + + [Fact] + public async Task UpdateAsync_ChangeThatLeavesTheTaskActive_SavesStampsAndPublishesIt() + { + // Arrange + var task = NewTask(); + await Tasks.AddAsync(task); + _time.Advance(TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.Progress = 40; + current.Message = "Working"; + return true; + }); + var stored = await Tasks.GetAsync(task.Id); + var active = await Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.Equal(40, stored!.Progress); + Assert.Equal(_time.GetUtcNow().UtcDateTime, result!.UpdatedUtc); + Assert.Equal(result.UpdatedUtc, stored.UpdatedUtc); + Assert.Equal([task.Id], active.ToArray()); + var update = Assert.Single(_harness.Publisher.Updates); + Assert.Equal(task.Id, update.Id); + Assert.Equal(OrganizationId, update.OrganizationId); + Assert.Equal(BackgroundTaskStatus.Running, update.Status); + Assert.Equal(40, update.Progress); + Assert.Equal("Working", update.Message); + Assert.Equal(task.CanCancel, update.CanCancel); + } + + [Fact] + public async Task UpdateAsync_LongRunningTask_KeepsIndexesForTheSavedRecordLifetime() + { + // Arrange + ObservingHistoryCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingHistoryCacheClient(inner)); + var tasks = harness.Tasks; + var task = NewTask(); + await tasks.AddAsync(task); + _time.Advance(TimeSpan.FromDays(6)); + + // Act + await tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.Progress = 50; + return true; + }); + _time.Advance(TimeSpan.FromDays(2)); + var stored = await tasks.GetAsync(task.Id); + var active = await tasks.GetActiveIdsAsync(OrganizationId); + var listed = await tasks.GetByOrganizationAsync(OrganizationId); + + // Assert + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Running, stored.Status); + Assert.Equal([task.Id], active); + Assert.Equal(task.Id, Assert.Single(listed).Id); + + // Arrange + IReadOnlyList duringHistoryRefresh = []; + cache!.BeforeHistoryRefresh = async () => duringHistoryRefresh = await tasks.GetByOrganizationAsync(OrganizationId); + + // Act + await tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); + _time.Advance(TimeSpan.FromDays(6)); + var finished = await tasks.GetByOrganizationAsync(OrganizationId); + var finishedActive = await tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.Equal(task.Id, Assert.Single(duringHistoryRefresh).Id); + Assert.Equal(BackgroundTaskStatus.Completed, Assert.Single(finished).Status); + Assert.Empty(finishedActive); + + // Act + _time.Advance(TimeSpan.FromDays(1) + TimeSpan.FromSeconds(1)); + var expired = await tasks.GetAsync(task.Id); + var expiredList = await tasks.GetByOrganizationAsync(OrganizationId); + + // Assert + Assert.Null(expired); + Assert.Empty(expiredList); + } + + [Fact] + public async Task UpdateAsync_TaskFinishes_RemovesItFromTheActiveIndexAndPublishesTheFinalState() + { + // Arrange + var task = NewTask(); + await Tasks.AddAsync(task); + + // Act + await Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); + var active = await Tasks.GetActiveIdsAsync(OrganizationId); + var listed = await Tasks.GetByOrganizationAsync(OrganizationId); + + // Assert + Assert.Empty(active); + Assert.Equal(BackgroundTaskStatus.Completed, Assert.Single(listed).Status); + var update = Assert.Single(_harness.Publisher.Updates); + Assert.Equal(BackgroundTaskStatus.Completed, update.Status); + Assert.Equal(100, update.Progress); + } + + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task UpdateAsync_TaskCannotBeStored_PreservesTheStoredStateAndActiveIndex(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, configureCache: options => + { + options.MaxEntrySize = 1; + options.SizeCalculator = value => value is BackgroundTask { Status: BackgroundTaskStatus.Completed } ? 2 : 1; + options.ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection; + }); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + })); + var stored = await harness.Tasks.GetAsync(task.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.Equal(BackgroundTaskStatus.Queued, stored!.Status); + Assert.Null(stored.CompletedUtc); + Assert.Equal([task.Id], active.ToArray()); + Assert.Empty(harness.Publisher.Updates); + } + + [Fact] + public async Task UpdateAsync_UnknownTask_ReturnsNullWithoutRunningTheUpdate() + { + // Arrange + bool called = false; + + // Act + var result = await Tasks.UpdateAsync(ObjectId.GenerateNewId().ToString(), _ => + { + called = true; + return true; + }); + + // Assert + Assert.Null(result); + Assert.False(called); + Assert.Empty(_harness.Publisher.Updates); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UpdateAsync_PublisherIsDown_StillSavesTheChange(bool transportCancelled) + { + // Arrange + var task = NewTask(); + await Tasks.AddAsync(task); + _harness.Publisher.Failure = transportCancelled + ? new TaskCanceledException("Message bus timed out") + : new TimeoutException("Message bus unavailable"); + + // Act + var result = await Tasks.UpdateAsync(task.Id, current => + { + current.Progress = 10; + return true; + }); + + // Assert + Assert.Equal(10, (await Tasks.GetAsync(task.Id))!.Progress); + Assert.Equal(10, result!.Progress); + } + + [Fact] + public async Task UpdateAsync_LockBusyAtFirst_RetriesWithThePolicyAndSucceeds() + { + // Arrange + var counting = new CountingLockProvider(); + using var harness = new BackgroundTaskTestHarness(TimeProvider.System, Log, decorateLockProvider: counting.Wrap); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + var held = await harness.LockProvider.AcquireAsync(BackgroundTaskService.GetUpdateLockResource(task.Id), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(1)); + counting.Reset(); + counting.OnAttempt = attempts => + { + // Release only after one attempt gave up, so the update can only succeed through a retry. + if (attempts == 2) + held.ReleaseAsync().GetAwaiter().GetResult(); + }; + + // Act + var result = await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Progress = 25; + return true; + }); + + // Assert + Assert.Equal(25, result!.Progress); + Assert.Equal(2, counting.Attempts); + } + + [Fact] + public async Task UpdateAsync_LockNeverReleased_GivesUpAfterTheConfiguredAttempts() + { + // Arrange + var counting = new CountingLockProvider(); + using var harness = new BackgroundTaskTestHarness(TimeProvider.System, Log, decorateLockProvider: counting.Wrap, lockAttempts: 3); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + await using var held = await harness.LockProvider.AcquireAsync(BackgroundTaskService.GetUpdateLockResource(task.Id), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(1)); + counting.Reset(); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.UpdateAsync(task.Id, _ => true)); + + // Assert + Assert.IsType(thrown); + Assert.Equal(3, counting.Attempts); + } + + [Fact] + public async Task CancelAsync_QueuedTask_CancelsImmediatelyAndRemembersTheRequest() + { + // Arrange + var task = NewTask(canCancel: true); + await Tasks.AddAsync(task); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.False(result.IsNotCancellable); + Assert.Equal(BackgroundTaskStatus.Cancelled, result.Task!.Status); + Assert.NotNull(result.Task.CompletedUtc); + Assert.True(await Tasks.IsCancelRequestedAsync(task.Id)); + Assert.Empty(await Tasks.GetActiveIdsAsync(OrganizationId)); + Assert.Equal(BackgroundTaskStatus.Cancelled, Assert.Single(_harness.Publisher.Updates).Status); + } + + [Fact] + public async Task CancelAsync_RunningTaskWithRecentProgress_OnlyRequestsCancellationForTheNextBatch() + { + // Arrange + var task = NewTask(canCancel: true); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + _time.Advance(BackgroundTaskService.StaleAfter - TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Running, result.Task!.Status); + Assert.True(await Tasks.IsCancelRequestedAsync(task.Id)); + Assert.Equal([task.Id], (await Tasks.GetActiveIdsAsync(OrganizationId)).ToArray()); + } + + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task CancelAsync_RequestCannotBeStored_DoesNotAcknowledgeCancellation(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, configureCache: options => + { + options.MaxEntrySize = 1; + options.SizeCalculator = value => value is bool ? 2 : 1; + options.ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection; + }); + var task = NewTask(canCancel: true); + await harness.Tasks.AddAsync(task); + await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + int publishedBefore = harness.Publisher.Updates.Count; + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CancelAsync(task.Id)); + var stored = await harness.Tasks.GetAsync(task.Id); + bool cancellationRequested = await harness.Tasks.IsCancelRequestedAsync(task.Id); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.Equal(BackgroundTaskStatus.Running, stored!.Status); + Assert.False(cancellationRequested); + Assert.Equal(publishedBefore, harness.Publisher.Updates.Count); + } + + [Fact] + public async Task CancelAsync_RunningTaskThatStoppedReportingProgress_CancelsImmediately() + { + // Arrange + var task = NewTask(canCancel: true); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, result.Task!.Status); + Assert.Empty(await Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Fact] + public async Task CancelAsync_TaskThatCannotBeCancelled_IsRejectedAndLeftRunning() + { + // Arrange + var task = NewTask(canCancel: false); + await Tasks.AddAsync(task); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.True(result.IsNotCancellable); + Assert.Equal(BackgroundTaskStatus.Queued, (await Tasks.GetAsync(task.Id))!.Status); + Assert.False(await Tasks.IsCancelRequestedAsync(task.Id)); + Assert.Empty(_harness.Publisher.Updates); + } + + [Fact] + public async Task CancelAsync_AbandonedTaskThatCannotBeCancelled_CanStillBeCleared() + { + // Arrange + var task = NewTask(canCancel: false); + await Tasks.AddAsync(task); + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.False(result.IsNotCancellable); + Assert.Equal(BackgroundTaskStatus.Cancelled, result.Task!.Status); + } + + [Fact] + public async Task CancelAsync_AbandonedRunningTask_SaysItStoppedResponding() + { + // Arrange + var task = NewTask(canCancel: false); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, result.Task!.Status); + Assert.Equal("Cancelled after it stopped responding.", result.Task.Message); + } + + [Fact] + public async Task CancelAsync_FinishedOrUnknownTask_ChangesNothing() + { + // Arrange + var task = NewTask(canCancel: true); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); + int publishedBefore = _harness.Publisher.Updates.Count; + + // Act + var finished = await Tasks.CancelAsync(task.Id); + var unknown = await Tasks.CancelAsync(ObjectId.GenerateNewId().ToString()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, finished.Task!.Status); + Assert.False(await Tasks.IsCancelRequestedAsync(task.Id)); + Assert.Null(unknown.Task); + Assert.Equal(publishedBefore, _harness.Publisher.Updates.Count); + } + + [Fact] + public async Task CreateAndEnqueueAsync_Success_QueuesTheWorkItemWithTheTaskIdAndPublishesQueued() + { + // Arrange + var task = NewTask(); + + // Act + var created = await Tasks.CreateAndEnqueueAsync(task, t => new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = "p1", TaskId = t.Id }); + var entry = await _harness.Queue.DequeueAsync(TestCancellationToken); + + // Assert + Assert.Same(task, created); + var workItem = Assert.IsType(_harness.Queue.Serializer.Deserialize(entry!.Value.Data, typeof(ResetProjectDataWorkItem))); + Assert.Equal(task.Id, workItem.TaskId); + var update = Assert.Single(_harness.Publisher.Updates); + Assert.Equal(BackgroundTaskStatus.Queued, update.Status); + Assert.Equal(task.Id, update.Id); + } + + [Fact] + public async Task CreateAndEnqueueAsync_WorkItemFactoryThrows_FailsTheTaskWithoutQueueingIt() + { + // Arrange + var task = NewTask(); + var failure = new InvalidOperationException("Unable to prepare the work item"); + + // Act + var thrown = await Record.ExceptionAsync(async () => await Tasks.CreateAndEnqueueAsync(task, _ => throw failure)); + var stored = await Tasks.GetAsync(task.Id); + var active = await Tasks.GetActiveIdsAsync(OrganizationId); + var queued = await _harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.Same(failure, thrown); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.NotNull(stored.CompletedUtc); + Assert.Empty(active); + Assert.Equal(0, queued.Enqueued); + Assert.Equal(BackgroundTaskStatus.Failed, Assert.Single(_harness.Publisher.Updates).Status); + } + + [Fact] + public async Task EnqueueAsync_QueueRejectsTheWorkItem_FailsTheTaskSoItIsNotLeftWaiting() + { + // Arrange + var queue = new Mock>(); + queue.SetupGet(q => q.Serializer).Returns(_harness.Queue.Serializer); + queue.Setup(q => q.EnqueueAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new TimeoutException("Queue unavailable")); + using var harness = new BackgroundTaskTestHarness(_time, Log, queue: queue.Object); + var task = NewTask(); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CreateAndEnqueueAsync(task, t => new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = "p1", TaskId = t.Id })); + + // Assert + Assert.IsType(thrown); + var stored = await harness.Tasks.GetAsync(task.Id); + Assert.Equal(BackgroundTaskStatus.Failed, stored!.Status); + Assert.NotNull(stored.CompletedUtc); + Assert.Empty(await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + Assert.Equal(BackgroundTaskStatus.Failed, Assert.Single(harness.Publisher.Updates).Status); + } + + [Theory] + [InlineData(BackgroundTaskStatus.Running, 40, 10)] + [InlineData(BackgroundTaskStatus.Completed, 100, 4)] + [InlineData(BackgroundTaskStatus.Cancelled, 40, 10)] + public async Task EnqueueAsync_AcknowledgementFailsAfterTaskAdvances_PreservesTheStoredState(BackgroundTaskStatus status, int expectedProgress, int expectedTotal) + { + // Arrange + var queue = new LostAcknowledgementQueue(new InMemoryQueueOptions { TimeProvider = _time, LoggerFactory = Log }); + using var harness = new BackgroundTaskTestHarness(_time, Log, queue: queue); + var task = NewTask(canCancel: true); + task.Type = DeleteEventsBulkOperationHandler.OperationType; + task.Total = 10; + task.Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Ids, + Ids = Enumerable.Range(0, 10).Select(_ => ObjectId.GenerateNewId().ToString()).ToList() + } + }; + queue.AfterEnqueue = async () => + { + await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.Progress = 40; + current.Processed = 4; + current.Succeeded = 3; + current.Failed = 1; + current.Bulk!.Cursor = "4"; + current.Message = "Processed four events"; + if (status is BackgroundTaskStatus.Completed) + current.Total = current.Processed; + + if (status is not BackgroundTaskStatus.Running) + current.Finish(status, current.Message, _time.GetUtcNow().UtcDateTime); + + return true; + }); + }; + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CreateAndEnqueueAsync(task, + current => new BulkOperationWorkItem { OrganizationId = OrganizationId, TaskId = current.Id, Type = current.Type })); + var stored = await harness.Tasks.GetAsync(task.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var queued = await queue.GetQueueStatsAsync(); + + // Assert + Assert.IsType(thrown); + Assert.NotNull(stored); + Assert.Equal(status, stored.Status); + Assert.Equal(expectedProgress, stored.Progress); + Assert.Equal(expectedTotal, stored.Total); + Assert.Equal(4, stored.Processed); + Assert.Equal(3, stored.Succeeded); + Assert.Equal(1, stored.Failed); + Assert.Equal("4", stored.Bulk!.Cursor); + Assert.Equal("Processed four events", stored.Message); + Assert.Equal(status is BackgroundTaskStatus.Running, active.Contains(task.Id)); + Assert.Equal(1, queued.Enqueued); + Assert.Equal(status, Assert.Single(harness.Publisher.Updates).Status); + } + + [Fact] + public void CacheKeysAndLockResources_ShareThePrefixAndAreNamedByKind() + { + // Arrange + string[] cacheKeys = + [ + BackgroundTaskService.GetTaskCacheKey("a"), + BackgroundTaskService.GetCancelCacheKey("a"), + BackgroundTaskService.GetOrganizationIndexCacheKey("o"), + BackgroundTaskService.GetActiveIndexCacheKey("o"), + BulkOperationService_GetPinnedSelectionCacheKey("a") + ]; + string[] lockResources = [BackgroundTaskService.GetUpdateLockResource("a"), BackgroundTaskService.GetCreateLockResource("o")]; + + // Act + var all = cacheKeys.Concat(lockResources).ToList(); + + // Assert + Assert.All(all, key => Assert.StartsWith(BackgroundTaskService.KeyPrefix + ":", key)); + Assert.Equal(all.Count, all.Distinct().Count()); + } + + private static string BulkOperationService_GetPinnedSelectionCacheKey(string id) => Exceptionless.Core.Services.BulkOperations.BulkOperationService.GetPinnedSelectionCacheKey(id); + + private static BackgroundTask NewTask(string organizationId = OrganizationId, bool canCancel = false) + { + return new BackgroundTask + { + OrganizationId = organizationId, + ProjectId = "537650f3b77efe23a47914f6", + Type = BackgroundTaskTypes.ResetProjectData, + Description = "Reset data for Test", + CreatedByUserId = "537650f3b77efe23a47914f8", + CanCancel = canCancel + }; + } + + private sealed class ObservingHistoryCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + public Func? BeforeHistoryRefresh { get; set; } + + public new async Task ListAddAsync(string key, IEnumerable values, TimeSpan? expiresIn = null) where T : notnull + { + if (key == BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId) && BeforeHistoryRefresh is { } observe) + { + BeforeHistoryRefresh = null; + await observe(); + } + + return await base.ListAddAsync(key, values, expiresIn); + } + } + + private sealed class ObservingPruneCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + public Func? BeforeTaskRead { get; set; } + public int IndexRemovals { get; private set; } + + async Task> ICacheClient.GetAsync(string key) + { + if (typeof(T) == typeof(BackgroundTask) && BeforeTaskRead is { } observe) + await observe(); + + return await UnscopedCache.GetAsync(key); + } + + Task ICacheClient.ListRemoveAsync(string key, IEnumerable values) + { + IndexRemovals++; + return UnscopedCache.ListRemoveAsync(key, values); + } + } + + private sealed class RecordingReadCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + public List ReadBatches { get; } = []; + + Task>> ICacheClient.GetAllAsync(IEnumerable keys) + { + var batch = keys.ToArray(); + ReadBatches.Add(batch); + return UnscopedCache.GetAllAsync(batch); + } + } + + private sealed class LostAcknowledgementQueue(InMemoryQueueOptions options) : InMemoryQueue(options) + { + public Func? AfterEnqueue { get; set; } + + protected override async Task EnqueueImplAsync(WorkItemData data, QueueEntryOptions options) + { + await base.EnqueueImplAsync(data, options); + if (AfterEnqueue is { } afterEnqueue) + await afterEnqueue(); + + throw new TimeoutException("The work item was queued but its acknowledgement was lost"); + } + } + + private sealed class CountingLockProvider + { + private int _attempts; + + public int Attempts => Volatile.Read(ref _attempts); + + /// Called with the attempt number (since the last reset) as an attempt to acquire a lock starts. + public Action? OnAttempt { get; set; } + + public void Reset() => Volatile.Write(ref _attempts, 0); + + public ILockProvider Wrap(ILockProvider inner) + { + var mock = new Mock(); + mock.Setup(p => p.AcquireAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (resource, expires, releaseOnDispose, cancellationToken) => + { + int attempt = Interlocked.Increment(ref _attempts); + OnAttempt?.Invoke(attempt); + return await inner.AcquireAsync(resource, expires, releaseOnDispose, cancellationToken); + }); + return mock.Object; + } + } +} diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs new file mode 100644 index 0000000000..910feae9d9 --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs @@ -0,0 +1,517 @@ +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Tests.Utility; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Serializer; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class BulkOperationServiceTests : TestWithLoggingBase, IDisposable +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private const string OtherOrganizationId = "537650f3b77efe23a47914f5"; + private const string ProjectId = "537650f3b77efe23a47914f6"; + private const string OtherProjectId = "537650f3b77efe23a47914f7"; + private readonly FakeTimeProvider _time = new(new DateTimeOffset(2026, 11, 3, 4, 5, 6, TimeSpan.Zero)); + private readonly BackgroundTaskTestHarness _harness; + private readonly BulkOperationService _service; + + public BulkOperationServiceTests(ITestOutputHelper output) : base(output) + { + _harness = new BackgroundTaskTestHarness(_time, Log); + _service = CreateService(_harness); + } + + public void Dispose() + { + _harness.Dispose(); + } + + [Fact] + public async Task CreateAsync_IdSelection_CountsDescribesAndQueuesAnUncancelledCancellableTask() + { + // Arrange + var task = NewTask("test.destructive", ProjectId, ids: 3); + + // Act + var result = await _service.CreateAsync(task); + var entry = await _harness.Queue.DequeueAsync(TestCancellationToken); + + // Assert + Assert.Same(task, result.Task); + Assert.Equal(3, task.Total); + Assert.Equal("Record 3", task.Description); + Assert.True(task.CanCancel); + Assert.Equal(BackgroundTaskStatus.Queued, task.Status); + var workItem = Assert.IsType(_harness.Queue.Serializer.Deserialize(entry!.Value.Data, typeof(BulkOperationWorkItem))); + Assert.Equal(task.Id, workItem.TaskId); + Assert.Equal(OrganizationId, workItem.OrganizationId); + Assert.Equal(BackgroundTaskStatus.Queued, Assert.Single(_harness.Publisher.Updates).Status); + } + + [Fact] + public async Task CreateAsync_UnknownTypeOrMissingBulkDetails_Throws() + { + // Arrange + var unknown = NewTask("test.unknown", ProjectId); + var withoutDetails = NewTask("test.destructive", ProjectId); + withoutDetails.Bulk = null; + + // Act + var unknownThrown = await Record.ExceptionAsync(async () => await _service.CreateAsync(unknown)); + var detailsThrown = await Record.ExceptionAsync(async () => await _service.CreateAsync(withoutDetails)); + + // Assert + Assert.IsType(unknownThrown); + Assert.IsType(detailsThrown); + Assert.Empty(await _harness.Tasks.GetByOrganizationAsync(OrganizationId)); + } + + [Fact] + public async Task CreateAsync_DestructiveTaskActiveOnTheSameProject_Conflicts() + { + // Arrange + var first = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Act + var second = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.Null(second.Task); + Assert.Equal(first.Task!.Id, second.Conflict?.Id); + Assert.Single(await _harness.Tasks.GetByOrganizationAsync(OrganizationId)); + Assert.Equal(1, (await _harness.Queue.GetQueueStatsAsync()).Enqueued); + } + + [Fact] + public async Task CreateAsync_TaskCompletesDuringActiveIndexRead_StillFindsTheConflictingTask() + { + // Arrange + ObservingTaskReadsCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingTaskReadsCacheClient(inner)); + var service = CreateService(harness); + var completing = NewTask("test.safe", ProjectId); + await harness.Tasks.AddAsync(completing); + for (int index = 1; index < 500; index++) + await harness.Tasks.AddAsync(NewTask("test.safe", ProjectId)); + + var conflicting = NewTask("test.destructive", ProjectId); + await harness.Tasks.AddAsync(conflicting); + cache!.AfterRead = async () => + { + await harness.Tasks.UpdateAsync(completing.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Completed", _time.GetUtcNow().UtcDateTime); + return true; + }); + }; + + // Act + var result = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.Null(result.Task); + Assert.Equal(conflicting.Id, result.Conflict?.Id); + Assert.Equal(0, (await harness.Queue.GetQueueStatsAsync()).Enqueued); + Assert.Equal(BackgroundTaskStatus.Completed, (await harness.Tasks.GetAsync(completing.Id))!.Status); + } + + [Fact] + public async Task CreateAsync_OrdinaryTaskRetriesAfterItsFailedSnapshot_KeepsItDiscoverable() + { + // Arrange + ObservingTaskReadsCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingTaskReadsCacheClient(inner)); + var service = CreateService(harness); + var retrying = NewTask(BackgroundTaskTypes.ResetProjectData, ProjectId); + retrying.Bulk = null; + await harness.Tasks.AddAsync(retrying); + await harness.Tasks.UpdateAsync(retrying.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + cache!.AfterRead = async () => + { + await harness.Tasks.UpdateAsync(retrying.Id, current => + { + current.Finish(BackgroundTaskStatus.Failed, "The attempt failed", _time.GetUtcNow().UtcDateTime); + return true; + }); + }; + cache.AfterBatchRead = async () => + { + cache.AfterBatchRead = null; + await harness.Tasks.UpdateAsync(retrying.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.CompletedUtc = null; + return true; + }); + }; + + // Act + var created = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + var stored = await harness.Tasks.GetAsync(retrying.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var listed = await harness.Tasks.GetByOrganizationAsync(OrganizationId, limit: 0); + + // Assert + Assert.NotNull(created.Task); + Assert.Equal(BackgroundTaskStatus.Running, stored!.Status); + Assert.Contains(retrying.Id, active); + Assert.Contains(listed, task => task.Id == retrying.Id); + } + + [Fact] + public async Task CreateAsync_LongConflictScan_KeepsTheAdmissionLeaseUntilTheTaskIsIndexed() + { + // Arrange + ObservingTaskReadsCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingTaskReadsCacheClient(inner)); + var service = CreateService(harness); + for (int index = 0; index < BackgroundTaskService.ReadBatchSize + 1; index++) + await harness.Tasks.AddAsync(NewTask("test.safe", ProjectId)); + + var heldDuringScan = new List(); + cache!.AfterBatchRead = async () => + { + _time.Advance(TimeSpan.FromSeconds(16)); + heldDuringScan.Add(await harness.LockProvider.IsLockedAsync(BackgroundTaskService.GetCreateLockResource(OrganizationId))); + }; + + // Act + var result = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.NotNull(result.Task); + Assert.Equal(2, heldDuringScan.Count); + Assert.All(heldDuringScan, held => Assert.True(held)); + Assert.Equal(1, (await harness.Queue.GetQueueStatsAsync()).Enqueued); + } + + [Fact] + public async Task CreateAsync_AdmissionReadOutlivesLease_DoesNotAdmitAfterAnotherCreator() + { + // Arrange + ObservingTaskReadsCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingTaskReadsCacheClient(inner)); + var service = CreateService(harness); + BulkOperationCreateResult? competing = null; + cache!.AfterRead = async () => + { + _time.Advance(TimeSpan.FromSeconds(31)); + competing = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + }; + + // Act + var expired = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.NotNull(competing); + Assert.NotNull(competing.Task); + Assert.True(expired.IsBusy); + Assert.Null(expired.Task); + Assert.Equal([competing.Task.Id], active); + Assert.Equal(1, (await harness.Queue.GetQueueStatsAsync()).Enqueued); + } + + [Theory] + [InlineData(null, ProjectId)] + [InlineData(ProjectId, null)] + [InlineData(null, null)] + public async Task CreateAsync_OrganizationWideAndProjectScopedDestructiveTasks_Conflict(string? activeProjectId, string? candidateProjectId) + { + // Arrange + var active = await _service.CreateAsync(NewTask("test.destructive", activeProjectId)); + + // Act + var candidate = await _service.CreateAsync(NewTask("test.destructive", candidateProjectId)); + + // Assert + Assert.Null(candidate.Task); + Assert.Equal(active.Task!.Id, candidate.Conflict?.Id); + } + + [Fact] + public async Task CreateAsync_DestructiveTaskActiveOnAnotherProjectOrOrganization_DoesNotConflict() + { + // Arrange + await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Act + var otherProject = await _service.CreateAsync(NewTask("test.destructive", OtherProjectId)); + var otherOrganization = await _service.CreateAsync(NewTask("test.destructive", null, OtherOrganizationId)); + + // Assert + Assert.NotNull(otherProject.Task); + Assert.NotNull(otherOrganization.Task); + } + + [Fact] + public async Task CreateAsync_NonDestructiveTasks_NeitherConflictNorBlockDestructiveOnes() + { + // Arrange + var firstSafe = await _service.CreateAsync(NewTask("test.safe", ProjectId)); + + // Act + var secondSafe = await _service.CreateAsync(NewTask("test.safe", ProjectId)); + var destructive = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + var safeAfterDestructive = await _service.CreateAsync(NewTask("test.safe", ProjectId)); + + // Assert + Assert.NotNull(firstSafe.Task); + Assert.NotNull(secondSafe.Task); + Assert.NotNull(destructive.Task); + Assert.NotNull(safeAfterDestructive.Task); + } + + [Fact] + public async Task CreateAsync_ActiveDestructiveTaskStoppedReportingProgress_IsRetiredAndReplaced() + { + // Arrange + var stale = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var replacement = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + var retired = await _harness.Tasks.GetAsync(stale.Task!.Id); + + // Assert + Assert.NotNull(replacement.Task); + Assert.Equal(BackgroundTaskStatus.Failed, retired!.Status); + Assert.NotNull(retired.CompletedUtc); + Assert.Equal([replacement.Task.Id], (await _harness.Tasks.GetActiveIdsAsync(OrganizationId)).ToArray()); + Assert.Contains(_harness.Publisher.Updates, u => u.Id == stale.Task.Id && u.Status == BackgroundTaskStatus.Failed); + } + + [Fact] + public async Task CreateAsync_FinishedTasksStillInTheActiveIndex_AreDroppedFromIt() + { + // Arrange + var finished = NewTask("test.destructive", ProjectId); + var expired = NewTask("test.destructive", ProjectId); + await _harness.Tasks.AddAsync(finished); + await _harness.Tasks.AddAsync(expired); + // Simulate an index that was not cleaned: the task finished without being removed, and another record expired. + finished.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + await _harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(finished.Id), finished, BackgroundTaskService.TaskLifetime); + await _harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(expired.Id)); + + // Act + var created = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + var active = await _harness.Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.NotNull(created.Task); + Assert.Equal([created.Task.Id], active.ToArray()); + } + + [Fact] + public async Task CreateAsync_QueueRejectsTheWorkItem_FailsTheTaskSoItDoesNotBlockTheProject() + { + // Arrange + var queue = new Mock>(); + queue.SetupGet(q => q.Serializer).Returns(_harness.Queue.Serializer); + queue.SetupSequence(q => q.EnqueueAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new TimeoutException("Queue unavailable")) + .ReturnsAsync("queued"); + using var failingHarness = new BackgroundTaskTestHarness(_time, Log, queue: queue.Object); + var service = CreateService(failingHarness); + var task = NewTask("test.destructive", ProjectId); + + // Act + var thrown = await Record.ExceptionAsync(async () => await service.CreateAsync(task)); + var orphan = await failingHarness.Tasks.GetAsync(task.Id); + var activeAfterFailure = await failingHarness.Tasks.GetActiveIdsAsync(OrganizationId); + var retry = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.IsType(thrown); + Assert.Equal(BackgroundTaskStatus.Failed, orphan!.Status); + Assert.Empty(activeAfterFailure); + Assert.NotNull(retry.Task); + Assert.Equal(BackgroundTaskStatus.Queued, retry.Task.Status); + } + + [Fact] + public async Task CreateAsync_ActiveIndexWriteFails_DoesNotLeaveAnUnqueuedTaskBlockingTheProject() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => new FailingActiveIndexCacheClient(inner)); + var service = CreateService(harness); + var task = NewTask("test.destructive", ProjectId); + + // Act + var thrown = await Record.ExceptionAsync(async () => await service.CreateAsync(task)); + var stored = await harness.Tasks.GetAsync(task.Id); + var activeAfterFailure = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + long enqueuedAfterFailure = (await harness.Queue.GetQueueStatsAsync()).Enqueued; + var retry = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.IsType(thrown); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Empty(activeAfterFailure); + Assert.Equal(0, enqueuedAfterFailure); + Assert.NotNull(retry.Task); + } + + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task CreateAsync_SelectionCannotBeStored_FailsWithoutQueueingAndAllowsAnotherOperation(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var selectionCache = new InMemoryCacheClient(new InMemoryCacheClientOptions + { + TimeProvider = _time, + LoggerFactory = Log, + MaxEntrySize = 1, + SizeCalculator = _ => 2, + ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection + }); + var resolver = new Mock(); + resolver.SetupGet(r => r.Entity).Returns(BulkOperationEntity.Events); + resolver.Setup(r => r.PrepareAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new BulkSelectionPreparation(BulkOperationCheckResult.Ok, 1, "{}")); + var service = new BulkOperationService(_harness.Tasks, selectionCache, _harness.LockProvider, + [new StubHandler("test.destructive", isDestructive: true)], [resolver.Object], _time); + var task = NewTask("test.destructive", ProjectId); + task.Bulk!.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter }; + + // Act + var thrown = await Record.ExceptionAsync(async () => await service.CreateAsync(task)); + var failed = await _harness.Tasks.GetAsync(task.Id); + var activeAfterFailure = await _harness.Tasks.GetActiveIdsAsync(OrganizationId); + long enqueuedAfterFailure = (await _harness.Queue.GetQueueStatsAsync()).Enqueued; + var retry = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.NotNull(failed); + Assert.Equal(BackgroundTaskStatus.Failed, failed.Status); + Assert.NotNull(failed.CompletedUtc); + Assert.Empty(activeAfterFailure); + Assert.Equal(0, enqueuedAfterFailure); + Assert.Contains(_harness.Publisher.Updates, update => update.Id == task.Id && update.Status == BackgroundTaskStatus.Failed); + resolver.Verify(r => r.ReleaseAsync(It.Is(saved => saved.Id == task.Id && saved.Status == BackgroundTaskStatus.Failed), "{}", null), Times.Once); + Assert.NotNull(retry.Task); + Assert.Equal(BackgroundTaskStatus.Queued, retry.Task.Status); + } + + [Fact] + public async Task PublishLostAsync_RecordIsGone_PublishesAFinalFailedUpdateFromTheWorkItem() + { + // Arrange + var workItem = new BulkOperationWorkItem { TaskId = Foundatio.Repositories.Utility.ObjectId.GenerateNewId().ToString(), OrganizationId = OrganizationId, Type = "test.destructive", Description = "Record 7" }; + + // Act + await _service.PublishLostAsync(workItem); + + // Assert + var update = Assert.Single(_harness.Publisher.Updates); + Assert.Equal(workItem.TaskId, update.Id); + Assert.Equal(OrganizationId, update.OrganizationId); + Assert.Equal("Record 7", update.Description); + Assert.Equal(BackgroundTaskStatus.Failed, update.Status); + Assert.False(String.IsNullOrEmpty(update.Message)); + } + + private BulkOperationService CreateService(BackgroundTaskTestHarness harness) + { + return new BulkOperationService(harness.Tasks, harness.Cache, harness.LockProvider, + [new StubHandler("test.destructive", isDestructive: true), new StubHandler("test.safe", isDestructive: false)], [], _time); + } + + private static BackgroundTask NewTask(string type, string? projectId, string organizationId = OrganizationId, int ids = 1) + { + return new BackgroundTask + { + OrganizationId = organizationId, + ProjectId = projectId, + Type = type, + CreatedByUserId = "537650f3b77efe23a47914f8", + Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Ids, + Ids = Enumerable.Range(0, ids).Select(_ => Foundatio.Repositories.Utility.ObjectId.GenerateNewId().ToString()).ToList() + } + } + }; + } + + private sealed class StubHandler(string type, bool isDestructive) : IBulkOperationHandler + { + public string Type => type; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => isDestructive; + + public Task AuthorizeAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task ValidateAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public string Describe(BackgroundTask task) => $"Record {task.Total}"; + public Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) => Task.FromResult(new BulkBatchResult(ids.Count, 0)); + } + + private sealed class ObservingTaskReadsCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + public Func? AfterRead { get; set; } + public Func? AfterBatchRead { get; set; } + + public new async Task>> GetAllAsync(IEnumerable keys) + { + var result = await base.GetAllAsync(keys); + if (AfterBatchRead is { } afterRead) + await afterRead(); + + return result; + } + + public new async Task>> GetListAsync(string key, int? page = null, int pageSize = 100) where T : notnull + { + var result = await base.GetListAsync(key, page, pageSize); + if (key == BackgroundTaskService.GetActiveIndexCacheKey(OrganizationId) && AfterRead is { } afterRead) + { + AfterRead = null; + await afterRead(); + } + + return result; + } + } + + private sealed class FailingActiveIndexCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + private bool _failed; + + public new async Task ListAddAsync(string key, IEnumerable values, TimeSpan? expiresIn = null) where T : notnull + { + long added = await base.ListAddAsync(key, values, expiresIn); + if (!_failed && key == BackgroundTaskService.GetActiveIndexCacheKey(OrganizationId)) + { + _failed = true; + throw new TimeoutException("The active index write succeeded but its acknowledgement was lost"); + } + + return added; + } + } +} diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/BulkSelectionResourceTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/BulkSelectionResourceTests.cs new file mode 100644 index 0000000000..32412088ed --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/BulkSelectionResourceTests.cs @@ -0,0 +1,146 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class BulkSelectionResourceTests(ITestOutputHelper output) : TestWithLoggingBase(output) +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private readonly FakeTimeProvider _time = new(); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CreateAsync_ConflictingSelection_ReleasesOnlyTheRejectedSelectionWithoutChangingTheResult(bool cleanupFails) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var resolver = new RecordingResolver { CleanupFails = cleanupFails }; + var service = CreateService(harness, resolver); + var accepted = await service.CreateAsync(NewTask()); + + // Act + var rejected = await service.CreateAsync(NewTask()); + + // Assert + Assert.Null(rejected.Task); + Assert.Equal(accepted.Task!.Id, rejected.Conflict?.Id); + var release = Assert.Single(resolver.Releases); + Assert.Equal("selection-2", release.PinnedState); + Assert.Equal("selection-1", await service.GetPinnedSelectionAsync(accepted.Task.Id)); + Assert.True((await harness.Tasks.GetAsync(accepted.Task.Id))!.IsActive); + } + + [Theory] + [InlineData(BackgroundTaskStatus.Queued, BackgroundTaskStatus.Failed, true)] + [InlineData(BackgroundTaskStatus.Running, BackgroundTaskStatus.Running, false)] + [InlineData(BackgroundTaskStatus.Completed, BackgroundTaskStatus.Completed, true)] + [InlineData(BackgroundTaskStatus.Cancelled, BackgroundTaskStatus.Cancelled, true)] + public async Task CreateAsync_EnqueueAcknowledgementIsLost_UsesSavedStateToReleaseOrRetainTheSelection( + BackgroundTaskStatus acceptedStatus, BackgroundTaskStatus expectedStatus, bool shouldRelease) + { + // Arrange + var queue = new LostAcknowledgementQueue(new InMemoryQueueOptions { TimeProvider = _time, LoggerFactory = Log }); + using var harness = new BackgroundTaskTestHarness(_time, Log, queue: queue); + var resolver = new RecordingResolver(); + var service = CreateService(harness, resolver); + var task = NewTask(); + queue.AfterAccept = async () => + { + await harness.Tasks.UpdateAsync(task.Id, current => + { + if (acceptedStatus is BackgroundTaskStatus.Queued) + return false; + + current.Start(_time.GetUtcNow().UtcDateTime); + current.Bulk!.Cursor = "latest-snapshot-cursor"; + if (acceptedStatus is not BackgroundTaskStatus.Running) + current.Finish(acceptedStatus, "Finished before the enqueue acknowledgement.", _time.GetUtcNow().UtcDateTime); + + return true; + }); + }; + + // Act + var thrown = await Record.ExceptionAsync(async () => await service.CreateAsync(task)); + var saved = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.IsType(thrown); + Assert.NotNull(saved); + Assert.Equal(expectedStatus, saved.Status); + Assert.Equal(shouldRelease ? 1 : 0, resolver.Releases.Count); + Assert.All(resolver.Releases, release => + { + Assert.Equal("selection-1", release.PinnedState); + Assert.Equal(saved.Bulk!.Cursor, release.Cursor); + }); + Assert.Equal("selection-1", await service.GetPinnedSelectionAsync(task.Id)); + Assert.Equal(1, (await queue.GetQueueStatsAsync()).Enqueued); + } + + private BulkOperationService CreateService(BackgroundTaskTestHarness harness, RecordingResolver resolver) => + new(harness.Tasks, harness.Cache, harness.LockProvider, [new DeleteHandler()], [resolver], _time); + + private static BackgroundTask NewTask() => new() + { + OrganizationId = OrganizationId, + ProjectId = "537650f3b77efe23a47914f6", + Type = "test.delete", + Bulk = new BulkOperationDetails { Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter } } + }; + + private sealed class RecordingResolver : IBulkSelectionResolver + { + private int _prepared; + public bool CleanupFails { get; init; } + public List<(string? PinnedState, string? Cursor)> Releases { get; } = []; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection) => Task.FromResult(BulkOperationCheckResult.Ok); + + public Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken) => + Task.FromResult(new BulkSelectionPreparation(BulkOperationCheckResult.Ok, 1, $"selection-{++_prepared}")); + + public Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) => + throw new NotSupportedException("These tests do not execute the queued selection."); + + public Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor) + { + Releases.Add((pinnedState, cursor)); + return CleanupFails ? Task.FromException(new TimeoutException("Snapshot cleanup unavailable")) : Task.CompletedTask; + } + } + + private sealed class DeleteHandler : IBulkOperationHandler + { + public string Type => "test.delete"; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => true; + public Task AuthorizeAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task ValidateAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public string Describe(BackgroundTask task) => "Delete selected events"; + public Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) => + throw new NotSupportedException("These tests do not execute the queued selection."); + } + + private sealed class LostAcknowledgementQueue(InMemoryQueueOptions options) : InMemoryQueue(options) + { + public Func? AfterAccept { get; set; } + + protected override async Task EnqueueImplAsync(WorkItemData data, QueueEntryOptions options) + { + await base.EnqueueImplAsync(data, options); + if (AfterAccept is not null) + await AfterAccept(); + throw new TimeoutException("The queue accepted the work but its acknowledgement was lost."); + } + } +} diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs new file mode 100644 index 0000000000..2917ff98ec --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs @@ -0,0 +1,316 @@ +using System.Globalization; +using System.Text; +using Elastic.Clients.Elasticsearch.Mapping; +using Exceptionless.Core; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Repositories.Options; +using Exceptionless.Core.Services.BulkOperations; +using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; +using Foundatio.Serializer; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class EventBulkSelectionResolverPlanTests : TestWithServices +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private readonly Organization _organization = new() { Id = OrganizationId, HasPremiumFeatures = true }; + private readonly Mock _events = new(); + private readonly Mock _snapshots; + private readonly BulkOperationOptions _options = new() { BatchSize = 1 }; + private readonly EventBulkSelectionResolver _resolver; + + public EventBulkSelectionResolverPlanTests(ITestOutputHelper output) : base(output) + { + var organizations = new Mock(); + organizations.Setup(repository => repository.GetByIdAsync(OrganizationId, It.IsAny>())).ReturnsAsync(_organization); + _snapshots = _events.As(); + _snapshots.Setup(repository => repository.ClosePointInTimeAsync(It.IsAny())).ReturnsAsync(true); + _events.Setup(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Results("prepared", 1, false, "selected")); + var validator = GetService(); + var serializer = GetService(); + var configuration = GetService(); + // These filters use only the configured event fields, so no server mapping is needed. + var mapping = new TypeMappingDescriptor(); + configuration.Events.ConfigureIndexMapping(mapping); + configuration.Events.QueryParser.Configuration!.UseMappings(() => mapping, configuration.Client.Infer); + var appOptions = GetService(); + _resolver = new EventBulkSelectionResolver(_events.Object, organizations.Object, Mock.Of(), validator, serializer, configuration, appOptions, _options); + } + + [Fact] + public async Task PrepareAsync_PremiumFeaturesRemovedAfterValidation_RejectsBeforeSearching() + { + // Arrange + var task = NewTask("message:selected"); + + // Act + var accepted = await _resolver.ValidateSelectionAsync(_organization, task.Bulk!.Selection); + _organization.HasPremiumFeatures = false; + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Assert + Assert.True(accepted.IsOk); + Assert.Equal(BulkOperationCheckStatus.PlanLimit, preparation.Check.Status); + Assert.Equal("Please upgrade your plan to use premium search features.", preparation.Check.Message); + Assert.Null(preparation.PinnedState); + _events.VerifyNoOtherCalls(); + } + + [Fact] + public async Task GetPageAsync_PremiumFeaturesRemovedAfterPreparation_RejectsBeforeSearching() + { + // Arrange + var task = NewTask("message:selected"); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + _organization.HasPremiumFeatures = false; + + // Act + var thrown = await Record.ExceptionAsync(async () => await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken)); + + // Assert + Assert.True(preparation.Check.IsOk); + Assert.Equal(1, preparation.Total); + var rejection = Assert.IsType(thrown); + Assert.Equal(BulkOperationCheckStatus.PlanLimit, rejection.Check.Status); + Assert.Equal("Please upgrade your plan to use premium search features.", rejection.Message); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Once); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("prepared"), Times.Once); + _events.VerifyNoOtherCalls(); + } + + [Fact] + public async Task GetPageAsync_PremiumFeaturesRemovedForFreeFilter_StillSearches() + { + // Arrange + var task = NewTask("type:log"); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + _organization.HasPremiumFeatures = false; + + // Act + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); + + // Assert + Assert.True(preparation.Check.IsOk); + Assert.Equal(1, preparation.Total); + Assert.Equal(["selected"], page.Ids); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PrepareAsync_EmptySelection_RemainsEmptyWhenStoredAndResumed(bool organizationMissing) + { + // Arrange + var task = NewTask("type:log"); + if (organizationMissing) + task.OrganizationId = "537650f3b77efe23a47914f5"; + _events.SetupSequence(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Results("prepared", 0, false)) + .ReturnsAsync(Results("later", 1, false, "newly-matching")); + + // Act + // The resolver uses the configured application serializer for the saved state, just as queued tasks do. + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); + await _resolver.ReleaseAsync(task, preparation.PinnedState, page.NextCursor); + + // Assert + Assert.True(preparation.Check.IsOk); + Assert.Equal(0, preparation.Total); + Assert.NotNull(preparation.PinnedState); + Assert.Empty(page.Ids); + Assert.Null(page.NextCursor); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(organizationMissing ? 0 : 1)); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("prepared"), Times.Exactly(organizationMissing ? 0 : 1)); + } + + [Theory] + [InlineData(null)] + [InlineData("{}")] + [InlineData("{\"event_filter\":\"type:log\",\"stack_ids\":[\"stack\"]}")] + [InlineData("{\"version\":1,\"total\":0}")] + public async Task GetPageAsync_MissingOrLegacySelection_FailsWithoutSearchingLiveEvents(string? pinned) + { + // Arrange + var task = NewTask("type:log"); + + // Act + var thrown = await Record.ExceptionAsync(async () => await _resolver.GetPageAsync(task, pinned, null, 1, TestCancellationToken)); + + // Assert + var rejection = Assert.IsType(thrown); + Assert.Contains("start the operation again", rejection.Message); + _events.VerifyNoOtherCalls(); + } + + [Fact] + public async Task GetPageAsync_ExpiredSnapshot_FailsWithoutOpeningAnotherSnapshot() + { + // Arrange + var task = NewTask("type:log"); + _events.SetupSequence(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Results("prepared", 2, true, "first")) + // Foundatio returns an empty result without a PIT id for a 404. + .ReturnsAsync(new FindResults()); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var thrown = await Record.ExceptionAsync(async () => await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken)); + + // Assert + Assert.IsType(thrown); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.Is(options => options.GetPointInTimeId() == "prepared")), Times.Once); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task GetPageAsync_RepositoryCapsPageSize_ContinuesWithTheLatestSnapshotAndResumesAfterTheFinalCheckpoint() + { + // Arrange + var task = NewTask("message:selected"); + _events.SetupSequence(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Results("prepared", 3, true, "first")) + .ReturnsAsync(Results("renewed", 0, true, "first")) + .ReturnsAsync(Results("latest", 0, false, "second", "third")); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + // Simulate a repository returning a full capped page even though the requested size is larger. + var first = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 20000, TestCancellationToken); + var last = await _resolver.GetPageAsync(task, preparation.PinnedState, first.NextCursor, 20000, TestCancellationToken); + _organization.HasPremiumFeatures = false; + var resumed = await _resolver.GetPageAsync(task, preparation.PinnedState, last.NextCursor, 20000, TestCancellationToken); + await _resolver.ReleaseAsync(task, preparation.PinnedState, last.NextCursor); + + // Assert + Assert.Equal(["first"], first.Ids); + Assert.Equal(["second", "third"], last.Ids); + Assert.Empty(resumed.Ids); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.Is(options => options.GetPointInTimeId() == "renewed" && !options.ShouldTrackTotalHits())), Times.Once); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(3)); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("latest"), Times.Once); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("prepared"), Times.Never); + } + + [Fact] + public async Task GetPageAsync_SmallSelectionHasAnUnavailablePrefix_ReturnsLaterPinnedIdsAndClosesItsSnapshotImmediately() + { + // Arrange + _options.BatchSize = 3; + var task = NewTask("type:log"); + _events.SetupSequence(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Results("prepared", 3, true, "first", "second", "third")) + .ReturnsAsync(Results(null, 0, false, "third")); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); + var exhausted = await _resolver.GetPageAsync(task, preparation.PinnedState, page.NextCursor, 1, TestCancellationToken); + + // Assert + Assert.Equal(["third"], page.Ids); + Assert.Empty(exhausted.Ids); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("prepared"), Times.Once); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + + [Fact] + public async Task GetPageAsync_RetentionIncreases_KeepsTheInitialBoundsAndCurrentPolicyWithAnInvariantDateAnchor() + { + // Arrange + var utcNow = new DateTimeOffset(2026, 11, 13, 11, 30, 0, TimeSpan.Zero); + TimeProvider.SetUtcNow(utcNow); + _organization.CreatedUtc = utcNow.UtcDateTime.AddDays(-30); + _organization.RetentionDays = 1; + var task = NewTask("type:log"); + task.Bulk!.Selection.UtcStart = utcNow.UtcDateTime.AddDays(-7); + var configuration = GetService(); + var filters = new List(); + var retentionPolicies = new List(); + _events.Setup(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .Returns((IRepositoryQuery query, ICommandOptions _) => + { + using var stream = new MemoryStream(); + configuration.Client.RequestResponseSerializer.Serialize(query.GetElasticFilters(), stream); + filters.Add(Encoding.UTF8.GetString(stream.ToArray())); + retentionPolicies.Add(query.GetAppFilter()!.Organizations.Single().RetentionDays); + return Task.FromResult(Results(filters.Count == 1 ? "prepared" : "renewed", 2, true, "selected")); + }); + var originalCulture = CultureInfo.CurrentCulture; + BulkSelectionPreparation? preparation = null; + string? cursor = null; + try + { + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("th-TH"); + + // Act + preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + _organization.RetentionDays = 30; + TimeProvider.Advance(TimeSpan.FromHours(1)); + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); + cursor = page.NextCursor; + + // Assert + Assert.Equal(["selected"], page.Ids); + Assert.Equal(2, filters.Count); + Assert.Equal(filters[0], filters[1]); + Assert.Contains("2026-11-13T11:30:00.000Z", filters[0]); + Assert.DoesNotContain("now", filters[0], StringComparison.OrdinalIgnoreCase); + Assert.Equal([1, 30], retentionPolicies); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + if (preparation is not null) + await _resolver.ReleaseAsync(task, preparation.PinnedState, cursor); + } + } + + private static FindResults Results(string? pointInTimeId, long total, bool hasMore, params string[] ids) + { + var data = new Dictionary(); + if (pointInTimeId is not null) + data[ElasticDataKeys.PointInTimeId] = pointInTimeId; + + var hits = ids.Select(id => new FindHit(id, null, 0, + data: new Dictionary { [ElasticDataKeys.Sorts] = new object[] { id } })); + var results = new FindResults(hits, total, data: data); + ((IFindResults)results).HasMore = hasMore; + return results; + } + + private BackgroundTask NewTask(string filter) + { + var now = TimeProvider.GetUtcNow().UtcDateTime; + return new BackgroundTask + { + OrganizationId = OrganizationId, + CreatedUtc = now, + Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Filter, + Filter = filter, + UtcStart = now.AddDays(-1), + UtcEnd = now, + UtcCreatedBefore = now + } + } + }; + } +} diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs new file mode 100644 index 0000000000..be0eb29b5e --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs @@ -0,0 +1,392 @@ +using Exceptionless.Core; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Core.Utility; +using Foundatio.Repositories; +using Foundatio.Serializer; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class EventBulkSelectionResolverTests : IntegrationTestsBase +{ + private readonly EventBulkSelectionResolver _resolver; + private readonly IEventRepository _eventRepository; + + public EventBulkSelectionResolverTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _eventRepository = GetService(); + // Exercise retained snapshots with a small fixture that crosses the actual preparation boundary. + _resolver = CreateResolver(batchSize: 2); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + await GetService().CreateDataAsync(); + } + + [Fact] + public async Task GetPageAsync_MoreEventsThanOnePage_PagesWithTheCursorInDateOrderWithoutGapsOrRepeats() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var task = NewFilterTask(""); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 2); + + // Assert + Assert.Equal(5, preparation.Total); + Assert.Equal([2, 2, 1], pages.Select(p => p.Count).ToArray()); + Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), pages.SelectMany(p => p).ToArray()); + } + + [Fact] + public async Task GetPageAsync_EarlierPagesAreDeletedBetweenPages_ContinuesAfterTheCursorWithoutSkippingAnything() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var task = NewFilterTask(""); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + var ordered = events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(); + + string? cursor = null; + try + { + // Act + var first = await _resolver.GetPageAsync(task, preparation.PinnedState, cursor, 2, TestCancellationToken); + cursor = first.NextCursor; + await _eventRepository.RemoveAsync(events.Where(e => first.Ids.Contains(e.Id)).ToList(), o => o.ImmediateConsistency()); + var second = await _resolver.GetPageAsync(task, preparation.PinnedState, cursor, 2, TestCancellationToken); + cursor = second.NextCursor; + var third = await _resolver.GetPageAsync(task, preparation.PinnedState, cursor, 2, TestCancellationToken); + cursor = third.NextCursor; + + // Assert + Assert.Equal(ordered[..2], first.Ids.ToArray()); + Assert.Equal(ordered[2..4], second.Ids.ToArray()); + Assert.Equal(ordered[4..], third.Ids.ToArray()); + } + finally + { + await _resolver.ReleaseAsync(task, preparation.PinnedState, cursor); + } + } + + [Fact] + public async Task GetPageAsync_EventsStoredAfterTheCutoff_AreNeverSelected() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, early) = await CreateDataAsync(d => d.Event().TestProject().Message("early").Date(now.AddHours(-1)).CreatedDate(now.UtcDateTime)); + var cutoff = now.UtcDateTime.AddMinutes(1); + TimeProvider.Advance(TimeSpan.FromMinutes(5)); + var (_, late) = await CreateDataAsync(d => d.Event().TestProject().Message("late").Date(now.AddHours(-1)).CreatedDate(TimeProvider.GetUtcNow().UtcDateTime)); + var task = NewFilterTask("", createdBefore: cutoff); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 10); + + // Assert + Assert.Equal(1, preparation.Total); + Assert.Equal(early.Select(e => e.Id).ToArray(), pages.SelectMany(p => p).ToArray()); + Assert.DoesNotContain(late.Single().Id, pages.SelectMany(p => p)); + } + + [Fact] + public async Task GetPageAsync_StacksChangeStatusAfterPrepare_StillPagesTheEventsThatWereConfirmed() + { + // Arrange + var (stacks, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("open").Status(StackStatus.Open); + d.Event().TestProject().Message("fixed").Status(StackStatus.Fixed); + }); + var task = NewFilterTask("status:open"); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + var openStack = stacks.Single(s => s.Status == StackStatus.Open); + openStack.Status = StackStatus.Fixed; + await GetService().SaveAsync(openStack, o => o.ImmediateConsistency()); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 10); + + // Assert + Assert.Equal(1, preparation.Total); + Assert.Equal([events.Single(e => e.Message == "open").Id], pages.SelectMany(p => p).ToArray()); + } + + [Theory] + [InlineData(2)] + [InlineData(10)] + public async Task GetPageAsync_SessionValuesChangeBetweenBatches_KeepsOnlyTheOriginallyMatchingEvents(int preparationSize) + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(data => + { + for (int i = 0; i < 3; i++) + data.Event().TestProject().Type(Event.KnownTypes.Session).Date(now.AddMinutes(-i - 1)).Value(90).Message($"selected {i}"); + data.Event().TestProject().Type(Event.KnownTypes.Session).Date(now.AddMinutes(-4)).Value(30).Message("not selected"); + }); + var resolver = CreateResolver(preparationSize); + var task = NewFilterTask("type:session value:[60 TO 120]"); + var preparation = await resolver.PrepareAsync(task, TestCancellationToken); + var expected = events.Where(e => e.Value == 90).OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(); + string? cursor = null; + var actual = new List(); + try + { + // Act + var first = await resolver.GetPageAsync(task, preparation.PinnedState, cursor, 1, TestCancellationToken); + cursor = first.NextCursor; + actual.AddRange(first.Ids); + await _eventRepository.RemoveAsync(events.Where(e => first.Ids.Contains(e.Id)).ToList(), options => options.ImmediateConsistency()); + + var newlyMatching = events.Single(e => e.Message == "not selected"); + var noLongerMatching = events.Single(e => e.Message == "selected 1"); + // Active-session processing changes existing values without changing created_utc. + newlyMatching.UpdateSessionStart(newlyMatching.Date.UtcDateTime.AddSeconds(90)); + noLongerMatching.UpdateSessionStart(noLongerMatching.Date.UtcDateTime.AddSeconds(180)); + await _eventRepository.SaveAsync([newlyMatching, noLongerMatching], options => options.ImmediateConsistency()); + + var second = await resolver.GetPageAsync(task, preparation.PinnedState, cursor, 1, TestCancellationToken); + cursor = second.NextCursor; + actual.AddRange(second.Ids); + var third = await resolver.GetPageAsync(task, preparation.PinnedState, cursor, 1, TestCancellationToken); + cursor = third.NextCursor; + actual.AddRange(third.Ids); + var exhausted = await resolver.GetPageAsync(task, preparation.PinnedState, cursor, 1, TestCancellationToken); + + // Assert + Assert.Equal(3, preparation.Total); + Assert.Equal(90m, newlyMatching.Value); + Assert.Equal(180m, noLongerMatching.Value); + Assert.Equal(expected, actual); + Assert.Empty(exhausted.Ids); + Assert.DoesNotContain(newlyMatching.Id, actual); + Assert.Contains(noLongerMatching.Id, actual); + } + finally + { + await resolver.ReleaseAsync(task, preparation.PinnedState, cursor); + } + } + + [Fact] + public async Task PrepareAsync_NoEventsMatch_ReturnsAnEmptyFixedSelection() + { + // Arrange + await CreateDataAsync(data => data.Event().TestProject().Type(Event.KnownTypes.Log)); + var task = NewFilterTask("type:session"); + + // Act + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); + + // Assert + Assert.True(preparation.Check.IsOk); + Assert.Equal(0, preparation.Total); + Assert.NotNull(preparation.PinnedState); + Assert.Empty(page.Ids); + Assert.Null(page.NextCursor); + } + + [Theory] + [InlineData(1)] + [InlineData(10)] + public async Task GetPageAsync_RetentionShrinksAfterPreparation_ExcludesOlderSelectedEventsAndKeepsRecentOnes(int preparationSize) + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var organizations = GetService(); + var organization = await organizations.GetByIdAsync(SampleDataService.TEST_ORG_ID); + organization!.CreatedUtc = now.UtcDateTime.AddDays(-30); + organization.RetentionDays = 7; + await organizations.SaveAsync(organization, options => options.ImmediateConsistency()); + var (_, events) = await CreateDataAsync(data => + { + data.Event().TestProject().Date(now.AddHours(-1)).Message("recent"); + data.Event().TestProject().Date(now.AddDays(-4)).Message("older"); + }); + var resolver = CreateResolver(preparationSize); + var task = NewFilterTask(""); + var preparation = await resolver.PrepareAsync(task, TestCancellationToken); + + // Act + organization.RetentionDays = 1; + await organizations.SaveAsync(organization, options => options.ImmediateConsistency()); + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 1, resolver); + + // Assert + Assert.Equal(2, preparation.Total); + Assert.Equal([events.Single(e => e.Message == "recent").Id], pages.SelectMany(page => page).ToArray()); + } + + [Fact] + public async Task GetPageAsync_ProjectScopedTask_OnlySelectsThatProjectsEvents() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("own").Date(now.AddHours(-1)); + d.Event().TestProject().Project(SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID).Message("other project").Date(now.AddHours(-1)); + d.Event().FreeProject().Message("other organization").Date(now.AddHours(-1)); + }); + var task = NewFilterTask(""); + task.ProjectId = SampleDataService.TEST_PROJECT_ID; + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 10); + + // Assert + Assert.Equal(1, preparation.Total); + Assert.Equal([events.Single(e => e.Message == "own").Id], pages.SelectMany(p => p).ToArray()); + } + + [Fact] + public async Task PrepareAsync_RelativeDates_ArePinnedToTheCutoff() + { + // Arrange + var task = NewFilterTask("date:[now-2h TO now]"); + + // Act + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Assert + Assert.True(preparation.Check.IsOk); + Assert.NotNull(preparation.PinnedState); + Assert.DoesNotContain("now", preparation.PinnedState, StringComparison.OrdinalIgnoreCase); + Assert.Contains("||-2h", preparation.PinnedState); + } + + [Fact] + public async Task GetPageAsync_RelativeDateTerm_UsesTheSelectionCutoffInsteadOfExecutionTime() + { + // Arrange + var now = TimeProvider.GetUtcNow().UtcDateTime; + var cutoff = now.AddDays(-1); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("confirmed first").Date(cutoff.AddHours(-1)) + .Mutate(e => e.Data = new DataDictionary { ["scheduled"] = cutoff }) + .MutateStack(stack => stack.CreatedUtc = cutoff.AddHours(-2)); + d.Event().TestProject().Message("confirmed second").Date(cutoff.AddHours(-2)) + .Mutate(e => e.Data = new DataDictionary { ["scheduled"] = cutoff.Date.AddHours(12) }) + .MutateStack(stack => stack.CreatedUtc = cutoff.AddHours(-2)); + d.Event().TestProject().Message("different day").Date(cutoff.AddHours(-1)) + .Mutate(e => e.Data = new DataDictionary { ["scheduled"] = now }) + .MutateStack(stack => stack.CreatedUtc = cutoff.AddHours(-2)); + }); + // Use yesterday's selection anchor so the regression does not need to wait for Elasticsearch's clock to advance. + var task = NewFilterTask("data.scheduled:now/d", createdBefore: cutoff); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 1); + + // Assert + Assert.Equal(2, preparation.Total); + Assert.Equal(events.Where(e => e.Message!.StartsWith("confirmed", StringComparison.Ordinal)).OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), + pages.SelectMany(page => page).ToArray()); + } + + [Theory] + [InlineData(SampleDataService.TEST_ORG_ID, "type:log", BulkOperationCheckStatus.Ok)] + [InlineData(SampleDataService.TEST_ORG_ID, "type:(log", BulkOperationCheckStatus.Invalid)] + [InlineData(SampleDataService.FREE_ORG_ID, "tag:premium-tag", BulkOperationCheckStatus.PlanLimit)] + public async Task ValidateSelectionAsync_Filter_IsCheckedForSyntaxAndPlan(string organizationId, string filter, BulkOperationCheckStatus expected) + { + // Arrange + var organization = await GetService().GetByIdAsync(organizationId); + var selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Filter = filter }; + + // Act + var result = await _resolver.ValidateSelectionAsync(organization!, selection); + + // Assert + Assert.Equal(expected, result.Status); + if (expected is BulkOperationCheckStatus.Invalid) + Assert.Equal("selection.filter", result.Field); + } + + private async Task>> GetAllPagesAsync(BackgroundTask task, string? pinnedState, int size, EventBulkSelectionResolver? resolver = null) + { + resolver ??= _resolver; + var pages = new List>(); + string? cursor = null; + try + { + for (int guard = 0; guard < 20; guard++) + { + var page = await resolver.GetPageAsync(task, pinnedState, cursor, size, TestCancellationToken); + cursor = page.NextCursor ?? cursor; + if (page.Ids.Count == 0) + return pages; + + pages.Add(page.Ids.ToList()); + } + + throw new InvalidOperationException("Paging did not finish."); + } + finally + { + await resolver.ReleaseAsync(task, pinnedState, cursor); + } + } + + private EventBulkSelectionResolver CreateResolver(int batchSize) + { + var organizations = GetService(); + var projects = GetService(); + var validator = GetService(); + var serializer = GetService(); + var configuration = GetService(); + var appOptions = GetService(); + return new EventBulkSelectionResolver(_eventRepository, organizations, projects, validator, serializer, configuration, appOptions, + new BulkOperationOptions { BatchSize = batchSize }); + } + + private BackgroundTask NewFilterTask(string filter, DateTime? createdBefore = null) + { + var now = TimeProvider.GetUtcNow().UtcDateTime; + return new BackgroundTask + { + OrganizationId = SampleDataService.TEST_ORG_ID, + Type = "events.delete", + CreatedUtc = now, + CreatedByUserId = "537650f3b77efe23a47914f8", + Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Filter, + Filter = filter, + UtcStart = now.AddDays(-7), + UtcEnd = now, + UtcCreatedBefore = createdBefore ?? now + } + } + }; + } +} diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs new file mode 100644 index 0000000000..558edbd8f6 --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs @@ -0,0 +1,96 @@ +using System.Globalization; +using Exceptionless.Core.Services.BulkOperations; +using Foundatio.Xunit; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class RelativeDateFreezerTests : TestWithLoggingBase +{ + private static readonly DateTime UtcNow = new(2026, 11, 3, 4, 5, 6, 789, DateTimeKind.Utc); + + public RelativeDateFreezerTests(ITestOutputHelper output) : base(output) { } + + // Built-in fields are looked up in the index mapping; here only date and created_utc are dates. + private static ValueTask IsDateFieldAsync(string field) => ValueTask.FromResult(field is "date" or "created_utc"); + + [Theory] + [InlineData("date:>now-1h", "date:>\"2026-11-03T04:05:06.789Z||-1h\"")] + [InlineData("type:error AND date:[now-7d TO now]", "type:error AND date:[\"2026-11-03T04:05:06.789Z||-7d\" TO \"2026-11-03T04:05:06.789Z||\"]")] + [InlineData("date:(>now-2h AND <=now)", "date:(>\"2026-11-03T04:05:06.789Z||-2h\" AND <=\"2026-11-03T04:05:06.789Z||\")")] + [InlineData("data.date:(>=now/d AND =\"2026-11-03T04:05:06.789Z||/d\" AND <\"2026-11-03T04:05:06.789Z||/d+1d\")")] + [InlineData("data.date:[now/d-4d TO now/d+1d}", "idx.date-d:[\"2026-11-03T04:05:06.789Z||/d-4d\" TO \"2026-11-03T04:05:06.789Z||/d+1d\"}")] + [InlineData("date:now/d", "date:\"2026-11-03T04:05:06.789Z||/d\"")] + [InlineData("NOT date:now/d^2", "NOT date:\"2026-11-03T04:05:06.789Z||/d\"^2")] + [InlineData("date:(now/d OR now/d-1d)", "date:(\"2026-11-03T04:05:06.789Z||/d\" OR \"2026-11-03T04:05:06.789Z||/d-1d\")")] + [InlineData("data.date:now/d", "idx.date-d:\"2026-11-03T04:05:06.789Z||/d\"")] + [InlineData("data.date:(now/d OR now/d-1d)", "idx.date-d:(\"2026-11-03T04:05:06.789Z||/d\" OR \"2026-11-03T04:05:06.789Z||/d-1d\")")] + public async Task FreezeAsync_RelativeDatesOnDateFields_AreAnchoredToTheGivenTime(string filter, string expected) + { + // Arrange + var createdUtc = UtcNow; + Func> isDateFieldAsync = IsDateFieldAsync; + + // Act + string? frozen = await RelativeDateFreezer.FreezeAsync(filter, createdUtc, isDateFieldAsync); + + // Assert + Assert.Equal(expected, frozen); + } + + [Fact] + public async Task FreezeAsync_NonGregorianCulture_UsesAnInvariantUtcAnchor() + { + // Arrange + var originalCulture = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("th-TH"); + try + { + // Act + string? frozen = await RelativeDateFreezer.FreezeAsync("date:>now-1h", UtcNow, IsDateFieldAsync); + + // Assert + Assert.Equal("date:>\"2026-11-03T04:05:06.789Z||-1h\"", frozen); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + } + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("type:error")] + [InlineData("message:now")] + [InlineData("environment:now/d")] + [InlineData("environment:(now/d OR now/d-1d)")] + [InlineData("data.name:(hello OR now/d)")] + [InlineData("ref.parent:now/d")] + [InlineData("date:NOW/d")] + [InlineData("date:[NOW-1d TO NOW]")] + [InlineData("date:now/d*")] + [InlineData("date:now/d?")] + [InlineData("date:now/d~1")] + [InlineData("date:/now.*/")] + [InlineData("date:[2026-01-01 TO 2026-02-01]")] + [InlineData("tag:[nowhere TO zebra]")] + [InlineData("environment:[now TO z]")] + [InlineData("environment:(>now AND now-2h AND <=now)")] + [InlineData("data.name:[now TO z]")] + [InlineData("ref.parent:[now TO z]")] + public async Task FreezeAsync_WithoutRelativeDatesOnDateFields_LeavesTheFilterAlone(string? filter) + { + // Arrange + var createdUtc = UtcNow; + Func> isDateFieldAsync = IsDateFieldAsync; + + // Act + string? frozen = await RelativeDateFreezer.FreezeAsync(filter, createdUtc, isDateFieldAsync); + + // Assert + Assert.DoesNotContain("||", frozen ?? String.Empty); + } +} diff --git a/tests/Exceptionless.Tests/Services/EventDeletionServiceTests.cs b/tests/Exceptionless.Tests/Services/EventDeletionServiceTests.cs new file mode 100644 index 0000000000..424d5fb9cf --- /dev/null +++ b/tests/Exceptionless.Tests/Services/EventDeletionServiceTests.cs @@ -0,0 +1,108 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Foundatio.Caching; +using Foundatio.Messaging; +using Foundatio.Repositories; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Services; + +public class EventDeletionServiceTests : TestWithServices +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private const string FirstProjectId = "537650f3b77efe23a47914f6"; + private const string SecondProjectId = "537650f3b77efe23a47914f7"; + private readonly IOrganizationRepository _organizations; + private readonly IProjectRepository _projects; + private readonly IMessagePublisher _publisher; + private readonly NotificationService _notifications; + private readonly ILogger _deletionLogger; + + public EventDeletionServiceTests(ITestOutputHelper output) : base(output) + { + _organizations = GetService(); + _projects = GetService(); + _publisher = GetService(); + _notifications = GetService(); + _deletionLogger = GetService>(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeleteAsync_UsageTransportFails_PreservesDeletionAndRecordsRemainingProjects(bool dependencyCancelled) + { + // Arrange + using var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { TimeProvider = TimeProvider, LoggerFactory = Log }); + var failure = dependencyCancelled + ? (Exception)new TaskCanceledException("Usage transport timed out") + : new TimeoutException("Usage transport timed out"); + var usageCache = new FaultingUsageCache(cache, failure); + var events = new Mock(); + events.Setup(repository => repository.RemoveAsync(It.IsAny>(), It.IsAny())) + .Returns(Task.CompletedTask); + var service = CreateService(events.Object, usageCache); + PersistentEvent[] selected = + [ + new() { Id = "first", OrganizationId = OrganizationId, ProjectId = FirstProjectId }, + new() { Id = "second", OrganizationId = OrganizationId, ProjectId = SecondProjectId } + ]; + + // Act + var thrown = await Record.ExceptionAsync(() => service.DeleteAsync(selected)); + var keys = cache.Keys.Where(key => key.EndsWith($":{OrganizationId}:{SecondProjectId}:deleted", StringComparison.Ordinal)); + var remainingProjectUsage = await cache.GetAllAsync(keys); + + // Assert + Assert.Null(thrown); + events.Verify(repository => repository.RemoveAsync(selected, It.IsAny()), Times.Once); + Assert.True(usageCache.Failed); + var recorded = Assert.Single(remainingProjectUsage).Value; + Assert.True(recorded.HasValue); + Assert.Equal(1, recorded.Value); + } + + [Fact] + public async Task DeleteAsync_DeletionIsCancelled_PropagatesWithoutRecordingUsage() + { + // Arrange + using var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { TimeProvider = TimeProvider, LoggerFactory = Log }); + var failure = new TaskCanceledException("Deletion transport timed out"); + var events = new Mock(); + events.Setup(repository => repository.RemoveAsync(It.IsAny>(), It.IsAny())) + .ThrowsAsync(failure); + var service = CreateService(events.Object, cache); + PersistentEvent[] selected = [new() { Id = "first", OrganizationId = OrganizationId, ProjectId = FirstProjectId }]; + + // Act + var thrown = await Record.ExceptionAsync(() => service.DeleteAsync(selected)); + + // Assert + Assert.Same(failure, thrown); + Assert.Empty(cache.Keys); + } + + private EventDeletionService CreateService(IEventRepository events, ICacheClient cache) + { + var usage = new UsageService(_organizations, _projects, cache, _publisher, _notifications, TimeProvider, Log); + return new EventDeletionService(events, usage, _deletionLogger); + } + + private sealed class FaultingUsageCache(ICacheClient inner, Exception failure) : ScopedCacheClient(inner, null), ICacheClient + { + public bool Failed { get; private set; } + + Task ICacheClient.IncrementAsync(string key, long amount, TimeSpan? expiresIn) + { + if (key.EndsWith($":{FirstProjectId}:deleted", StringComparison.Ordinal)) + { + Failed = true; + return Task.FromException(failure); + } + + return UnscopedCache.IncrementAsync(key, amount, expiresIn); + } + } +} diff --git a/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs new file mode 100644 index 0000000000..37e089d69e --- /dev/null +++ b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs @@ -0,0 +1,86 @@ +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Services.BackgroundTasks; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Tests.Utility; + +/// +/// A on in-memory infrastructure for isolated tests: no Elasticsearch, a recording +/// publisher instead of the message bus, and a quick resilience policy for lock retries. +/// +public sealed class BackgroundTaskTestHarness : IDisposable +{ + public BackgroundTaskTestHarness(TimeProvider timeProvider, ILoggerFactory loggerFactory, BackgroundTaskOptions? options = null, Func? decorateLockProvider = null, + IQueue? queue = null, int lockAttempts = 3, Action? configureCache = null, Func? decorateCache = null) + { + TimeProvider = timeProvider; + var cacheOptions = new InMemoryCacheClientOptions { CloneValues = true, TimeProvider = timeProvider, LoggerFactory = loggerFactory }; + configureCache?.Invoke(cacheOptions); + Cache = new InMemoryCacheClient(cacheOptions); + MessageBus = new InMemoryMessageBus(new InMemoryMessageBusOptions { TimeProvider = timeProvider, LoggerFactory = loggerFactory }); + LockProvider = new CacheLockProvider(Cache, MessageBus, timeProvider, loggerFactory); + if (decorateLockProvider is not null) + LockProvider = decorateLockProvider(LockProvider); + Queue = queue ?? new InMemoryQueue(new InMemoryQueueOptions { TimeProvider = timeProvider, LoggerFactory = loggerFactory }); + Publisher = new RecordingPublisher(); + + var policies = new ResiliencePolicyProvider(timeProvider, loggerFactory) + .WithPolicy(b => b.WithMaxAttempts(lockAttempts).WithDelay(TimeSpan.FromMilliseconds(10))); + var taskCache = decorateCache?.Invoke(Cache) ?? Cache; + Tasks = new BackgroundTaskService(taskCache, LockProvider, Publisher, Queue, policies, options ?? new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.FromMilliseconds(100) }, + timeProvider, loggerFactory.CreateLogger()); + } + + public TimeProvider TimeProvider { get; } + public InMemoryCacheClient Cache { get; } + public InMemoryMessageBus MessageBus { get; } + public ILockProvider LockProvider { get; } + public IQueue Queue { get; } + public RecordingPublisher Publisher { get; } + public BackgroundTaskService Tasks { get; } + + public void Dispose() + { + Queue.Dispose(); + MessageBus.Dispose(); + Cache.Dispose(); + } + + public sealed class RecordingPublisher : IMessagePublisher + { + private readonly List _messages = []; + + /// When set, publishing throws, like a message bus that is down. + public Exception? Failure { get; set; } + + /// Lets a test hold publication while another task update proceeds. + public Func? BeforePublish { get; set; } + + public IReadOnlyList Updates + { + get + { + lock (_messages) + return _messages.OfType().ToList(); + } + } + + public async Task PublishAsync(Type messageType, object message, MessageOptions? options = null, CancellationToken cancellationToken = default) + { + if (Failure is not null) + throw Failure; + + if (BeforePublish is { } beforePublish) + await beforePublish(); + + lock (_messages) + _messages.Add(message); + } + } +}