diff --git a/README.md b/README.md
index 57afbe85fe..30e94dbebe 100644
--- a/README.md
+++ b/README.md
@@ -49,6 +49,8 @@ Notes:
3. Use Aspire CLI 13.6.5 or newer (`aspire update --self` to update). The AppHost uses the CLI bundle for the dashboard and resource orchestrator; when no CLI is installed, the SDK downloads the matching version through DNX.
4. Try `dotnet build -mt` for multithreaded builds. Performance depends on the workload and machine.
+See the [background task developer guide](docs/background-tasks.md) for task lifecycle, API contracts, cancellation, bulk execution, and extension points.
+
## UI Development
The Svelte 5 UI in `src/Exceptionless.Web/ClientApp` is the only application frontend and serves at `/`. Aspire starts Svelte together with the API. Primary hash-route bookmarks resolve to the current pages, and existing signed-in sessions are retained.
diff --git a/benchmarks/BackgroundTasks/BackgroundTasks.csproj b/benchmarks/BackgroundTasks/BackgroundTasks.csproj
new file mode 100644
index 0000000000..64784b27ff
--- /dev/null
+++ b/benchmarks/BackgroundTasks/BackgroundTasks.csproj
@@ -0,0 +1,12 @@
+
+
+ Exe
+ net11.0
+ enable
+ enable
+ true
+
+
+
+
+
diff --git a/benchmarks/BackgroundTasks/Program.cs b/benchmarks/BackgroundTasks/Program.cs
new file mode 100644
index 0000000000..2e3049ba50
--- /dev/null
+++ b/benchmarks/BackgroundTasks/Program.cs
@@ -0,0 +1,176 @@
+using System.Diagnostics;
+using System.Text.Json;
+using Exceptionless.Core.Models.BackgroundTasks;
+using Exceptionless.Core.Models.BulkOperations;
+using Exceptionless.Core.Services.BackgroundTasks;
+using Foundatio.Caching;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Foundatio.Messaging;
+using Foundatio.Queues;
+using Foundatio.Repositories.Utility;
+using Foundatio.Resilience;
+using Microsoft.Extensions.Logging.Abstractions;
+
+// A manual service microbenchmark, deliberately outside the correctness test suite.
+// Compare the same command before and after a change. This does not measure Redis network latency.
+string label = args.FirstOrDefault() ?? "current";
+bool stress = args.Skip(1).Contains("--stress", StringComparer.Ordinal);
+int[] sizes = args.Length > 1 && !stress ? args.Skip(1).Select(Int32.Parse).ToArray() : [100, 1_000, 10_000, 50_001];
+Workload[] workloads = stress
+ ? [new("large-selection-history", 1_000, 2, 1_000), new("many-active-tasks", 1_000, 200, 0), new("same-second-history", 1_000, 2, 1_000, SameSecond: true)]
+ : sizes.Select(size => new Workload("history", size, 2, 0)).ToArray();
+foreach (var workload in workloads)
+{
+ int historyCount = workload.HistoryCount;
+ using var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { CloneValues = true, MaxItems = null });
+ using var messageBus = new InMemoryMessageBus();
+ using var queue = new InMemoryQueue();
+ var measuredCache = new MeasuredCacheClient(cache);
+ var locks = new CacheLockProvider(cache, messageBus);
+ var tasks = new BackgroundTaskService(measuredCache, locks, messageBus, queue,
+ new ResiliencePolicyProvider(), new BackgroundTaskOptions(), TimeProvider.System, NullLogger.Instance);
+
+ const string organizationId = "537650f3b77efe23a47914f4";
+ var createdUtc = DateTime.UtcNow.AddSeconds(-historyCount - workload.ActiveCount);
+ createdUtc = new DateTime(createdUtc.Ticks - createdUtc.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Utc);
+ var ids = new List(historyCount + workload.ActiveCount);
+ var selectionIds = Enumerable.Range(0, workload.SelectionIds).Select(_ => ObjectId.GenerateNewId().ToString()).ToList();
+ for (int i = 0; i < historyCount + workload.ActiveCount; i++)
+ {
+ bool active = i < workload.ActiveCount;
+ var taskCreatedUtc = workload.SameSecond && !active
+ ? createdUtc.AddSeconds(workload.ActiveCount).AddTicks(i - workload.ActiveCount)
+ : createdUtc.AddSeconds(i);
+ var task = new BackgroundTask
+ {
+ // Model servers whose ObjectId machine bytes disagree with precise creation order inside one second.
+ Id = workload.SameSecond && !active
+ ? new ObjectId(taskCreatedUtc, machine: historyCount + workload.ActiveCount - i, pid: 1, increment: 0).ToString()
+ : ObjectId.GenerateNewId(taskCreatedUtc).ToString(),
+ OrganizationId = organizationId,
+ Type = "benchmark.work",
+ Description = "Benchmark task",
+ CreatedByUserId = "537650f3b77efe23a47914f5",
+ CreatedUtc = taskCreatedUtc,
+ UpdatedUtc = taskCreatedUtc,
+ Status = active ? BackgroundTaskStatus.Running : BackgroundTaskStatus.Completed,
+ Progress = active ? 10 : 100,
+ Bulk = selectionIds.Count == 0 ? null : new BulkOperationDetails
+ {
+ Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = selectionIds }
+ }
+ };
+ ids.Add(task.Id);
+ await cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(task.Id), task, BackgroundTaskService.TaskLifetime);
+ }
+
+ // Seed equivalent persisted state directly so index construction is outside the measurement.
+ await cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(organizationId), ids, BackgroundTaskService.TaskLifetime);
+ await cache.ListAddAsync(BackgroundTaskService.GetActiveIndexCacheKey(organizationId), ids.Take(workload.ActiveCount), BackgroundTaskService.TaskLifetime);
+ async Task RunAsync(string operation)
+ {
+ if (operation == "list")
+ {
+ var result = await tasks.GetByOrganizationAsync(organizationId);
+ return result.Count == Math.Min(historyCount, BackgroundTaskService.ListLimit) + workload.ActiveCount
+ && result[0].Id == ids[^1]
+ && result.Count(t => t.IsActive) == workload.ActiveCount;
+ }
+
+ var updated = await tasks.UpdateAsync(ids[0], task =>
+ {
+ task.Progress = operation == "progress" ? 20 : 99;
+ return operation == "progress";
+ });
+ return updated is { Progress: 20, IsActive: true };
+ }
+
+ foreach (string operation in new[] { "list", "progress", "no-op" })
+ {
+ for (int i = 0; i < 3; i++)
+ await RunAsync(operation);
+
+ int iterations = historyCount >= 10_000 ? 7 : 21;
+ var milliseconds = new List(iterations);
+ var allocatedBytes = new List(iterations);
+ bool correct = true;
+ for (int i = 0; i < iterations; i++)
+ {
+ GC.Collect();
+ GC.WaitForPendingFinalizers();
+ GC.Collect();
+ measuredCache.Reset();
+ long beforeAllocation = GC.GetTotalAllocatedBytes(precise: true);
+ long started = Stopwatch.GetTimestamp();
+ correct &= await RunAsync(operation);
+ milliseconds.Add(Stopwatch.GetElapsedTime(started).TotalMilliseconds);
+ allocatedBytes.Add(GC.GetTotalAllocatedBytes(precise: true) - beforeAllocation);
+ }
+
+ milliseconds.Sort();
+ allocatedBytes.Sort();
+ Console.WriteLine(JsonSerializer.Serialize(new
+ {
+ label,
+ workload = workload.Name,
+ operation,
+ historyCount,
+ activeCount = workload.ActiveCount,
+ selectionIds = workload.SelectionIds,
+ iterations,
+ medianMilliseconds = milliseconds[iterations / 2],
+ medianAllocatedBytes = allocatedBytes[iterations / 2],
+ measuredCache.ListReads,
+ measuredCache.SingleReads,
+ measuredCache.BatchReads,
+ measuredCache.ListWrites,
+ measuredCache.ExpirationWrites,
+ correct
+ }));
+ }
+}
+
+sealed record Workload(string Name, int HistoryCount, int ActiveCount, int SelectionIds, bool SameSecond = false);
+
+// Instrument calls while preserving the real cache implementation and its cloning behavior.
+sealed class MeasuredCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient
+{
+ public int ListReads { get; private set; }
+ public int SingleReads { get; private set; }
+ public int BatchReads { get; private set; }
+ public int ListWrites { get; private set; }
+ public int ExpirationWrites { get; private set; }
+
+ public void Reset() => ListReads = SingleReads = BatchReads = ListWrites = ExpirationWrites = 0;
+
+ Task> ICacheClient.GetAsync(string key)
+ {
+ SingleReads++;
+ return UnscopedCache.GetAsync(key);
+ }
+
+ Task>> ICacheClient.GetAllAsync(IEnumerable keys)
+ {
+ BatchReads++;
+ return UnscopedCache.GetAllAsync(keys);
+ }
+
+ Task>> ICacheClient.GetListAsync(string key, int? page, int pageSize)
+ {
+ ListReads++;
+ return UnscopedCache.GetListAsync(key, page, pageSize);
+ }
+
+ Task ICacheClient.ListAddAsync(string key, IEnumerable values, TimeSpan? expiresIn)
+ {
+ ListWrites++;
+ return UnscopedCache.ListAddAsync(key, values, expiresIn);
+ }
+
+ Task ICacheClient.SetExpirationAsync(string key, TimeSpan expiresIn)
+ {
+ ExpirationWrites++;
+ return UnscopedCache.SetExpirationAsync(key, expiresIn);
+ }
+}
diff --git a/benchmarks/BackgroundTasks/README.md b/benchmarks/BackgroundTasks/README.md
new file mode 100644
index 0000000000..fa42f7b29e
--- /dev/null
+++ b/benchmarks/BackgroundTasks/README.md
@@ -0,0 +1,85 @@
+# Background task service benchmark
+
+This manual executable measures task listing, active progress updates, and skipped mutations using the real `BackgroundTaskService` and Foundatio in-memory cache. It is outside the normal test suite and solution. It adds no benchmark package or production dependency.
+
+## Run
+
+From the repository root with the .NET 11 SDK:
+
+```sh
+dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current
+```
+
+The first argument labels the results. Optional following arguments replace the finished-history sizes:
+
+```sh
+dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current 100 1000 10000 50001
+```
+
+The stress profile checks larger task payloads, a larger active index, and creation bursts across servers:
+
+```sh
+dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current --stress
+```
+
+| Stress workload | Finished tasks | Active tasks | Explicit selection IDs per task | Creation order |
+| --- | ---: | ---: | ---: | --- |
+| Large selection history | 1,000 | 2 | 1,000 | Separate seconds |
+| Many active tasks | 1,000 | 200 | 0 | Separate seconds |
+| Same-second history | 1,000 | 2 | 1,000 | All finished tasks in one second, with ID order opposite to precise creation order |
+
+To compare revisions, create a separate worktree at the baseline, copy this folder's `Program.cs` and `.csproj` into the same location there, and run the same command in each checkout. Keep the SDK, configuration, dependencies, and machine identical. Run the measurements sequentially without simultaneous builds or test suites.
+
+## What is measured
+
+- Seed each history size plus the profile's older active tasks. Setup is outside the measurement. Cache values are cloned as in application use; capacity eviction is disabled so it does not change the selected workload.
+- Warm each operation three times, then take 21 samples for smaller histories or seven for histories of 10,000 or more.
+- Report median elapsed time and total managed allocation per call, including asynchronous work. Force collection outside each sample to make retained setup state comparable.
+- Count service-level index reads, individual and batched record reads, index renewals, and expiry updates. The measuring decorator delegates directly to the underlying cache.
+- Check that listing returns the newest finished task, the expected count, and every active task. Check that progress updates retain the running task and requested progress. A skipped mutation deliberately changes its local copy before returning false; the returned result must still describe the stored progress.
+
+Allocation means bytes allocated during an operation, not retained process memory. This is a service microbenchmark, not an Elasticsearch throughput test or a Redis latency forecast. `GetAllAsync` enables batching, but provider behavior and Redis Cluster hash-slot placement determine actual network requests. The default profile uses two active entries; the stress profile measures a larger active index separately.
+
+## Recorded comparison
+
+Measured on Linux x64, .NET SDK 11.0.100, Release, Foundatio 13.0.5. The baseline is [`fc804c08`](https://github.com/exceptionless/Exceptionless/commit/fc804c084506f5f78fc80b11209b49e437423587). The candidate is the service implementation in [`54f7cf26`](https://github.com/exceptionless/Exceptionless/commit/54f7cf264bbeb3f6a79227edd5e1c3972d56bf55), before the same-second ordering and skipped-mutation corrections. [comparison.json](comparison.json) contains every recorded sample summary and the candidate service blob identifier.
+
+| Finished history | Baseline listing | Candidate listing | Baseline allocation | Candidate allocation |
+| ---: | ---: | ---: | ---: | ---: |
+| 100 | 0.1340 ms | 0.1573 ms | 0.050 MB | 0.061 MB |
+| 1,000 | 0.9845 ms | 0.6104 ms | 0.308 MB | 0.188 MB |
+| 10,000 | 16.6519 ms | 5.2518 ms | 10.578 MB | 1.364 MB |
+| 50,001 | 152.1701 ms | 17.0455 ms | 233.967 MB | 6.241 MB |
+
+The 50,001 baseline result is **incorrect**: the old page cap omits the newest task. Its timing is included to expose the old work and failure, not as a comparison between two correct implementations. All candidate result checks passed.
+
+At 10,000 finished tasks, both versions return the correct result; the candidate used about 87% fewer allocated bytes and 68% less elapsed time. The candidate performs two index reads and two batched record reads for these fixtures. The baseline performs 22 index reads and 52 individual record reads at 10,000 tasks, rising to 101 index reads at 50,001.
+
+The small-history tradeoff is visible: at 100 tasks, dictionary batching adds about 12 KB of allocation and 0.023 ms of in-process work. It reduces sequential cache calls, which this local benchmark does not price as remote latency. Avoid adding provider-specific branches or caching layers to optimize that small absolute cost without a representative workload.
+
+### Retention cost
+
+Progress updates now renew the small active index and its key expiry so a task remains discoverable for the lifetime of its saved record. They do not rewrite the history index. With two active tasks, median allocation increased from 6,640 to 9,128 bytes at the first three history sizes: about 2.5 KB per update. At 1,000 history entries, the observed median was 0.0773 ms before and 0.1047 ms after. Submillisecond timings vary with runtime and host scheduling; the extra renewal and expiry calls are the stable change.
+
+Terminal updates renew history before removing active membership. These measurements cover active progress, not terminal-history writes. Index retention is a correctness obligation; report its write cost alongside read improvements instead of claiming that every operation became faster.
+
+## Remaining scaling boundary
+
+The portable cache API has no ordered recent-history query. Listing still reads and sorts all live history IDs, and returning all active tasks requires materializing that active set. Correct ordering across servers also requires reading every candidate in the cutoff second, even when only a few finished records are requested. The service trims finished candidates between bounded read batches, so retained task payloads scale with active tasks, the requested limit, and one batch rather than the entire burst. A future store with ordered discovery can change that access pattern when real volume warrants it; the [task guide](../../docs/background-tasks.md) describes that extraction boundary.
+
+## Correctness and stress comparison
+
+[extended-comparison.json](extended-comparison.json) compares `54f7cf26` with the corrected service, using identical expanded benchmark source. It records both service blob identifiers, the benchmark blob, all 42 summaries, and the same environment and sampling rules. All 21 candidate checks passed. The baseline fails every skipped-mutation return check and the same-second listing check.
+
+| Listing workload | Before | After | Allocation before | Allocation after | Record batches after |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| 1,000 history, each with 1,000 selection IDs | 0.5614 ms | 0.5584 ms | 625,304 B | 623,520 B | 2 |
+| 1,000 history, 200 active tasks | 1.0354 ms | 0.8396 ms | 371,792 B | 363,496 B | 5 |
+| 10,000 history, 2 active tasks | 3.0749 ms | 3.0660 ms | 1,364,320 B | 1,362,536 B | 2 |
+| 1,000 same-second tasks with large selections | 0.7746 ms **incorrect** | 5.0146 ms | 625,640 B **incorrect** | 9,282,976 B | 21 |
+
+Correct same-second ordering costs more reads and decoding allocation because the old version silently discarded newer records. The corrected service retains only 50 finished candidates between batches, with at most another 50 candidates during trimming; it does not retain all 1,000 task payloads. This is an algorithmic bound, not a measured peak-memory claim.
+
+The successful progress path keeps the same cache-call counts and allocation in these fixtures. At 200 active tasks it allocates 26,248 bytes per update, versus 9,208 with two active tasks; the cache's active-index work is a real scaling cost. A skipped mutation adds one read only on that path: 488 bytes for the small record and 8,896 bytes when its selection contains 1,000 IDs. This upholds the stored-result contract without cloning every successful update.
+
+Small-history elapsed times varied in both directions: the 100-task list measured 0.1344 ms before and 0.1722 ms after, while its allocation fell by 1,784 bytes. These results do not support a claim that every operation became faster. The stable benefits are bounded retained history payloads, fewer temporary sorting allocations, unchanged successful-update cache calls, and correct results under creation bursts.
diff --git a/benchmarks/BackgroundTasks/comparison.json b/benchmarks/BackgroundTasks/comparison.json
new file mode 100644
index 0000000000..29bae3a484
--- /dev/null
+++ b/benchmarks/BackgroundTasks/comparison.json
@@ -0,0 +1,238 @@
+{
+ "environment": {
+ "platform": "Linux x64",
+ "sdk": "11.0.100",
+ "configuration": "Release",
+ "foundatio": "13.0.5"
+ },
+ "baseline_commit": "fc804c084506f5f78fc80b11209b49e437423587",
+ "candidate_service_blob": "cd9773fe7feafda0cbabb1e44edf1527b37536f0",
+ "baseline": [
+ {
+ "label": "fc804c08",
+ "operation": "list",
+ "historyCount": 100,
+ "iterations": 21,
+ "medianMilliseconds": 0.134,
+ "medianAllocatedBytes": 49640,
+ "ListReads": 2,
+ "SingleReads": 52,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "fc804c08",
+ "operation": "progress",
+ "historyCount": 100,
+ "iterations": 21,
+ "medianMilliseconds": 0.0717,
+ "medianAllocatedBytes": 6640,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "fc804c08",
+ "operation": "list",
+ "historyCount": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.9845,
+ "medianAllocatedBytes": 308328,
+ "ListReads": 4,
+ "SingleReads": 52,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "fc804c08",
+ "operation": "progress",
+ "historyCount": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.0773,
+ "medianAllocatedBytes": 6640,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "fc804c08",
+ "operation": "list",
+ "historyCount": 10000,
+ "iterations": 7,
+ "medianMilliseconds": 16.6519,
+ "medianAllocatedBytes": 10577920,
+ "ListReads": 22,
+ "SingleReads": 52,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "fc804c08",
+ "operation": "progress",
+ "historyCount": 10000,
+ "iterations": 7,
+ "medianMilliseconds": 0.1462,
+ "medianAllocatedBytes": 6640,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "fc804c08",
+ "operation": "list",
+ "historyCount": 50001,
+ "iterations": 7,
+ "medianMilliseconds": 152.1701,
+ "medianAllocatedBytes": 233967176,
+ "ListReads": 101,
+ "SingleReads": 52,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": false
+ },
+ {
+ "label": "fc804c08",
+ "operation": "progress",
+ "historyCount": 50001,
+ "iterations": 7,
+ "medianMilliseconds": 0.1139,
+ "medianAllocatedBytes": 6584,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ }
+ ],
+ "candidate": [
+ {
+ "label": "candidate",
+ "operation": "list",
+ "historyCount": 100,
+ "iterations": 21,
+ "medianMilliseconds": 0.1573,
+ "medianAllocatedBytes": 61232,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate",
+ "operation": "progress",
+ "historyCount": 100,
+ "iterations": 21,
+ "medianMilliseconds": 0.0643,
+ "medianAllocatedBytes": 9128,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate",
+ "operation": "list",
+ "historyCount": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.6104,
+ "medianAllocatedBytes": 188400,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate",
+ "operation": "progress",
+ "historyCount": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.1047,
+ "medianAllocatedBytes": 9128,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate",
+ "operation": "list",
+ "historyCount": 10000,
+ "iterations": 7,
+ "medianMilliseconds": 5.2518,
+ "medianAllocatedBytes": 1364296,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate",
+ "operation": "progress",
+ "historyCount": 10000,
+ "iterations": 7,
+ "medianMilliseconds": 0.1774,
+ "medianAllocatedBytes": 9128,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate",
+ "operation": "list",
+ "historyCount": 50001,
+ "iterations": 7,
+ "medianMilliseconds": 17.0455,
+ "medianAllocatedBytes": 6241400,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate",
+ "operation": "progress",
+ "historyCount": 50001,
+ "iterations": 7,
+ "medianMilliseconds": 0.1606,
+ "medianAllocatedBytes": 9128,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ }
+ ]
+}
diff --git a/benchmarks/BackgroundTasks/extended-comparison.json b/benchmarks/BackgroundTasks/extended-comparison.json
new file mode 100644
index 0000000000..6450439704
--- /dev/null
+++ b/benchmarks/BackgroundTasks/extended-comparison.json
@@ -0,0 +1,741 @@
+{
+ "recordedUtcDate": "2026-11-14",
+ "environment": {
+ "os": "Linux x64",
+ "sdk": "11.0.100",
+ "configuration": "Release",
+ "foundatio": "13.0.5",
+ "cache": "InMemoryCacheClient",
+ "cloneValues": true,
+ "maxItems": null
+ },
+ "baselineCommit": "54f7cf264bbeb3f6a79227edd5e1c3972d56bf55",
+ "baselineServiceBlob": "cd9773fe7feafda0cbabb1e44edf1527b37536f0",
+ "candidateServiceBlob": "2224e293332b8927e8852a8ddddeb62c9111fe4d",
+ "benchmarkProgramBlob": "2e3049ba50050610ae519bdc5865aefa1b3717de",
+ "method": "Three warmups, then 21 samples per operation (seven for 10,000 or more history entries); median elapsed time and total managed allocated bytes; GC outside each sample; baseline and candidate run sequentially with identical benchmark source and no concurrent test suites or builds.",
+ "limitations": [
+ "Allocation is total bytes allocated per call, not retained or peak memory.",
+ "In-process cache measurements do not forecast Redis latency or Elasticsearch throughput.",
+ "The baseline same-second listing is incorrect; its lower cost omits required candidates.",
+ "Baseline no-op mutation returns the uncommitted callback copy and fails the contract check.",
+ "Timings are host-sensitive; operation counts and allocation expose stable work differences.",
+ "Terminal-history writes and bulk execution are outside this benchmark."
+ ],
+ "measurements": [
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "list",
+ "historyCount": 100,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.1344,
+ "medianAllocatedBytes": 61256,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "progress",
+ "historyCount": 100,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.0616,
+ "medianAllocatedBytes": 9208,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "no-op",
+ "historyCount": 100,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.0404,
+ "medianAllocatedBytes": 5720,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": false
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "list",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.4538,
+ "medianAllocatedBytes": 188424,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "progress",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.0663,
+ "medianAllocatedBytes": 9208,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "no-op",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.0433,
+ "medianAllocatedBytes": 5720,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": false
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "list",
+ "historyCount": 10000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 3.0749,
+ "medianAllocatedBytes": 1364320,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "progress",
+ "historyCount": 10000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 0.1048,
+ "medianAllocatedBytes": 9208,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "no-op",
+ "historyCount": 10000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 0.0474,
+ "medianAllocatedBytes": 5720,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": false
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "list",
+ "historyCount": 50001,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 27.6162,
+ "medianAllocatedBytes": 6241424,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "progress",
+ "historyCount": 50001,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 0.2345,
+ "medianAllocatedBytes": 9208,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "history",
+ "operation": "no-op",
+ "historyCount": 50001,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 0.1602,
+ "medianAllocatedBytes": 5720,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": false
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "large-selection-history",
+ "operation": "list",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.5614,
+ "medianAllocatedBytes": 625304,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "large-selection-history",
+ "operation": "progress",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.0973,
+ "medianAllocatedBytes": 26024,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "large-selection-history",
+ "operation": "no-op",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.0729,
+ "medianAllocatedBytes": 14128,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": false
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "many-active-tasks",
+ "operation": "list",
+ "historyCount": 1000,
+ "activeCount": 200,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 1.0354,
+ "medianAllocatedBytes": 371792,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 5,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "many-active-tasks",
+ "operation": "progress",
+ "historyCount": 1000,
+ "activeCount": 200,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.1029,
+ "medianAllocatedBytes": 26248,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "many-active-tasks",
+ "operation": "no-op",
+ "historyCount": 1000,
+ "activeCount": 200,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.0432,
+ "medianAllocatedBytes": 5720,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": false
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "same-second-history",
+ "operation": "list",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.7746,
+ "medianAllocatedBytes": 625640,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": false
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "same-second-history",
+ "operation": "progress",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.1199,
+ "medianAllocatedBytes": 26024,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "baseline-54f7cf26",
+ "workload": "same-second-history",
+ "operation": "no-op",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.0734,
+ "medianAllocatedBytes": 14128,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": false
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "list",
+ "historyCount": 100,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.1722,
+ "medianAllocatedBytes": 59472,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "progress",
+ "historyCount": 100,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.0456,
+ "medianAllocatedBytes": 9208,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "no-op",
+ "historyCount": 100,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.03,
+ "medianAllocatedBytes": 6208,
+ "ListReads": 0,
+ "SingleReads": 2,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "list",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.6504,
+ "medianAllocatedBytes": 186640,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "progress",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.0692,
+ "medianAllocatedBytes": 9208,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "no-op",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.0418,
+ "medianAllocatedBytes": 6208,
+ "ListReads": 0,
+ "SingleReads": 2,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "list",
+ "historyCount": 10000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 3.066,
+ "medianAllocatedBytes": 1362536,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "progress",
+ "historyCount": 10000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 0.0656,
+ "medianAllocatedBytes": 9208,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "no-op",
+ "historyCount": 10000,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 0.0393,
+ "medianAllocatedBytes": 6208,
+ "ListReads": 0,
+ "SingleReads": 2,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "list",
+ "historyCount": 50001,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 21.9808,
+ "medianAllocatedBytes": 6239640,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "progress",
+ "historyCount": 50001,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 0.234,
+ "medianAllocatedBytes": 9208,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "history",
+ "operation": "no-op",
+ "historyCount": 50001,
+ "activeCount": 2,
+ "selectionIds": 0,
+ "iterations": 7,
+ "medianMilliseconds": 0.1751,
+ "medianAllocatedBytes": 6208,
+ "ListReads": 0,
+ "SingleReads": 2,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "large-selection-history",
+ "operation": "list",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.5584,
+ "medianAllocatedBytes": 623520,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 2,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "large-selection-history",
+ "operation": "progress",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.0947,
+ "medianAllocatedBytes": 26024,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "large-selection-history",
+ "operation": "no-op",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.0528,
+ "medianAllocatedBytes": 23024,
+ "ListReads": 0,
+ "SingleReads": 2,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "many-active-tasks",
+ "operation": "list",
+ "historyCount": 1000,
+ "activeCount": 200,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.8396,
+ "medianAllocatedBytes": 363496,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 5,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "many-active-tasks",
+ "operation": "progress",
+ "historyCount": 1000,
+ "activeCount": 200,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.1004,
+ "medianAllocatedBytes": 26248,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "many-active-tasks",
+ "operation": "no-op",
+ "historyCount": 1000,
+ "activeCount": 200,
+ "selectionIds": 0,
+ "iterations": 21,
+ "medianMilliseconds": 0.0434,
+ "medianAllocatedBytes": 6208,
+ "ListReads": 0,
+ "SingleReads": 2,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "same-second-history",
+ "operation": "list",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 5.0146,
+ "medianAllocatedBytes": 9282976,
+ "ListReads": 2,
+ "SingleReads": 0,
+ "BatchReads": 21,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "same-second-history",
+ "operation": "progress",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.0854,
+ "medianAllocatedBytes": 26024,
+ "ListReads": 0,
+ "SingleReads": 1,
+ "BatchReads": 0,
+ "ListWrites": 1,
+ "ExpirationWrites": 1,
+ "correct": true
+ },
+ {
+ "label": "candidate-2224e293",
+ "workload": "same-second-history",
+ "operation": "no-op",
+ "historyCount": 1000,
+ "activeCount": 2,
+ "selectionIds": 1000,
+ "iterations": 21,
+ "medianMilliseconds": 0.0584,
+ "medianAllocatedBytes": 23024,
+ "ListReads": 0,
+ "SingleReads": 2,
+ "BatchReads": 0,
+ "ListWrites": 0,
+ "ExpirationWrites": 0,
+ "correct": true
+ }
+ ]
+}
diff --git a/docs/_config.ts b/docs/_config.ts
index 20f2af36dc..3667a21c17 100644
--- a/docs/_config.ts
+++ b/docs/_config.ts
@@ -15,6 +15,7 @@ site.ignore(
".gitignore",
".npmrc",
"_site",
+ "background-tasks.md",
"deno.json",
"deno.lock",
"news/index.md",
diff --git a/docs/background-tasks.md b/docs/background-tasks.md
new file mode 100644
index 0000000000..7bcf5d09ff
--- /dev/null
+++ b/docs/background-tasks.md
@@ -0,0 +1,192 @@
+# Background tasks
+
+Background tasks let users follow work performed by `WorkItemJob`: bulk actions, project data resets, and sample-data generation. The task record describes progress; the work queue controls delivery. This guide covers the application contracts and the boundary for a future Foundatio extraction.
+
+## Ownership and persistence
+
+| Component | Owns | Source |
+| --- | --- | --- |
+| `BackgroundTaskService` | Cached records, organization indexes, serialized updates, cancellation requests, progress notifications | [Service](../src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs) |
+| `WorkItemJob` and its queue | Delivery, acknowledgment, work-item retries, execution locks | [Registration](../src/Exceptionless.Core/Bootstrapper.cs) |
+| `BackgroundTaskTrackingWorkItemHandler` | Progress and lifecycle tracking around an existing ordinary work-item handler | [Tracking wrapper](../src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs) |
+| `BulkOperationService` and `BulkOperationWorkItemHandler` | Destructive-operation admission, batch execution, checkpoints, bounded continuation, selection cleanup | [Admission](../src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs), [worker](../src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs) |
+| Selection resolver and action handler | Event membership, current access/retention rules, and the action's domain side effects | [Contracts](../src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs) |
+| HTTP and WebSocket adapters | Authentication, organization access, public projections, client discovery and reconciliation | [API handler](../src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs), [client](../src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts) |
+
+The task record is the source of truth for saved progress. Counts and the bulk cursor share one record and are saved together. The queue payload carries the task identifier and the information required by its handler. Notifications are best effort; polling reads the saved state when a notification is delayed or lost.
+
+Task records use the configured `ICacheClient` and expire seven days after their last save. They provide limited history rather than a durable audit log. Index entries have independent per-member expiration. Every active update renews active membership; a terminal update renews finished-history membership. This keeps long-running work discoverable without rewriting the larger history index on every progress report. The service also refreshes the index key's lifetime because renewing an existing member does not extend the key in every provider.
+
+Index cleanup re-reads suspected stale entries under the same task-update lease. It skips busy tasks, retains tasks that became active, and removes history only when the record is still absent. This avoids waiting behind progress writes or deleting a retry's renewed membership. Readers clean up within their existing batches; normal active reads acquire no additional locks.
+
+Task mutations require isolated snapshots from the cache: serialized providers provide this boundary, and the in-memory provider must use `CloneValues = true`. A callback must retain the task's identity and ownership. A future store adapter must preserve these requirements rather than exposing a shared mutable task instance.
+
+Updates are serialized while their update lease remains valid. Like admission locks, these cache leases are not fencing tokens: a process or store call that outlives its lease can race a newer writer. A future durable store needs conditional revisions or equivalent fencing before promising stronger write isolation.
+
+Record and index writes are separate operations. A cache failure can leave incomplete discovery even when a record was saved; a no-op mutation does not repair every prior index failure. History refresh precedes active removal so normal completion has overlapping membership and a failed refresh preserves the existing fallback. Treat this cache-backed history as best effort, and use a store with stronger indexing guarantees if a future consumer requires transactional discovery.
+
+## Creating and updating a task
+
+Ordinary callers use `CreateAndEnqueueAsync`. Authorization and project validation happen before this call. The factory runs after the task has its identifier, so the work item can reference it:
+
+```csharp
+using Exceptionless.Core.Models.BackgroundTasks;
+using Exceptionless.Core.Models.WorkItems;
+using Exceptionless.Core.Services.BackgroundTasks;
+
+static Task QueueSampleDataAsync(
+ BackgroundTaskService tasks,
+ string organizationId,
+ string projectId,
+ string userId)
+{
+ return tasks.CreateAndEnqueueAsync(new BackgroundTask
+ {
+ OrganizationId = organizationId,
+ ProjectId = projectId,
+ Type = BackgroundTaskTypes.GenerateProjectSampleData,
+ Description = "Generate sample data",
+ CreatedByUserId = userId
+ }, task => new GenerateSampleEventsWorkItem
+ {
+ OrganizationId = organizationId,
+ ProjectId = projectId,
+ TaskId = task.Id,
+ EventCount = 250,
+ DaysBack = 7
+ });
+}
+```
+
+This uses the same work item as [SampleDataService](../src/Exceptionless.Core/Utility/SampleDataService.cs). `CanCancel` defaults to false because this handler cannot stop midway safely. Set it to true only for an implementation that honors cancellation.
+
+| Service method | Contract |
+| --- | --- |
+| `CreateAndEnqueueAsync` | Creates and indexes a queued task, then prepares and enqueues its work item. Factory or queue failures fail a still-queued task. |
+| `GetAsync` | Returns the stored record, or null when absent. Internal callers must enforce their own access policy. |
+| `GetByIdsAsync` | Returns existing tasks once per ID in input order, omitting missing records. Cache reads contain at most 50 records; callers enforce scope. |
+| `GetByOrganizationAsync` | Returns all discoverable active tasks plus up to `limit` finished tasks, newest first; the default finished limit is 50. |
+| `PruneIndexesAsync` | Rechecks suspected stale IDs under their update leases. Removes finished tasks from the active index and missing tasks from both indexes; skips busy, active, or foreign tasks. |
+| `UpdateAsync` | Locks one task, reloads it, runs a synchronous mutation, and saves only when the callback returns true. Returns the stored state, including after a skipped mutation, or null when the record is absent. |
+| `CancelAsync` | Records a cancellation request and returns the current task outcome, including a distinct non-cancellable result. |
+| `IsCancelRequestedAsync` | Lets a cooperative worker check the persisted cancellation request at a safe boundary. |
+
+`AddAsync` followed by `EnqueueAsync` is the lower-level path used by bulk admission, where selection state must be saved before enqueueing. Keep that sequence inside the owning operation service. Raw persistence is private so callers use the locking, indexing, and notification path.
+
+Task storage and queue submission are separate operations. An enqueue exception can occur after the queue accepted the item; cleanup changes only a task that is still queued and preserves an outcome already recorded by the worker. Keep work-item factories free of domain side effects. The returned creation snapshot may already be older than worker progress by the time it reaches the client.
+
+An `UpdateAsync` callback should only inspect and change the supplied task. Perform expensive work outside it, then checkpoint the result. The service owns `UpdatedUtc`: every committed mutation receives a timestamp strictly newer than the previous record, including when a clock repeats or moves backward. The persistence lock is released before publishing; a slow message bus does not hold up another mutation.
+
+Returning false discards the mutation and reloads the stored state for the result. This extra read occurs only on skipped updates; successful progress updates do not clone another task just to support rollback. Advisory publication has no caller cancellation token, so a transport timeout reported as `TaskCanceledException` is logged like other publication failures. It cannot reverse an accepted enqueue or a committed update.
+
+## Lifecycle and retry rules
+
+`IsActive` means `Queued` or `Running`. The worker owns valid transitions for its task type; the service supplies serialized mutations.
+
+| Event | Saved behavior |
+| --- | --- |
+| Accepted submission | `Queued`, with creation and update timestamps |
+| Work starts or reports progress | `Running`; the first start timestamp is retained |
+| Successful completion | `Completed`, completion timestamp, progress 100 |
+| Ordinary handler throws | `Failed` with a generic public message; the queue may retry it |
+| Ordinary queue retries a failed task | `Running` again; completion time, message, and percent are reset, and active membership is restored |
+| Bulk transient error | Keep the task active and continue from its saved cursor; three consecutive transient failures end it as `Failed` |
+| Bulk permanent error, lost selection, or exhausted checkpoint retries | `Failed`; the task does not reopen automatically |
+| Duplicate delivery of completed or cancelled tracked work | Skip execution |
+
+A non-success ordinary work-item result, such as shutdown, leaves tracking active so queue delivery can resume it. Sample generation returns non-success when shutdown leaves a required batch unstarted; shutdown after its final required batch remains successful. Ordinary queue retries can replay earlier side effects. The wrapper has no provider-neutral retry-exhaustion signal; repeated non-success deliveries can therefore exhaust retries while their task remains active. Bulk shutdown also retains its saved progress and selection. An ordinary work item with no `TaskId` runs without tracking. If an ordinary task record disappears, its existing queue payload can still run; a bulk task requires its saved record and selection and stops if either is lost.
+
+`Failed` therefore describes a failed attempt for an ordinary tracked work item and a terminal outcome for a bulk operation. Consumers must allow a newer ordinary-work update to move `Failed` back to `Running`. Retry policy belongs to the queue or bulk worker, not to a status label alone.
+
+The tracking wrapper keeps the initial start decision strict so a storage failure cannot accidentally run completed or cancelled work. Subsequent progress and outcome tracking is best effort. The wrapper forwards the job's original progress callback before tracking I/O, including execution-lock renewal when the handler enables it. A tracking transport failure cannot replace the handler's original exception. Cancellation of the actual work and its original progress callback still propagates normally.
+
+## Cancellation and staleness
+
+Cancellation is cooperative. A queued cancellable task can be marked `Cancelled` immediately. A running bulk task checks the request between batches, finishes the batch already in flight, then records `Cancelled`. Completed side effects are retained.
+
+`StaleAfter` is two hours without a saved update. Staleness is computed when needed; it is not a separate status or a timer that automatically kills work. An abandoned task can be cleared through cancellation even when its type normally cannot be stopped. Bulk admission can retire an abandoned conflicting operation before accepting a replacement.
+
+A worker checks saved activity at the next batch boundary. Retirement therefore stops it from starting another batch even if it missed the admission decision while paused. This cannot undo side effects from a batch that was already in flight.
+
+Cancelling a finished task is idempotent. A successful cancellation request can still return `Running` while the worker finishes its current batch. Continue observing the task until its state changes. The task snapshot does not expose a separate cancellation-pending field.
+
+## HTTP and notification contracts
+
+All routes are under `/api/v2`. Task lookup and cancellation enforce organization access; inaccessible records return the same not-found response as missing records.
+
+| Route | Result |
+| --- | --- |
+| `POST /bulk-operations` | HTTP 202 with the accepted task; conflicting destructive work returns 409 |
+| `GET /tasks/{id}` | One task snapshot; missing or inaccessible tasks return 404 |
+| `GET /organizations/{id}/tasks` | All active tasks plus the newest 50 finished tasks |
+| `POST /tasks/{id}/cancel` | HTTP 202 with the current snapshot; a live non-cancellable task returns 422 |
+
+The HTTP [view model](../src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs) includes status, counts, message, creator, timestamps, and a selection summary for bulk work. That summary can contain the filter, original time expression, resolved UTC bounds, and explicit ID count. It omits the ID array, operation parameters, pinned membership, snapshot identifier, and resume cursor. The smaller [BackgroundTaskUpdated message](../src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs) also omits the selection summary.
+
+Messages can be duplicated, delayed, or delivered out of order. Compare `UpdatedUtc` at .NET tick precision before applying them; JavaScript's `Date` alone discards submillisecond precision. Unknown task identifiers trigger rediscovery, including when the initial list request has not finished. Bursts share the required reconciliation request. A distinct task created after that request's snapshot can be discovered by the next poll.
+
+Known progress updates do not cancel list reads. At the cache commit, the client merges the HTTP result with changes received since the read began: newer task revisions win, newly inserted or changed tasks survive omission from an older response, and unchanged omitted records can disappear. This uses one baseline per in-flight query and releases it after the read; it does not buffer every progress message.
+
+The panel polls the current organization every three seconds while any task is active and once a minute while idle. Its fixed cadence changes only with the organization or active/idle state, so progress revisions cannot continually restart it. A tick skips hidden tabs and an already-running query; returning to an organization also refetches its list. Idle polling covers missed notifications, work started elsewhere, and ordinary queue retries after a failed attempt; it does not guess when a provider has exhausted retries. The idle cost is one task-list request per minute per visible tab. Notification delivery remains an optimization for latency.
+
+Dismissal and completion handling follow the task's outcome revision. A newer terminal outcome can appear and refresh affected data even when the client missed its intervening `Running` state. Repeated copies of the same outcome do not repeat that refresh. If a previously active task disappears from a later list, the panel also refreshes affected data once: the task may have finished outside the 50-task history window or its record may have expired. Omission alone does not establish its final status.
+
+The first successful task list also reconciles affected event and stack queries when it contains finished tasks. This covers completion during delayed initial discovery or missed notifications. Old finished history can cause one additional refresh on first discovery for that organization; later identical polls do not. Reconciliation supersedes reads started before the outcome, including uncached initial reads, and simultaneous completions share the replacement request. This guarantees a new query attempt, while normal Elasticsearch search visibility still applies.
+
+## Bulk selection and execution
+
+Explicit ID selection accepts at most 1,000 distinct IDs. Larger requests use a filter. The resolver captures filter membership during preparation: a complete small selection keeps fixed IDs, while a larger selection keeps an Elasticsearch point-in-time snapshot and search-after cursor. Relative dates use a fixed UTC anchor and invariant formatting.
+
+For filter selections, current organization/project scope, search-plan rules, and retention can narrow the eligible set. Later arrivals or edits cannot expand it. Missing, expired, or legacy selection state fails safely and requires starting a new operation; the worker does not silently re-run a live filter. Empty selections have an explicit persisted representation because the application serializer omits empty collections.
+
+Preparation runs before the organization admission lock. Admission serializes destructive bulk operations that overlap the same project while its lease is valid; an organization-wide operation overlaps every project. Each batch executes at most the configured number of IDs, with a default batch size of 500. Event deletion fetches only identity, daily-index routing, organization, project, and stack fields, then uses the same deletion service as individual deletes.
+
+Admission renews its lease between bounded conflict-read batches, before stale-task retirement, and before indexing the candidate. Monotonic elapsed time detects a scan or renewal that outlived the prior lease; that request returns busy instead of admitting work from an old snapshot. Cache leases are not fencing tokens: a process suspended across lease expiry during the final store writes, or a cache that loses a live lock key, cannot provide a cross-store transactional exclusion guarantee. Handlers must still tolerate replay and already-running side effects.
+
+The snapshot has a two-hour idle lease, renewed while paging. The cached immutable selection also has its own seven-day retention, renewed as the active worker checkpoints so a task older than seven days can still resume. Renewing that expiry does not rewrite the selection payload. Terminal or rejected work releases the snapshot best effort; active retries and shutdown retain it. Expiry bounds the resource when cleanup cannot reach the backend.
+
+The [pinned Foundatio.Repositories implementation](https://github.com/FoundatioFx/Foundatio.Repositories/blob/690e3f7d2c9a81cad2f658e10c29cc609f1e79f7/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReadOnlyRepositoryBase.cs) can open a snapshot and then throw before returning its identifier during preparation. The caller cannot release an identifier it never received; that snapshot remains bounded by its idle lease. Fixing this boundary belongs in the provider's failed-query cleanup, not in a second Elasticsearch access path in the task service.
+
+Completion does not force an Elasticsearch index refresh. Deletion uses the repository's normal eventual search consistency, so search results can briefly retain deleted events until the index refreshes.
+
+Usage accounting after successful deletion is best effort. Its transport failures, including cancellation without a caller token, do not turn a completed deletion into a failed batch or prevent accounting for other projects. A failure of the actual deletion still propagates.
+
+Deletion and checkpoint persistence are separate writes. The worker retries saving a known batch result before considering another execution. A process can still stop after a side effect but before its checkpoint is saved, so handlers must tolerate replay. Missing items are reported as failed; counts are not an exactly-once ledger of external side effects.
+
+## Adding work
+
+For an ordinary tracked job:
+
+1. Implement the existing work-item handler and give its work-item type an optional `TaskId` through `IBackgroundTaskWorkItem`.
+2. Register it through `TrackWithBackgroundTask` in `Bootstrapper`, following reset-data or sample-data generation.
+3. Create it through `CreateAndEnqueueAsync` after authorization, and report progress through `WorkItemContext.ReportProgressAsync` so execution-lock renewal still happens.
+4. Define which side effects may replay, whether cancellation is safe, and which queue failures are retryable.
+
+For a bulk action, implement `IBulkOperationHandler` with a unique operation type, authorization, parameter validation, description, and bounded batch execution. Reuse the existing entity resolver when membership semantics match. A new entity family also needs an `IBulkSelectionResolver` that prepares stable membership, returns opaque resumable cursors, and releases its resources. Keep all domain side effects in the existing domain service.
+
+## Scaling and a future Foundatio boundary
+
+Keep immutable selection data separate from frequently updated progress where the existing format permits it. Process one batch at a time, project only required fields, and use the cache's bulk-read operation for groups of task records. Listing reads each index once, orders the IDs, and loads records in batches of at most 50. It has no silent history-page cutoff. Conflict detection uses the same bounded record reads while checking the active index.
+
+ObjectId order determines the creation second, but IDs from different servers do not determine creation order inside that second. Listing reads the full cutoff second and selects by precise `CreatedUtc`, then ID. It retains only the requested finished history between batches, plus all active tasks. A burst in the cutoff second increases record reads without retaining the burst's entire task payload.
+
+Returning all active tasks inherently requires work proportional to that active set. The generic cache-list API does not provide a portable ordered recent-history query, so listing still reads and orders the live history IDs. `GetAllAsync` lets providers combine reads; it does not promise one network round trip on every provider or Redis Cluster topology. Active-index renewal adds writes to a progress update, while the history index is renewed only on terminal updates. Measure these operations separately when changing retention or storage.
+
+| Reusable responsibility | Keep in the application adapter |
+| --- | --- |
+| Task identity, status, progress, timestamps, and serialized state updates | ObjectId HTTP routes and Exceptionless model interfaces |
+| Store retention, discovery, bounded reads, and cancellation signaling | Organization/project/user authorization and admission policy |
+| Worker tracking and progress reporting using existing Foundatio primitives | Event filters, plan limits, retention policy, Elasticsearch snapshots, deletion accounting |
+| Optional advisory change notifications | Public HTTP/WebSocket projections, UI text, and task-panel behavior |
+
+The current `BackgroundTask` includes organization/project/user fields and `BulkOperationDetails`; it is still an application model. Preserve that honest boundary until another consumer establishes the common contract. An extraction should define identity, owner scope, private checkpoint serialization, retention/index consistency, and retry ownership explicitly, while reusing `ICacheClient`, queues, locks, messaging, resilience, and `TimeProvider`. Introduce a storage abstraction when it is needed to offer indexed discovery or durability that the cache contract cannot provide. Avoid a generic payload bag or a parallel queue/state-machine framework solely to anticipate reuse.
+
+## Verification and operations
+
+Service tests use real Foundatio in-memory infrastructure and controlled time. Regression coverage belongs at the failure boundary: factory/index/queue failures, concurrent updates, delayed notifications, cancellation, retention, full selection paging, and checkpoint recovery. Integration tests exercise real serialization, Elasticsearch routing, authorization, usage accounting, and selection mutation. Browser tests cover discovery, concurrent task completion, and cancellation access.
+
+Provider behavior remains part of this contract. Foundatio 13.0.5's [in-memory expiry cleanup](https://github.com/FoundatioFx/Foundatio/blob/be10785cfed4e1850350dfaa024e1c3f25b7fac7/src/Foundatio/Caching/InMemoryCacheClient.cs) can observe an expired entry and then remove a concurrently renewed replacement by key. The controlled-time retention regression exposed its `Removed item was not expired` diagnostic. This provider race is not repaired by task-service locking; the cache-loss recovery described above still applies. It is distinct from the Redis provider and should be addressed in the in-memory cache implementation before relying on stronger retention guarantees there.
+
+Start with the focused classes under [BackgroundTasks](../tests/Exceptionless.Tests/Services/BackgroundTasks) and [BulkOperations](../tests/Exceptionless.Tests/Services/BulkOperations), then the [worker tests](../tests/Exceptionless.Tests/Jobs/WorkItemHandlers), endpoint tests, and the full CI run. Manual performance measurements live in [benchmarks/BackgroundTasks](../benchmarks/BackgroundTasks); they report service time, allocation, and cache calls separately from provider/network throughput.
+
+Deploy the updated job before or with the web application. Cache loss can remove progress and prepared selections; treat that as an interrupted operation requiring user-visible recovery. This feature adds no database migration or index. Preserve public DTOs and message shapes when changing internal storage or moving reusable components to Foundatio.
diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs
index ee36fdf0d6..4a08e2c6f5 100644
--- a/src/Exceptionless.Core/Bootstrapper.cs
+++ b/src/Exceptionless.Core/Bootstrapper.cs
@@ -26,6 +26,8 @@
using Exceptionless.Core.Seed;
using Exceptionless.Core.Serialization;
using Exceptionless.Core.Services;
+using Exceptionless.Core.Services.BackgroundTasks;
+using Exceptionless.Core.Services.BulkOperations;
using Exceptionless.Core.Services.SourceMaps;
using Exceptionless.Core.Utility;
using Exceptionless.Core.Validation;
@@ -93,6 +95,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
services.AddSingleton(s =>
{
var handlers = new WorkItemHandlers();
+ handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
@@ -102,12 +105,12 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
- handlers.Register(s.GetRequiredService);
+ handlers.Register(() => TrackWithBackgroundTask(s, s.GetRequiredService()));
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
handlers.Register(s.GetRequiredService);
- handlers.Register(s.GetRequiredService);
+ handlers.Register(() => TrackWithBackgroundTask(s, s.GetRequiredService()));
return handlers;
});
@@ -235,6 +238,13 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO
});
services.AddSingleton();
services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddSingleton(typeof(IBulkOperationHandler), typeof(Bootstrapper).Assembly);
+ services.AddSingleton(typeof(IBulkSelectionResolver), typeof(Bootstrapper).Assembly);
services.AddSingleton(provider => provider.GetRequiredService());
services.AddSingleton();
services.AddSingleton();
@@ -530,6 +540,11 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log
logger.LogWarning("Jobs running in process");
}
+ private static IWorkItemHandler TrackWithBackgroundTask(IServiceProvider container, IWorkItemHandler handler)
+ {
+ return new BackgroundTaskTrackingWorkItemHandler(handler, container.GetRequiredService(), container.GetRequiredService());
+ }
+
private static IQueue CreateQueue(IServiceProvider container, TimeSpan? workItemTimeout = null) where T : class
{
var loggerFactory = container.GetRequiredService();
diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs
new file mode 100644
index 0000000000..11185a1e64
--- /dev/null
+++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs
@@ -0,0 +1,170 @@
+using Exceptionless.Core.Models.BackgroundTasks;
+using Exceptionless.Core.Models.WorkItems;
+using Exceptionless.Core.Services.BackgroundTasks;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Foundatio.Queues;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Jobs.WorkItemHandlers;
+
+///
+/// Wraps the handler of a work item that is tracked by a background task. Progress, success and failure are written to the
+/// task in the job process that runs the work. Progress reaches the job's original callback before tracking I/O, preserving
+/// lock renewal when enabled. Work items without a task id pass straight through.
+///
+internal sealed class BackgroundTaskTrackingWorkItemHandler : IWorkItemHandler
+{
+ private readonly IWorkItemHandler _inner;
+ private readonly BackgroundTaskService _taskService;
+ private readonly TimeProvider _timeProvider;
+
+ public BackgroundTaskTrackingWorkItemHandler(IWorkItemHandler inner, BackgroundTaskService taskService, TimeProvider timeProvider)
+ {
+ _inner = inner;
+ _taskService = taskService;
+ _timeProvider = timeProvider;
+ }
+
+ public bool AutoRenewLockOnProgress
+ {
+ get => _inner.AutoRenewLockOnProgress;
+ set => _inner.AutoRenewLockOnProgress = value;
+ }
+
+ public ILogger Log
+ {
+ get => _inner.Log;
+ set => _inner.Log = value;
+ }
+
+ public Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default)
+ {
+ return _inner.GetWorkItemLockAsync(workItem, cancellationToken);
+ }
+
+ public void LogProcessingQueueEntry(IQueueEntry queueEntry, Type workItemDataType, object workItem)
+ {
+ _inner.LogProcessingQueueEntry(queueEntry, workItemDataType, workItem);
+ }
+
+ public void LogAutoCompletedQueueEntry(IQueueEntry queueEntry, Type workItemDataType, object workItem)
+ {
+ _inner.LogAutoCompletedQueueEntry(queueEntry, workItemDataType, workItem);
+ }
+
+ public async Task HandleItemAsync(WorkItemContext context)
+ {
+ if (context.Data is not IBackgroundTaskWorkItem { TaskId: { Length: > 0 } taskId })
+ {
+ await _inner.HandleItemAsync(context);
+ return;
+ }
+
+ string? lastMessage = null;
+ var tracked = new WorkItemContext(context.Data, context.JobId, context.WorkItemLock, context.CancellationToken, async (progress, message) =>
+ {
+ // Give the job its renewal opportunity before tracking waits on cache or messaging.
+ await context.ReportProgressAsync(progress, message);
+
+ lastMessage = message ?? lastMessage;
+ await TryUpdateAsync(taskId, task =>
+ {
+ if (!task.IsActive)
+ return false;
+
+ task.Start(UtcNow);
+ task.Progress = Math.Clamp(progress, 0, 100);
+ task.Message = message;
+ return true;
+ });
+ });
+
+ if (!await StartAsync(taskId))
+ return;
+
+ try
+ {
+ await _inner.HandleItemAsync(tracked);
+ }
+ catch (Exception)
+ {
+ await TryUpdateAsync(taskId, task =>
+ {
+ if (!task.IsActive)
+ return false;
+
+ // The exception is logged by the job; its message can contain internal details, so the user sees a generic one.
+ task.Finish(BackgroundTaskStatus.Failed, "An error occurred. It will be retried automatically if possible.", UtcNow);
+ return true;
+ });
+
+ throw;
+ }
+
+ context.Result = tracked.Result;
+
+ // A result that is not a success (for example a shutdown) means the queue runs the work item again, so the task keeps running.
+ if (!tracked.Result.IsSuccess)
+ return;
+
+ await TryUpdateAsync(taskId, task =>
+ {
+ if (!task.IsActive)
+ return false;
+
+ task.Finish(BackgroundTaskStatus.Completed, lastMessage ?? "Completed.", UtcNow);
+ return true;
+ });
+ }
+
+ private DateTime UtcNow => _timeProvider.GetUtcNow().UtcDateTime;
+
+ ///
+ /// Marks the task running. A queue retry runs the work item again after it failed, which brings the task back to running
+ /// and back into the active index. A cancelled or completed task must not run again, so a failed start decision must
+ /// be retried by the queue rather than treated as a missing task record.
+ ///
+ private async Task StartAsync(string taskId)
+ {
+ bool wasFinished = false;
+ var task = await _taskService.UpdateAsync(taskId, current =>
+ {
+ if (current.Status is BackgroundTaskStatus.Cancelled or BackgroundTaskStatus.Completed)
+ return false;
+
+ wasFinished = current.Status is BackgroundTaskStatus.Failed;
+ current.Start(UtcNow);
+ if (wasFinished)
+ {
+ current.CompletedUtc = null;
+ current.Message = null;
+ current.Progress = 0;
+ }
+
+ return true;
+ });
+
+ return task is not { Status: BackgroundTaskStatus.Cancelled or BackgroundTaskStatus.Completed };
+ }
+
+ private async Task TryUpdateAsync(string taskId, Func update)
+ {
+ BackgroundTask? result = null;
+ await TryAsync(taskId, async () => result = await _taskService.UpdateAsync(taskId, update));
+ return result;
+ }
+
+ // Tracking is best effort and uses no caller cancellation token. A transport failure must not replace the worker's outcome.
+ private async Task TryAsync(string taskId, Func action)
+ {
+ try
+ {
+ await action();
+ }
+ catch (Exception ex)
+ {
+ Log.LogWarning(ex, "Unable to update background task {TaskId}: {Message}", taskId, ex.Message);
+ }
+ }
+}
diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs
new file mode 100644
index 0000000000..2ea69febab
--- /dev/null
+++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs
@@ -0,0 +1,399 @@
+using System.Text.Json;
+using Exceptionless.Core.Models.BackgroundTasks;
+using Exceptionless.Core.Models.BulkOperations;
+using Exceptionless.Core.Models.WorkItems;
+using Exceptionless.Core.Repositories.Base;
+using Exceptionless.Core.Services.BackgroundTasks;
+using Exceptionless.Core.Services.BulkOperations;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Foundatio.Resilience;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Jobs.WorkItemHandlers;
+
+///
+/// Runs a stored bulk task in batches. After every batch the counts and the resume cursor are saved and a progress
+/// message is published; between batches it checks for cancellation. A restarted job continues from the saved cursor.
+///
+public class BulkOperationWorkItemHandler : WorkItemHandlerBase
+{
+ ///
+ /// How many temporary errors in a row (without a batch completing in between) end the task.
+ ///
+ public const int MaxTransientFailures = 3;
+
+ ///
+ /// Saved as the cursor once the last batch is done, so a run that restarts before the task is marked complete finds
+ /// nothing left to do instead of starting the selection over.
+ ///
+ internal const string ExhaustedCursor = "exhausted";
+
+ private readonly BulkOperationService _service;
+ private readonly BackgroundTaskService _taskService;
+ private readonly IReadOnlyDictionary _resolvers;
+ private readonly BulkOperationOptions _options;
+ private readonly ILockProvider _lockProvider;
+ private readonly TimeProvider _timeProvider;
+ private readonly IResiliencePolicy _checkpointPolicy;
+
+ public BulkOperationWorkItemHandler(BulkOperationService service, BackgroundTaskService taskService, IEnumerable resolvers, BulkOperationOptions options,
+ ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory)
+ {
+ _service = service;
+ _taskService = taskService;
+ _resolvers = resolvers.ToDictionary(r => r.Entity);
+ _options = options;
+ _lockProvider = lockProvider;
+ _timeProvider = timeProvider;
+ _checkpointPolicy = new ResiliencePolicyBuilder(Log, timeProvider)
+ .WithMaxAttempts(MaxTransientFailures)
+ .WithDelay(options.RetryDelay)
+ .WithShouldRetry((_, exception) => !IsPermanent(exception))
+ .Build();
+
+ // Progress is reported after every batch. Renewing the queue entry and the task lock then keeps a long run from
+ // timing out and being delivered to a second worker while this one is still deleting.
+ AutoRenewLockOnProgress = true;
+ }
+
+ public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default)
+ {
+ string cacheKey = $"{nameof(BulkOperationWorkItem)}:{((BulkOperationWorkItem)workItem).TaskId}";
+ return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(15), cancellationToken);
+ }
+
+ public override async Task HandleItemAsync(WorkItemContext context)
+ {
+ var workItem = context.GetData()!;
+ ArgumentException.ThrowIfNullOrWhiteSpace(workItem.TaskId);
+
+ BackgroundTask? task = null;
+ BackgroundTask? selectionTask = null;
+ string? pinnedSelection = null;
+ string? latestSelectionCursor = null;
+ try
+ {
+ task = await _taskService.GetAsync(workItem.TaskId);
+ if (task is null)
+ {
+ // The record expired or the cache was cleared. Say so, otherwise the progress panel waits forever.
+ Log.LogWarning("Bulk operation {TaskId} no longer exists", workItem.TaskId);
+ await _service.PublishLostAsync(workItem);
+ return;
+ }
+
+ selectionTask = task;
+ latestSelectionCursor = task.Bulk?.Cursor;
+ using var _ = Log.BeginScope(new ExceptionlessState().Organization(task.OrganizationId).Tag("BulkOperation"));
+ if (!task.IsActive)
+ {
+ Log.LogInformation("Skipping bulk operation {TaskId} because it is already {Status}", task.Id, task.Status);
+ return;
+ }
+
+ var handler = _service.GetHandler(task.Type);
+ if (handler is null || !_resolvers.TryGetValue(handler.Entity, out var resolver))
+ {
+ await FinishAsync(workItem, BackgroundTaskStatus.Failed, $"Unknown bulk operation type '{task.Type}'.");
+ return;
+ }
+
+ if (await _taskService.IsCancelRequestedAsync(task.Id))
+ {
+ await FinishAsync(workItem, BackgroundTaskStatus.Cancelled, "Cancelled by user.");
+ return;
+ }
+
+ if (task.Bulk!.Selection.Kind is BulkOperationSelectionKind.Filter)
+ {
+ pinnedSelection = await _service.GetPinnedSelectionAsync(task.Id);
+ if (pinnedSelection is null)
+ {
+ await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The saved selection expired, so the operation could not continue.");
+ return;
+ }
+
+ await _service.RefreshSelectionExpirationAsync(task);
+ }
+
+ task = await _taskService.UpdateAsync(task.Id, current =>
+ {
+ if (!current.IsActive)
+ return false;
+
+ current.Start(UtcNow);
+ return true;
+ });
+
+ if (task is null)
+ {
+ await _service.PublishLostAsync(workItem);
+ return;
+ }
+
+ if (task.Status is not BackgroundTaskStatus.Running)
+ return;
+
+ Log.LogInformation("Running bulk operation {TaskId} {Type} from cursor {Cursor}", task.Id, task.Type, task.Bulk!.Cursor);
+ await context.ReportProgressAsync(GetPercent(task), task.Description);
+
+ while (true)
+ {
+ if (context.CancellationToken.IsCancellationRequested)
+ {
+ // The host is shutting down. Leave the task running so the queue retry resumes it from the cursor.
+ context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {task.Id} for shutdown");
+ return;
+ }
+
+ // Admission can retire a stale worker while it is between batches. Observe the saved outcome before
+ // starting another action, including when the old worker resumes after its replacement was admitted.
+ task = await _taskService.GetAsync(workItem.TaskId);
+ if (task is null)
+ {
+ await _service.PublishLostAsync(workItem);
+ return;
+ }
+
+ if (!task.IsActive)
+ return;
+
+ if (await _taskService.IsCancelRequestedAsync(task.Id))
+ {
+ await FinishAsync(workItem, BackgroundTaskStatus.Cancelled, $"Cancelled by user after {task.Processed:N0} of {task.Total:N0}.");
+ return;
+ }
+
+ var page = await GetPageAsync(task, resolver, pinnedSelection, context.CancellationToken);
+ latestSelectionCursor = page.NextCursor ?? latestSelectionCursor;
+ if (page.Ids.Count == 0)
+ break;
+
+ // The user who started the task is authorized once, when it is created. Each batch re-checks the scope
+ // (organization and project) but not the user: someone removed from the organization mid-run still finishes
+ // the deletion they legitimately started, and any member of the organization can cancel it.
+ var result = await handler.ExecuteBatchAsync(task, page.Ids, context.CancellationToken);
+
+ task = await SaveBatchAsync(task, page, result);
+
+ if (task is null)
+ {
+ await _service.PublishLostAsync(workItem);
+ return;
+ }
+
+ if (!task.IsActive)
+ return;
+
+ await _service.RefreshSelectionExpirationAsync(task);
+ await context.ReportProgressAsync(GetPercent(task), $"{task.Processed:N0} of {task.Total:N0}");
+
+ if (page.NextCursor is null)
+ break;
+ }
+
+ task = await _taskService.UpdateAsync(task.Id, current =>
+ {
+ if (!current.IsActive)
+ return false;
+
+ current.Total = current.Processed;
+ current.Finish(BackgroundTaskStatus.Completed, current.Failed > 0 ? $"Completed with {current.Failed:N0} failures." : "Completed.", UtcNow);
+ return true;
+ });
+
+ if (task is null)
+ await _service.PublishLostAsync(workItem);
+ }
+ catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested)
+ {
+ context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {workItem.TaskId} for shutdown");
+ }
+ catch (BatchCheckpointException ex)
+ {
+ Log.LogError(ex, "Bulk operation {TaskId} could not save completed batch progress: {Message}", workItem.TaskId, ex.Message);
+ await FinishAsync(workItem, BackgroundTaskStatus.Failed, ex.Message);
+ }
+ catch (BulkSelectionRejectedException ex)
+ {
+ Log.LogWarning(ex, "Bulk operation {TaskId} selection is no longer available: {Message}", workItem.TaskId, ex.Message);
+ await FinishAsync(workItem, BackgroundTaskStatus.Failed, ex.Message);
+ }
+ catch (Exception ex) when (!IsPermanent(ex))
+ {
+ // Elasticsearch, cache and network hiccups: continue in a new work item from the saved counts and cursor. Rethrowing
+ // would use the queue's retries, which cover the whole work item, so a task that makes progress between errors
+ // would run out of deliveries while still marked running. After MaxTransientFailures in a row, give up.
+ var failed = await _taskService.UpdateAsync(workItem.TaskId, current =>
+ {
+ if (!current.IsActive)
+ return false;
+
+ current.Bulk!.RetryCount++;
+ current.Message = current.Bulk!.RetryCount >= MaxTransientFailures ? "The operation stopped because of repeated errors." : "Retrying after a temporary error.";
+ if (current.Bulk!.RetryCount >= MaxTransientFailures)
+ current.Finish(BackgroundTaskStatus.Failed, current.Message, UtcNow);
+
+ return true;
+ });
+
+ if (failed is { IsActive: false })
+ {
+ Log.LogError(ex, "Bulk operation {TaskId} failed after repeated errors: {Message}", workItem.TaskId, ex.Message);
+ return;
+ }
+
+ Log.LogWarning(ex, "Bulk operation {TaskId} hit a temporary error: {Message}", workItem.TaskId, ex.Message);
+ if (failed is null)
+ {
+ await _service.PublishLostAsync(workItem);
+ return;
+ }
+
+ try
+ {
+ if (_options.RetryDelay > TimeSpan.Zero)
+ await Task.Delay(_options.RetryDelay * failed.Bulk!.RetryCount, _timeProvider, context.CancellationToken);
+
+ await _service.ContinueAsync(workItem);
+ }
+ catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested)
+ {
+ context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {workItem.TaskId} for shutdown");
+ }
+ catch (Exception continuationException)
+ {
+ Log.LogError(continuationException, "Bulk operation {TaskId} could not queue another run: {Message}", workItem.TaskId, continuationException.Message);
+ await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The operation stopped because another run could not be queued.");
+ }
+ }
+ catch (Exception ex)
+ {
+ Log.LogError(ex, "Bulk operation {TaskId} failed: {Message}", workItem.TaskId, ex.Message);
+ await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The operation stopped because of an unexpected error.");
+ }
+ finally
+ {
+ try
+ {
+ // FinishAsync and enqueue acknowledgement races can change the stored outcome without changing the local
+ // task. Only terminal/lost work releases its selection; active continuation and shutdown must retain it.
+ var saved = await _taskService.GetAsync(workItem.TaskId);
+ if (saved?.IsActive != true)
+ {
+ var finished = saved ?? selectionTask;
+ if (finished is null && workItem.Type is not null)
+ {
+ // The selection may outlive a lost task record. The work item still identifies its resolver.
+ finished = new BackgroundTask
+ {
+ Id = workItem.TaskId,
+ OrganizationId = workItem.OrganizationId,
+ Type = workItem.Type,
+ Bulk = new BulkOperationDetails { Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter } }
+ };
+ }
+
+ if (finished is not null)
+ await _service.ReleaseSelectionAsync(finished, pinnedSelection, latestSelectionCursor);
+ }
+ }
+ catch (Exception ex)
+ {
+ // A bounded snapshot lease is the fallback when the task store or search backend is unavailable.
+ Log.LogWarning(ex, "Could not check bulk operation {TaskId} for selection cleanup: {Message}", workItem.TaskId, ex.Message);
+ }
+ }
+ }
+
+ private async Task SaveBatchAsync(BackgroundTask task, BulkSelectionPage page, BulkBatchResult result)
+ {
+ // Keep the known result while retrying its checkpoint, including during shutdown. Replaying a successful delete
+ // would classify its now-missing ids as failures or omit them from a filter selection's counts. A process stopping
+ // before any checkpoint is saved can still lose that result; the deletion and task record are separate writes.
+ try
+ {
+ return await _checkpointPolicy.ExecuteAsync(async _ => await _taskService.UpdateAsync(task.Id, current =>
+ {
+ // A save can succeed before releasing its update lock throws. The prior cursor makes another attempt harmless
+ // if that checkpoint was committed already, or if the task has since advanced or finished.
+ if (!current.IsActive || !String.Equals(current.Bulk!.Cursor, task.Bulk!.Cursor, StringComparison.Ordinal))
+ return false;
+
+ current.Processed += page.Ids.Count;
+ current.Succeeded += result.Succeeded;
+ current.Failed += result.Failed;
+ current.Bulk.Cursor = page.NextCursor ?? ExhaustedCursor;
+ current.Bulk.RetryCount = 0;
+ current.Total = Math.Max(current.Total, current.Processed);
+ current.Message = result.Message;
+ current.Progress = GetPercent(current);
+ return true;
+ }), CancellationToken.None);
+ }
+ catch (Exception ex)
+ {
+ // Replaying a batch known to have succeeded would lose its result again. Stop if its bounded checkpoint
+ // retries are exhausted, keeping the last durable counts and making the uncertain outcome explicit.
+ throw new BatchCheckpointException(ex);
+ }
+ }
+
+ ///
+ /// Errors that retrying cannot fix. Everything else (Elasticsearch, cache and network failures surface as many different
+ /// exception types) is treated as temporary and bounded by .
+ ///
+ internal static bool IsPermanent(Exception exception)
+ {
+ for (var current = exception; current is not null; current = current.InnerException)
+ {
+ if (current is DocumentLimitExceededException or ArgumentException or InvalidOperationException or NotSupportedException or FormatException or JsonException or InvalidCastException or NullReferenceException or KeyNotFoundException)
+ return true;
+ }
+
+ return false;
+ }
+
+ private async Task GetPageAsync(BackgroundTask task, IBulkSelectionResolver resolver, string? pinnedSelection, CancellationToken cancellationToken)
+ {
+ int size = Math.Max(1, _options.BatchSize);
+ if (task.Bulk!.Cursor == ExhaustedCursor)
+ return new BulkSelectionPage([], null);
+
+ if (task.Bulk!.Selection.Kind is not BulkOperationSelectionKind.Ids)
+ return await resolver.GetPageAsync(task, pinnedSelection, task.Bulk!.Cursor, size, cancellationToken);
+
+ // The cursor of an id selection is the index of the next id to process.
+ var ids = task.Bulk!.Selection.Ids ?? [];
+ int start = Int32.TryParse(task.Bulk!.Cursor, out int parsed) ? parsed : 0;
+ var batch = ids.Skip(start).Take(size).ToList();
+ int next = start + batch.Count;
+ return new BulkSelectionPage(batch, next < ids.Count ? next.ToString() : null);
+ }
+
+ private DateTime UtcNow => _timeProvider.GetUtcNow().UtcDateTime;
+
+ private async Task FinishAsync(BulkOperationWorkItem workItem, BackgroundTaskStatus status, string message)
+ {
+ var task = await _taskService.UpdateAsync(workItem.TaskId, current =>
+ {
+ if (!current.IsActive)
+ return false;
+
+ current.Finish(status, message, UtcNow);
+ return true;
+ });
+
+ if (task is null)
+ await _service.PublishLostAsync(workItem);
+ }
+
+ private static int GetPercent(BackgroundTask task)
+ {
+ return task.Total <= 0 ? 0 : (int)Math.Min(99, task.Processed * 100 / task.Total);
+ }
+
+ private sealed class BatchCheckpointException(Exception innerException)
+ : InvalidOperationException("The operation stopped because its progress could not be saved. Some items may have been processed.", innerException);
+}
diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs
index f88d91c33f..f8611080de 100644
--- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs
+++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs
@@ -84,18 +84,26 @@ public override async Task HandleItemAsync(WorkItemContext context)
for (int p = 0; p < projectList.Count; p++)
{
+ int projectEventCount = eventsPerProject + (p < remainder ? 1 : 0);
+ if (projectEventCount == 0)
+ continue;
+
if (context.CancellationToken.IsCancellationRequested)
- break;
+ {
+ context.Result = JobResult.CancelledWithMessage("Stopping sample event generation for shutdown");
+ return;
+ }
var project = projectList[p];
- int projectEventCount = eventsPerProject + (p < remainder ? 1 : 0);
-
var events = generator.Generate(organization.Id, project.Id, projectEventCount, minDate, utcNow);
foreach (var batch in events.Chunk(batchSize))
{
if (context.CancellationToken.IsCancellationRequested)
- break;
+ {
+ context.Result = JobResult.CancelledWithMessage("Stopping sample event generation for shutdown");
+ return;
+ }
await _eventPipeline.RunAsync(batch, organization, project, allowExtendedEventDateRange: true);
totalProcessed += batch.Length;
@@ -138,7 +146,10 @@ private async Task GenerateProjectSampleEventsAsync(WorkItemContext context, Ran
foreach (var batch in events.Chunk(batchSize))
{
if (context.CancellationToken.IsCancellationRequested)
- break;
+ {
+ context.Result = JobResult.CancelledWithMessage("Stopping sample event generation for shutdown");
+ return;
+ }
await _eventPipeline.RunAsync(batch, organization, project, allowExtendedEventDateRange: true);
totalProcessed += batch.Length;
diff --git a/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs
new file mode 100644
index 0000000000..504febcffb
--- /dev/null
+++ b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs
@@ -0,0 +1,76 @@
+using System.Text.Json.Serialization;
+using Exceptionless.Core.Models.BulkOperations;
+using Foundatio.Repositories.Models;
+
+namespace Exceptionless.Core.Models.BackgroundTasks;
+
+///
+/// Long-running work that a user can follow in the progress panel: a bulk action, a project data reset, sample data
+/// generation. Stored in the cache with a limited lifetime and indexed per organization; it is intentionally not
+/// persisted in Elasticsearch. Everything a bulk batch changes (counts and cursor) lives in this one record so a
+/// batch is saved atomically.
+///
+public class BackgroundTask : IIdentity, IOwnedByOrganization, IHaveDates
+{
+ public string Id { get; set; } = null!;
+ public string OrganizationId { get; set; } = null!;
+ public string? ProjectId { get; set; }
+
+ /// What the task does, e.g. events.delete or projects.reset-data.
+ public string Type { get; set; } = null!;
+
+ public string Description { get; set; } = null!;
+ public BackgroundTaskStatus Status { get; set; } = BackgroundTaskStatus.Queued;
+
+ /// Percent complete, 0 to 100.
+ public int Progress { get; set; }
+
+ /// Item counts. All zero for tasks that only report a percent.
+ public long Total { get; set; }
+ public long Processed { get; set; }
+ public long Succeeded { get; set; }
+ public long Failed { get; set; }
+
+ public string? Message { get; set; }
+
+ /// Whether the user can stop the task. Tasks that cannot be stopped midway are not cancellable.
+ public bool CanCancel { get; set; }
+
+ public string CreatedByUserId { get; set; } = null!;
+ public DateTime CreatedUtc { get; set; }
+ public DateTime? StartedUtc { get; set; }
+ public DateTime? CompletedUtc { get; set; }
+ public DateTime UpdatedUtc { get; set; }
+
+ /// Only set for bulk actions: what they apply to and where they resume.
+ public BulkOperationDetails? Bulk { get; set; }
+
+ [JsonIgnore]
+ public bool IsActive => Status is BackgroundTaskStatus.Queued or BackgroundTaskStatus.Running;
+
+ /// Marks the task as running. The start time is only recorded the first time.
+ public void Start(DateTime utcNow)
+ {
+ Status = BackgroundTaskStatus.Running;
+ StartedUtc ??= utcNow;
+ }
+
+ public void Finish(BackgroundTaskStatus status, string? message, DateTime utcNow)
+ {
+ Status = status;
+ Message = message;
+ CompletedUtc = utcNow;
+ if (status is BackgroundTaskStatus.Completed)
+ Progress = 100;
+ }
+}
+
+[JsonConverter(typeof(JsonStringEnumConverter))]
+public enum BackgroundTaskStatus
+{
+ Queued,
+ Running,
+ Completed,
+ Failed,
+ Cancelled
+}
diff --git a/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs
new file mode 100644
index 0000000000..ff51c9a86b
--- /dev/null
+++ b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs
@@ -0,0 +1,8 @@
+namespace Exceptionless.Core.Models.BackgroundTasks;
+
+/// The types of tasks that are not bulk actions. Bulk actions use the key of their handler, e.g. events.delete.
+public static class BackgroundTaskTypes
+{
+ public const string ResetProjectData = "projects.reset-data";
+ public const string GenerateProjectSampleData = "projects.sample-data";
+}
diff --git a/src/Exceptionless.Core/Models/BulkOperations/BulkOperationSelection.cs b/src/Exceptionless.Core/Models/BulkOperations/BulkOperationSelection.cs
new file mode 100644
index 0000000000..1bfec8a567
--- /dev/null
+++ b/src/Exceptionless.Core/Models/BulkOperations/BulkOperationSelection.cs
@@ -0,0 +1,50 @@
+using System.Text.Json.Serialization;
+
+namespace Exceptionless.Core.Models.BulkOperations;
+
+/// The part of a that only bulk actions have.
+public class BulkOperationDetails
+{
+ public BulkOperationSelection Selection { get; set; } = null!;
+ public Dictionary Parameters { get; set; } = new();
+
+ /// Consecutive batches that failed with a temporary error. Reset by the next successful batch.
+ public int RetryCount { get; set; }
+
+ /// Opaque position of the last batch that finished; the job resumes after it.
+ public string? Cursor { get; set; }
+}
+
+/// The kind of item an operation targets. Each entity has one selection resolver.
+[JsonConverter(typeof(JsonStringEnumConverter))]
+public enum BulkOperationEntity
+{
+ Events
+}
+
+[JsonConverter(typeof(JsonStringEnumConverter))]
+public enum BulkOperationSelectionKind
+{
+ Ids,
+ Filter
+}
+
+///
+/// Either an explicit list of ids or a filter. A filter selection always carries the absolute UTC window that was
+/// resolved when the operation was created, so items that arrive afterwards are never included.
+///
+public class BulkOperationSelection
+{
+ public BulkOperationSelectionKind Kind { get; set; }
+ public List? Ids { get; set; }
+ public string? Filter { get; set; }
+
+ /// The relative time expression the window was resolved from. Informational only.
+ public string? Time { get; set; }
+
+ public DateTime? UtcStart { get; set; }
+ public DateTime? UtcEnd { get; set; }
+
+ /// Items stored after this instant (UTC) are never part of a filter selection.
+ public DateTime? UtcCreatedBefore { get; set; }
+}
diff --git a/src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs b/src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs
new file mode 100644
index 0000000000..d16f69c23e
--- /dev/null
+++ b/src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs
@@ -0,0 +1,25 @@
+using Exceptionless.Core.Models.BackgroundTasks;
+
+namespace Exceptionless.Core.Messaging.Models;
+
+///
+/// Published after every progress report and every status change of a background task. Intentionally independent of the
+/// transport that delivers it to clients, and free of the selection and parameters.
+///
+public record BackgroundTaskUpdated
+{
+ public required string Id { get; set; }
+ public required string OrganizationId { get; set; }
+ public string? ProjectId { get; set; }
+ public required string Type { get; set; }
+ public required string Description { get; set; }
+ public BackgroundTaskStatus Status { get; set; }
+ public int Progress { get; set; }
+ public long Total { get; set; }
+ public long Processed { get; set; }
+ public long Succeeded { get; set; }
+ public long Failed { get; set; }
+ public string? Message { get; set; }
+ public bool CanCancel { get; set; }
+ public DateTime UpdatedUtc { get; set; }
+}
diff --git a/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs
new file mode 100644
index 0000000000..523bd67151
--- /dev/null
+++ b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs
@@ -0,0 +1,15 @@
+namespace Exceptionless.Core.Models.WorkItems;
+
+public record BulkOperationWorkItem : IBackgroundTaskWorkItem
+{
+ public required string TaskId { get; init; }
+
+ ///
+ /// The organization, type and description are carried so a final state can still be published if the task
+ /// record is gone when the job runs.
+ ///
+ public required string OrganizationId { get; init; }
+
+ public string? Type { get; init; }
+ public string? Description { get; init; }
+}
diff --git a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs
index 4bf9a8db67..ecc627b735 100644
--- a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs
+++ b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs
@@ -1,9 +1,10 @@
namespace Exceptionless.Core.Models.WorkItems;
-public record GenerateSampleEventsWorkItem
+public record GenerateSampleEventsWorkItem : IBackgroundTaskWorkItem
{
public string? OrganizationId { get; init; }
public string? ProjectId { get; init; }
public int EventCount { get; init; } = 250;
public int DaysBack { get; init; } = 7;
+ public string? TaskId { get; init; }
}
diff --git a/src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs
new file mode 100644
index 0000000000..3c3f7d499d
--- /dev/null
+++ b/src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs
@@ -0,0 +1,10 @@
+namespace Exceptionless.Core.Models.WorkItems;
+
+///
+/// A work item whose progress is tracked by a background task, so the user can follow it in the progress panel.
+///
+public interface IBackgroundTaskWorkItem
+{
+ /// The tracking task, or null when the work item was queued without one.
+ string? TaskId { get; }
+}
diff --git a/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs
index 4d4410c6c0..929a05b55b 100644
--- a/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs
+++ b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs
@@ -1,7 +1,8 @@
namespace Exceptionless.Core.Models.WorkItems;
-public record ResetProjectDataWorkItem
+public record ResetProjectDataWorkItem : IBackgroundTaskWorkItem
{
public required string OrganizationId { get; init; }
public required string ProjectId { get; init; }
+ public string? TaskId { get; init; }
}
diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs
index 43065cc189..a41f8c7dc2 100644
--- a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs
+++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs
@@ -8,6 +8,16 @@ namespace Exceptionless.Core.Repositories.Queries;
public class EventFieldsQueryVisitor : ChainableQueryVisitor
{
public override async Task VisitAsync(GroupNode node, IQueryVisitorContext context)
+ {
+ node.Field = GetCustomFieldName(node.Field, GetGroupChildTerms(node)) ?? node.Field;
+ foreach (var child in node.Children)
+ await child.AcceptAsync(this, context);
+ }
+
+ ///
+ /// The terms of a group's unfielded children, which pick the stored type of a custom field set on the group.
+ ///
+ internal static string[] GetGroupChildTerms(GroupNode node)
{
var childTerms = new List();
if (node.Left is TermNode { Field: null, Term: not null } leftTermNode)
@@ -32,9 +42,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte
childTerms.Add(rightTermRangeNode.Max);
}
- node.Field = GetCustomFieldName(node.Field, childTerms.ToArray()) ?? node.Field;
- foreach (var child in node.Children)
- await child.AcceptAsync(this, context);
+ return childTerms.ToArray();
}
public override Task VisitAsync(TermNode node, IQueryVisitorContext context)
@@ -67,7 +75,10 @@ public override Task VisitAsync(MissingNode node, IQueryVisitorContext context)
return Task.CompletedTask;
}
- private string? GetCustomFieldName(string? field, string?[] terms)
+ ///
+ /// Maps a custom data or reference field to the stored idx field. The suffix is the stored type, picked from the terms.
+ ///
+ internal static string? GetCustomFieldName(string? field, string?[] terms)
{
if (String.IsNullOrEmpty(field))
return null;
diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs
new file mode 100644
index 0000000000..8c86a76275
--- /dev/null
+++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs
@@ -0,0 +1,436 @@
+using Exceptionless.Core.Messaging.Models;
+using Exceptionless.Core.Models.BackgroundTasks;
+using Foundatio.Caching;
+using Foundatio.Jobs;
+using Foundatio.Lock;
+using Foundatio.Messaging;
+using Foundatio.Queues;
+using Foundatio.Repositories.Utility;
+using Foundatio.Resilience;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Services.BackgroundTasks;
+
+public class BackgroundTaskOptions
+{
+ /// How long one attempt to lock a task waits. A busy lock is retried by the resilience policy.
+ public TimeSpan LockAcquireTimeout { get; set; } = TimeSpan.FromSeconds(5);
+}
+
+///
+/// Stores background tasks in the cache (with a limited lifetime), keeps a per-organization index and an index of the
+/// active tasks, and publishes every change so the progress panel can follow along.
+///
+public class BackgroundTaskService
+{
+ public static readonly TimeSpan TaskLifetime = TimeSpan.FromDays(7);
+
+ /// An active task that has not reported progress for this long is considered abandoned.
+ public static readonly TimeSpan StaleAfter = TimeSpan.FromHours(2);
+
+ /// How many finished tasks the list shows, in addition to all active tasks.
+ public const int ListLimit = 50;
+
+ /// Every cache key and lock resource of background tasks starts with this prefix.
+ public const string KeyPrefix = "background-tasks";
+
+ /// Bounds task payloads loaded together, including selections attached to bulk tasks.
+ internal const int ReadBatchSize = 50;
+ private static readonly TimeSpan UpdateLockLifetime = TimeSpan.FromSeconds(30);
+ private readonly ICacheClient _cache;
+ private readonly ILockProvider _lockProvider;
+ private readonly IMessagePublisher _publisher;
+ private readonly IQueue _workItemQueue;
+ private readonly IResiliencePolicy _lockPolicy;
+ private readonly BackgroundTaskOptions _options;
+ private readonly TimeProvider _timeProvider;
+ private readonly ILogger _logger;
+
+ public BackgroundTaskService(ICacheClient cache, ILockProvider lockProvider, IMessagePublisher publisher, IQueue workItemQueue,
+ IResiliencePolicyProvider resiliencePolicyProvider, BackgroundTaskOptions options, TimeProvider timeProvider, ILogger logger)
+ {
+ _cache = cache;
+ _lockProvider = lockProvider;
+ _publisher = publisher;
+ _workItemQueue = workItemQueue;
+ _options = options;
+ _timeProvider = timeProvider;
+ _logger = logger;
+
+ // Several workers and web instances update the same record, so a busy lock is normal: wait a little, then try again.
+ _lockPolicy = resiliencePolicyProvider.GetPolicy(b => b.WithMaxAttempts(5).WithLinearDelay(TimeSpan.FromMilliseconds(250)), logger, timeProvider);
+ }
+
+ public static string GetTaskCacheKey(string id) => $"{KeyPrefix}:{id}";
+ public static string GetCancelCacheKey(string id) => $"{KeyPrefix}:{id}:cancel";
+ public static string GetOrganizationIndexCacheKey(string organizationId) => $"{KeyPrefix}:organization:{organizationId}";
+ public static string GetActiveIndexCacheKey(string organizationId) => $"{KeyPrefix}:organization:{organizationId}:active";
+ public static string GetUpdateLockResource(string id) => $"{KeyPrefix}:update:{id}";
+ public static string GetCreateLockResource(string organizationId) => $"{KeyPrefix}:create:{organizationId}";
+
+ public async Task GetAsync(string id)
+ {
+ if (String.IsNullOrEmpty(id))
+ return null;
+
+ var cached = await _cache.GetAsync(GetTaskCacheKey(id));
+ return cached.HasValue ? cached.Value : null;
+ }
+
+ ///
+ /// Returns the existing tasks in input order, once per id. Missing records are omitted. Reads are batched so a
+ /// remote cache can retrieve several records together; callers remain responsible for checking their scope.
+ ///
+ public async Task> GetByIdsAsync(IEnumerable ids)
+ {
+ var tasks = new List();
+ foreach (var keys in ids.Distinct(StringComparer.Ordinal).Select(GetTaskCacheKey).Chunk(ReadBatchSize))
+ {
+ var cached = await _cache.GetAllAsync(keys);
+ foreach (string key in keys)
+ {
+ if (cached.TryGetValue(key, out var value) && value.HasValue && value.Value is { } task)
+ tasks.Add(task);
+ }
+ }
+
+ return tasks;
+ }
+
+ ///
+ /// All active tasks and the newest finished tasks of the organization, newest first. Only finished history is limited.
+ ///
+ public async Task> GetByOrganizationAsync(string organizationId, int limit = ListLimit)
+ {
+ limit = Math.Max(0, limit);
+ var activeIds = (await GetActiveIdsAsync(organizationId)).ToHashSet(StringComparer.Ordinal);
+ // One snapshot avoids repeated full-list cloning in memory and offset shifts while entries expire.
+ var indexed = await _cache.GetListAsync(GetOrganizationIndexCacheKey(organizationId));
+ // ObjectIds order whole seconds; their machine/process bytes can reverse creation order within a second.
+ var newestIds = (indexed.HasValue ? indexed.Value : [])
+ .Concat(activeIds)
+ .Distinct(StringComparer.Ordinal)
+ .OrderByDescending(id => id, StringComparer.Ordinal);
+
+ var tasks = new List();
+ var finished = new List();
+ string? cutoffSecond = null;
+ // Read the entire cutoff second before choosing its newest tasks by CreatedUtc, including across cache batches.
+ foreach (var batch in newestIds.Where(id => finished.Count < limit || activeIds.Contains(id)
+ || (cutoffSecond is not null && id.StartsWith(cutoffSecond, StringComparison.Ordinal))).Chunk(ReadBatchSize))
+ {
+ var found = await GetByIdsAsync(batch);
+ var expired = batch.Except(found.Select(task => task.Id), StringComparer.Ordinal).ToArray();
+ foreach (var task in found)
+ {
+ if (!String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal))
+ continue;
+
+ if (task.IsActive)
+ tasks.Add(task);
+ else
+ {
+ finished.Add(task);
+ if (finished.Count == limit)
+ cutoffSecond = task.Id[..8];
+ }
+ }
+
+ // A burst within one second must not retain every task's payload while its remaining batches are read.
+ if (finished.Count > limit)
+ {
+ finished.Sort(CompareNewestFirst);
+ finished.RemoveRange(limit, finished.Count - limit);
+ }
+
+ if (expired.Length > 0)
+ await PruneIndexesAsync(organizationId, expired);
+ }
+
+ tasks.AddRange(finished);
+ tasks.Sort(CompareNewestFirst);
+ return tasks;
+ }
+
+ /// The ids of the organization's tasks that have not finished.
+ public async Task> GetActiveIdsAsync(string organizationId)
+ {
+ // Completion removes ids concurrently. Read one snapshot so a removal cannot shift an unread id past a page offset.
+ var ids = await _cache.GetListAsync(GetActiveIndexCacheKey(organizationId));
+ return ids.HasValue ? ids.Value.Distinct(StringComparer.Ordinal).ToList() : [];
+ }
+
+ ///
+ /// Rechecks suspected stale ids under their update locks before removing index entries. Busy tasks are skipped for
+ /// a later read; active tasks and tasks owned by another organization are retained. Only missing tasks lose history.
+ ///
+ public async Task PruneIndexesAsync(string organizationId, IReadOnlyCollection ids)
+ {
+ foreach (string id in ids.Distinct(StringComparer.Ordinal))
+ {
+ await using var updateLock = await _lockProvider.TryAcquireAsync(GetUpdateLockResource(id), UpdateLockLifetime, TimeSpan.Zero);
+ if (updateLock is null)
+ continue;
+
+ var task = await GetAsync(id);
+ if (task is not null && (task.IsActive || !String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal)))
+ continue;
+
+ if (task is null)
+ await _cache.ListRemoveAsync(GetOrganizationIndexCacheKey(organizationId), [id]);
+
+ await _cache.ListRemoveAsync(GetActiveIndexCacheKey(organizationId), [id]);
+ }
+ }
+
+ ///
+ /// Gives the task an id, stores it as queued and adds it to the indexes. Nothing is published or queued yet.
+ ///
+ public async Task AddAsync(BackgroundTask task)
+ {
+ var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
+ task.Id = ObjectId.GenerateNewId(utcNow).ToString();
+ task.Status = BackgroundTaskStatus.Queued;
+ task.CreatedUtc = utcNow;
+ task.UpdatedUtc = utcNow;
+
+ await SaveAsync(task);
+ try
+ {
+ if (await _cache.ListAddAsync(GetOrganizationIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime) != 1)
+ throw new InvalidOperationException("The background task could not be added to the organization index.");
+
+ if (await _cache.ListAddAsync(GetActiveIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime) != 1)
+ throw new InvalidOperationException("The background task could not be added to the active index.");
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Unable to index background task {TaskId}: {Message}", task.Id, ex.Message);
+ await FailToStartAsync(task.Id);
+ throw;
+ }
+ }
+
+ ///
+ /// Adds a task and queues the work item that runs it. The work item is created after the task has its id.
+ /// If preparation fails, the task is marked failed and no work is queued.
+ ///
+ public async Task CreateAndEnqueueAsync(BackgroundTask task, Func createWorkItem) where T : class
+ {
+ await AddAsync(task);
+
+ T workItem;
+ try
+ {
+ workItem = createWorkItem(task);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Unable to prepare background task {TaskId}: {Message}", task.Id, ex.Message);
+ await FailToStartAsync(task.Id);
+ throw;
+ }
+
+ await EnqueueAsync(task, workItem);
+ return task;
+ }
+
+ ///
+ /// Prepares and queues the work item of a task that was added and publishes it. When preparation or queueing fails,
+ /// a task that is still queued is failed and the error is rethrown. Worker progress and cancellation are preserved.
+ ///
+ public async Task EnqueueAsync(BackgroundTask task, T workItem, Func? beforeEnqueue = null) where T : class
+ {
+ try
+ {
+ if (beforeEnqueue is not null)
+ await beforeEnqueue();
+
+ await _workItemQueue.EnqueueAsync(workItem);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Unable to queue background task {TaskId}: {Message}", task.Id, ex.Message);
+ await FailToStartAsync(task.Id);
+ throw;
+ }
+
+ await PublishAsync(task);
+ }
+
+ ///
+ /// Queues another run of a work item, e.g. to continue a long task after a temporary error. The queue's retries cover a
+ /// whole work item, so continuing in a new work item does not use up the retries of the current one.
+ ///
+ public Task ContinueAsync(T workItem) where T : class
+ {
+ return _workItemQueue.EnqueueAsync(workItem);
+ }
+
+ ///
+ /// Loads the task, lets change it, then saves and publishes it. The change is skipped when
+ /// returns false. Returns the stored task, or null when it no longer exists.
+ ///
+ ///
+ /// Tasks live in the cache rather than in Elasticsearch, so the repository's PatchAsync does not apply. The lock makes the
+ /// load, change and save one step; a busy lock is retried with the resilience policy before giving up.
+ ///
+ public async Task UpdateAsync(string id, Func update)
+ {
+ BackgroundTask? task;
+ await using (var updateLock = await _lockPolicy.ExecuteAsync(async ct =>
+ await _lockProvider.AcquireAsync(GetUpdateLockResource(id), UpdateLockLifetime, _options.LockAcquireTimeout), CancellationToken.None))
+ {
+ task = await GetAsync(id);
+ if (task is null)
+ return null;
+
+ var previousUpdatedUtc = task.UpdatedUtc;
+ if (!update(task))
+ {
+ // A declined mutation returns the stored state even if the callback changed its copy.
+ return await GetAsync(id);
+ }
+
+ // Clients use this stamp to reject delayed notifications, including when a clock repeats or moves backwards.
+ var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
+ task.UpdatedUtc = utcNow > previousUpdatedUtc ? utcNow : previousUpdatedUtc.AddTicks(1);
+ await SaveAsync(task);
+ if (task.IsActive)
+ {
+ // Active discovery must live as long as progress, including tasks that run longer than TaskLifetime.
+ await RefreshIndexAsync(GetActiveIndexCacheKey(task.OrganizationId), task.Id);
+ }
+ else
+ {
+ // Renew history on completion, rather than rewriting the large history index on every progress report.
+ // Keep active membership until history is saved so a concurrent list cannot miss a long-running task.
+ await RefreshIndexAsync(GetOrganizationIndexCacheKey(task.OrganizationId), task.Id);
+ await _cache.ListRemoveAsync(GetActiveIndexCacheKey(task.OrganizationId), [task.Id]);
+ }
+ }
+
+ // The state is committed; a slow publisher must not block another update or cancellation.
+ await PublishAsync(task);
+ return task;
+ }
+
+ ///
+ /// Requests cancellation. A queued (or abandoned) task is cancelled right away; a running bulk task stops at the next
+ /// batch boundary, after the batch that is in flight finishes. Tasks that cannot be stopped are rejected while they are
+ /// alive, but an abandoned one can always be cleared.
+ ///
+ public async Task CancelAsync(string id)
+ {
+ var task = await GetAsync(id);
+ if (task is null)
+ return BackgroundTaskCancelResult.NotFound;
+
+ var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
+ if (!task.IsActive)
+ return BackgroundTaskCancelResult.Cancelled(task);
+
+ bool isAbandoned = utcNow - task.UpdatedUtc >= StaleAfter;
+ if (!task.CanCancel && !isAbandoned)
+ return BackgroundTaskCancelResult.NotCancellable(task);
+
+ if (!await _cache.SetAsync(GetCancelCacheKey(id), true, TaskLifetime))
+ throw new InvalidOperationException("The cancellation request could not be saved.");
+
+ var updated = await UpdateAsync(id, current =>
+ {
+ if (!current.IsActive)
+ return false;
+
+ bool isRunningNow = current.Status is BackgroundTaskStatus.Running && utcNow - current.UpdatedUtc < StaleAfter;
+ if (isRunningNow)
+ return false;
+
+ current.Finish(BackgroundTaskStatus.Cancelled, current.Status is BackgroundTaskStatus.Queued ? "Cancelled before it started." : "Cancelled after it stopped responding.", utcNow);
+ return true;
+ });
+
+ return updated is null ? BackgroundTaskCancelResult.NotFound : BackgroundTaskCancelResult.Cancelled(updated);
+ }
+
+ public async Task IsCancelRequestedAsync(string id)
+ {
+ return (await _cache.GetAsync(GetCancelCacheKey(id))).HasValue;
+ }
+
+ private async Task SaveAsync(BackgroundTask task)
+ {
+ if (!await _cache.SetAsync(GetTaskCacheKey(task.Id), task, TaskLifetime))
+ throw new InvalidOperationException("The background task could not be saved.");
+ }
+
+ public Task PublishAsync(BackgroundTask task)
+ {
+ return PublishAsync(new BackgroundTaskUpdated
+ {
+ Id = task.Id,
+ OrganizationId = task.OrganizationId,
+ ProjectId = task.ProjectId,
+ Type = task.Type,
+ Description = task.Description,
+ Status = task.Status,
+ Progress = task.Progress,
+ Total = task.Total,
+ Processed = task.Processed,
+ Succeeded = task.Succeeded,
+ Failed = task.Failed,
+ Message = task.Message,
+ CanCancel = task.CanCancel,
+ UpdatedUtc = task.UpdatedUtc
+ });
+ }
+
+ public async Task PublishAsync(BackgroundTaskUpdated message)
+ {
+ try
+ {
+ await _publisher.PublishAsync(message);
+ }
+ catch (Exception ex)
+ {
+ // No caller token is passed, so a transport cancellation is also an advisory notification failure.
+ _logger.LogWarning(ex, "Failed to publish background task update for {TaskId}: {Message}", message.Id, ex.Message);
+ }
+ }
+
+ private Task FailToStartAsync(string id)
+ {
+ return UpdateAsync(id, current =>
+ {
+ // Queueing may have succeeded before its acknowledgement failed, allowing a worker or user to advance the task.
+ if (current.Status is not BackgroundTaskStatus.Queued)
+ return false;
+
+ current.Finish(BackgroundTaskStatus.Failed, "The task could not be started. Please try again.", _timeProvider.GetUtcNow().UtcDateTime);
+ return true;
+ });
+ }
+
+ private async Task RefreshIndexAsync(string key, string id)
+ {
+ await _cache.ListAddAsync(key, [id], TaskLifetime);
+ // Refreshing an existing list member does not extend the key TTL in every cache provider.
+ await _cache.SetExpirationAsync(key, TaskLifetime);
+ }
+
+ private static int CompareNewestFirst(BackgroundTask left, BackgroundTask right)
+ {
+ int byCreated = right.CreatedUtc.CompareTo(left.CreatedUtc);
+ return byCreated != 0 ? byCreated : StringComparer.Ordinal.Compare(right.Id, left.Id);
+ }
+}
+
+public record BackgroundTaskCancelResult
+{
+ public BackgroundTask? Task { get; init; }
+ public bool IsNotCancellable { get; init; }
+
+ public static readonly BackgroundTaskCancelResult NotFound = new();
+ public static BackgroundTaskCancelResult Cancelled(BackgroundTask task) => new() { Task = task };
+ public static BackgroundTaskCancelResult NotCancellable(BackgroundTask task) => new() { Task = task, IsNotCancellable = true };
+}
diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs
new file mode 100644
index 0000000000..283b96b3e8
--- /dev/null
+++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs
@@ -0,0 +1,296 @@
+using Exceptionless.Core.Messaging.Models;
+using Exceptionless.Core.Models.BackgroundTasks;
+using Exceptionless.Core.Models.BulkOperations;
+using Exceptionless.Core.Models.WorkItems;
+using Exceptionless.Core.Services.BackgroundTasks;
+using Foundatio.Caching;
+using Foundatio.Lock;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace Exceptionless.Core.Services.BulkOperations;
+
+public class BulkOperationOptions
+{
+ /// How many ids are resolved and executed per batch.
+ public int BatchSize { get; set; } = 500;
+
+ /// How long to wait before continuing after a temporary error. Multiplied by the number of errors in a row.
+ public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(5);
+}
+
+///
+/// The bulk specific part of background tasks: the registry of handlers and selection resolvers, counting and pinning a
+/// selection, and the one-active-destructive-task-per-project rule. Storage, indexes and notifications are
+/// .
+///
+public class BulkOperationService
+{
+ private static readonly TimeSpan CreateLockLifetime = TimeSpan.FromSeconds(30);
+ private readonly BackgroundTaskService _taskService;
+ private readonly ICacheClient _cache;
+ private readonly ILockProvider _lockProvider;
+ private readonly IReadOnlyDictionary _handlers;
+ private readonly IReadOnlyDictionary _resolvers;
+ private readonly TimeProvider _timeProvider;
+ private readonly ILogger _logger;
+
+ public BulkOperationService(BackgroundTaskService taskService, ICacheClient cache, ILockProvider lockProvider,
+ IEnumerable handlers, IEnumerable resolvers, TimeProvider timeProvider, ILogger? logger = null)
+ {
+ _taskService = taskService;
+ _cache = cache;
+ _lockProvider = lockProvider;
+ _handlers = handlers.ToDictionary(h => h.Type, StringComparer.Ordinal);
+ _resolvers = resolvers.ToDictionary(r => r.Entity);
+ _timeProvider = timeProvider;
+ _logger = logger ?? NullLogger.Instance;
+ }
+
+ public static string GetPinnedSelectionCacheKey(string id) => $"{BackgroundTaskService.KeyPrefix}:{id}:selection";
+
+ public IBulkOperationHandler? GetHandler(string type) => _handlers.GetValueOrDefault(type);
+
+ ///
+ /// Queues another run that continues from the saved cursor. The queue's retries cover a whole work item, so a long task
+ /// continues in a new work item after a temporary error instead of using up the retries of the current one.
+ ///
+ public Task ContinueAsync(BulkOperationWorkItem workItem) => _taskService.ContinueAsync(workItem);
+
+ public async Task GetPinnedSelectionAsync(string id)
+ {
+ var cached = await _cache.GetAsync(GetPinnedSelectionCacheKey(id));
+ return cached.HasValue ? cached.Value : null;
+ }
+
+ /// Keeps an active task's immutable selection available without rewriting its payload.
+ public Task RefreshSelectionExpirationAsync(BackgroundTask task)
+ {
+ return task.Bulk?.Selection.Kind is BulkOperationSelectionKind.Filter
+ ? _cache.SetExpirationAsync(GetPinnedSelectionCacheKey(task.Id), BackgroundTaskService.TaskLifetime)
+ : Task.CompletedTask;
+ }
+
+ /// Releases a rejected or terminal task's selection without changing its durable outcome.
+ public async Task ReleaseSelectionAsync(BackgroundTask task, string? pinnedSelection = null, string? cursor = null)
+ {
+ if (task.Bulk?.Selection.Kind is not BulkOperationSelectionKind.Filter
+ || GetHandler(task.Type) is not { } handler || !_resolvers.TryGetValue(handler.Entity, out var resolver))
+ return;
+
+ try
+ {
+ pinnedSelection ??= await GetPinnedSelectionAsync(task.Id);
+ await resolver.ReleaseAsync(task, pinnedSelection, cursor ?? task.Bulk.Cursor);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, "Could not release the selection for bulk operation {TaskId}: {Message}", task.Id, ex.Message);
+ }
+ }
+
+ ///
+ /// Stores and enqueues the task unless it conflicts with an active destructive task on the same project.
+ /// Admission uses a renewed per-organization lease; a scan that outlives its lease returns busy.
+ ///
+ public async Task CreateAsync(BackgroundTask task)
+ {
+ var handler = GetHandler(task.Type) ?? throw new ArgumentException($"Unknown bulk operation type '{task.Type}'.", nameof(task));
+ var bulk = task.Bulk ?? throw new ArgumentException("A bulk task needs its bulk details.", nameof(task));
+
+ task.CanCancel = true;
+
+ string? pinnedSelection = null;
+ bool enqueueAttempted = false;
+ bool retainSelection = false;
+ try
+ {
+ // Count and pin before taking the lock, so a slow search never holds up other requests.
+ if (bulk.Selection.Kind is BulkOperationSelectionKind.Filter)
+ {
+ if (!_resolvers.TryGetValue(handler.Entity, out var resolver))
+ throw new InvalidOperationException($"No selection resolver is registered for {handler.Entity}.");
+
+ task.CreatedUtc = _timeProvider.GetUtcNow().UtcDateTime;
+ var preparation = await resolver.PrepareAsync(task, CancellationToken.None);
+ pinnedSelection = preparation.PinnedState;
+ if (!preparation.Check.IsOk)
+ return BulkOperationCreateResult.Rejected(preparation.Check);
+
+ task.Total = preparation.Total;
+ }
+ else
+ {
+ task.Total = bulk.Selection.Ids?.Count ?? 0;
+ }
+
+ task.Description = handler.Describe(task);
+
+ long leaseStarted = _timeProvider.GetTimestamp();
+ await using var createLock = await _lockProvider.TryAcquireAsync(BackgroundTaskService.GetCreateLockResource(task.OrganizationId), CreateLockLifetime, TimeSpan.FromSeconds(10));
+ if (createLock is null)
+ return BulkOperationCreateResult.Busy();
+
+ async Task RenewCreateLockAsync()
+ {
+ long renewalStarted = _timeProvider.GetTimestamp();
+ if (_timeProvider.GetElapsedTime(leaseStarted, renewalStarted) >= CreateLockLifetime)
+ return false;
+
+ await createLock.RenewAsync(CreateLockLifetime);
+ if (_timeProvider.GetElapsedTime(leaseStarted) >= CreateLockLifetime)
+ return false;
+
+ leaseStarted = renewalStarted;
+ return true;
+ }
+
+ // Start conservatively before acquisition, then establish a fresh lease. A slow scan may renew it, but an
+ // expired request must not admit work from its earlier snapshot after another creator has acquired the lock.
+ if (!await RenewCreateLockAsync())
+ return BulkOperationCreateResult.Busy();
+
+ if (handler.IsDestructive)
+ {
+ var conflict = await FindConflictingTaskAsync(task, RenewCreateLockAsync);
+ if (conflict is not null)
+ return conflict;
+ }
+
+ if (!await RenewCreateLockAsync())
+ return BulkOperationCreateResult.Busy();
+
+ await _taskService.AddAsync(task);
+ enqueueAttempted = true;
+ await _taskService.EnqueueAsync(task, new BulkOperationWorkItem
+ {
+ Description = task.Description,
+ TaskId = task.Id,
+ OrganizationId = task.OrganizationId,
+ Type = task.Type
+ }, async () =>
+ {
+ // A selection that cannot be saved must fail the task just like an enqueue failure, so an unqueued task
+ // cannot remain active and block the project's next operation.
+ if (pinnedSelection is not null && !await _cache.SetAsync(GetPinnedSelectionCacheKey(task.Id), pinnedSelection, BackgroundTaskService.TaskLifetime))
+ throw new InvalidOperationException("The operation's selection could not be saved.");
+ });
+
+ retainSelection = true;
+ return BulkOperationCreateResult.Created(task);
+ }
+ finally
+ {
+ if (!retainSelection && pinnedSelection is not null)
+ {
+ var cleanupTask = task;
+ if (enqueueAttempted)
+ {
+ try
+ {
+ // An enqueue can succeed before its acknowledgement is lost. The worker's saved state wins;
+ // never close an accepted active task's snapshot just because the creating request failed.
+ var saved = await _taskService.GetAsync(task.Id);
+ retainSelection = saved?.IsActive == true;
+ cleanupTask = saved ?? task;
+ }
+ catch (Exception ex)
+ {
+ retainSelection = true;
+ _logger.LogWarning(ex, "Could not check bulk operation {TaskId} before releasing its selection: {Message}", task.Id, ex.Message);
+ }
+ }
+
+ if (!retainSelection)
+ await ReleaseSelectionAsync(cleanupTask, pinnedSelection);
+ }
+ }
+ }
+
+ ///
+ /// Tells clients that a task can no longer run because its record is gone (for example the cache was cleared).
+ /// The work item carries enough to publish a final state so the progress panel does not wait forever.
+ ///
+ public Task PublishLostAsync(BulkOperationWorkItem workItem)
+ {
+ return _taskService.PublishAsync(new BackgroundTaskUpdated
+ {
+ Description = workItem.Description ?? "Bulk operation",
+ Id = workItem.TaskId,
+ Message = "The operation's progress was lost and it could not continue.",
+ OrganizationId = workItem.OrganizationId,
+ Status = BackgroundTaskStatus.Failed,
+ Type = workItem.Type ?? String.Empty,
+ UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime
+ });
+ }
+
+ private async Task FindConflictingTaskAsync(BackgroundTask candidate, Func> renewCreateLock)
+ {
+ // Only active tasks can conflict, and every one of them is in the active index no matter how much history
+ // the organization has.
+ var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
+ BackgroundTask? conflict = null;
+ var activeIds = await _taskService.GetActiveIdsAsync(candidate.OrganizationId);
+ foreach (var batch in activeIds.Chunk(BackgroundTaskService.ReadBatchSize))
+ {
+ if (!await renewCreateLock())
+ return BulkOperationCreateResult.Busy();
+
+ var tasks = await _taskService.GetByIdsAsync(batch);
+ var finished = batch.Except(tasks.Where(task => task.IsActive).Select(task => task.Id), StringComparer.Ordinal).ToArray();
+ foreach (var active in tasks.Where(task => task.IsActive))
+ {
+ var existing = active;
+ if (GetHandler(existing.Type) is not { IsDestructive: true })
+ continue;
+
+ // An organization wide task can touch every project, so it conflicts with any project's task.
+ bool overlaps = existing.ProjectId is null || candidate.ProjectId is null || String.Equals(existing.ProjectId, candidate.ProjectId, StringComparison.Ordinal);
+ if (!overlaps)
+ continue;
+
+ if (utcNow - existing.UpdatedUtc >= BackgroundTaskService.StaleAfter)
+ {
+ if (!await renewCreateLock())
+ return BulkOperationCreateResult.Busy();
+
+ // Retire it before admitting a replacement, so a delayed delivery cannot run it alongside the new one.
+ var retired = await _taskService.UpdateAsync(existing.Id, current =>
+ {
+ if (!current.IsActive || utcNow - current.UpdatedUtc < BackgroundTaskService.StaleAfter)
+ return false;
+
+ current.Finish(BackgroundTaskStatus.Failed, "Stopped because it made no progress for two hours.", utcNow);
+ return true;
+ });
+
+ if (retired is null || !retired.IsActive)
+ continue;
+
+ existing = retired;
+ }
+
+ conflict ??= existing;
+ }
+
+ if (finished.Length > 0)
+ await _taskService.PruneIndexesAsync(candidate.OrganizationId, finished);
+ }
+
+ return conflict is null ? null : BulkOperationCreateResult.Conflicting(conflict);
+ }
+}
+
+public record BulkOperationCreateResult
+{
+ public BackgroundTask? Task { get; init; }
+ public BackgroundTask? Conflict { get; init; }
+ public bool IsBusy { get; init; }
+ public BulkOperationCheckResult? Rejection { get; init; }
+
+ public static BulkOperationCreateResult Created(BackgroundTask task) => new() { Task = task };
+ public static BulkOperationCreateResult Conflicting(BackgroundTask conflict) => new() { Conflict = conflict };
+ public static BulkOperationCreateResult Busy() => new() { IsBusy = true };
+ public static BulkOperationCreateResult Rejected(BulkOperationCheckResult rejection) => new() { Rejection = rejection };
+}
diff --git a/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs
new file mode 100644
index 0000000000..148414a812
--- /dev/null
+++ b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs
@@ -0,0 +1,73 @@
+using Exceptionless.Core.Models.BackgroundTasks;
+using Exceptionless.Core.Models.BulkOperations;
+using Exceptionless.Core.Repositories;
+using Foundatio.Repositories;
+
+namespace Exceptionless.Core.Services.BulkOperations;
+
+///
+/// Deletes events. Each batch goes through , the same code the single-item
+/// DELETE /events/{ids} endpoint uses, so usage accounting and notifications are identical.
+///
+public class DeleteEventsBulkOperationHandler : IBulkOperationHandler
+{
+ public const string OperationType = "events.delete";
+
+ private readonly IEventRepository _eventRepository;
+ private readonly EventDeletionService _eventDeletionService;
+
+ public DeleteEventsBulkOperationHandler(IEventRepository eventRepository, EventDeletionService eventDeletionService)
+ {
+ _eventRepository = eventRepository;
+ _eventDeletionService = eventDeletionService;
+ }
+
+ public string Type => OperationType;
+ public BulkOperationEntity Entity => BulkOperationEntity.Events;
+ public bool IsDestructive => true;
+
+ public Task AuthorizeAsync(BulkOperationRequest request)
+ {
+ // Same rule as DELETE /events/{ids}: any user of the organization (or a global admin) may delete its events.
+ if (!request.Actor.CanAccessOrganization(request.Organization.Id))
+ return Task.FromResult(BulkOperationCheckResult.Denied("Organization not found."));
+
+ if (request.Project is not null && !String.Equals(request.Project.OrganizationId, request.Organization.Id, StringComparison.Ordinal))
+ return Task.FromResult(BulkOperationCheckResult.Denied("Project not found."));
+
+ return Task.FromResult(BulkOperationCheckResult.Ok);
+ }
+
+ public Task ValidateAsync(BulkOperationRequest request)
+ {
+ if (request.Parameters.Count > 0)
+ return Task.FromResult(BulkOperationCheckResult.Invalid("parameters", $"The {Type} operation does not take parameters."));
+
+ return Task.FromResult(BulkOperationCheckResult.Ok);
+ }
+
+ public string Describe(BackgroundTask task)
+ {
+ return task.Bulk!.Selection.Kind is BulkOperationSelectionKind.Ids
+ ? $"Delete {task.Total:N0} selected events"
+ : $"Delete {task.Total:N0} events matching filter";
+ }
+
+ public async Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken)
+ {
+ // Deletion needs scope, daily-index routing and notification fields, not the event's potentially large payload.
+ var events = await _eventRepository.GetByIdsAsync(ids.ToArray(), o => o.Cache(false)
+ .Include(e => e.Id, e => e.Date, e => e.OrganizationId, e => e.ProjectId, e => e.StackId));
+
+ // The task may only touch its own organization (and project), no matter what ids it was given.
+ var inScope = events
+ .Where(e => String.Equals(e.OrganizationId, task.OrganizationId, StringComparison.Ordinal)
+ && (task.ProjectId is null || String.Equals(e.ProjectId, task.ProjectId, StringComparison.Ordinal)))
+ .ToList();
+
+ await _eventDeletionService.DeleteAsync(inScope);
+
+ int failed = ids.Count - inScope.Count;
+ return new BulkBatchResult(inScope.Count, failed, failed > 0 ? $"{failed:N0} events were not found." : null);
+ }
+}
diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs
new file mode 100644
index 0000000000..c091aac15f
--- /dev/null
+++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs
@@ -0,0 +1,374 @@
+using System.Globalization;
+using Elastic.Clients.Elasticsearch.Mapping;
+using Elastic.Clients.Elasticsearch.QueryDsl;
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.BackgroundTasks;
+using Exceptionless.Core.Models.BulkOperations;
+using Exceptionless.Core.Queries.Validation;
+using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Repositories.Base;
+using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Repositories.Options;
+using Exceptionless.Core.Repositories.Queries;
+using Exceptionless.Core.Services.BackgroundTasks;
+using Exceptionless.DateTimeExtensions;
+using Foundatio.Repositories;
+using Foundatio.Repositories.Elasticsearch;
+using Foundatio.Repositories.Elasticsearch.Extensions;
+using Foundatio.Repositories.Extensions;
+using Foundatio.Repositories.Models;
+using Foundatio.Repositories.Options;
+using Foundatio.Serializer;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Services.BulkOperations;
+
+///
+/// Pages the ids of events that match an organization or project scope, a filter and an absolute UTC window, using the
+/// same query building as the events list so the selection is exactly what the user was looking at.
+///
+public class EventBulkSelectionResolver : IBulkSelectionResolver
+{
+ private const string CreatedUtcField = "created_utc";
+ private const string PremiumSearchUpgradeMessage = "Please upgrade your plan to use premium search features.";
+ private const string SnapshotUnavailableMessage = "The saved event selection expired or is unavailable. Please start the operation again.";
+ private readonly IEventRepository _eventRepository;
+ private readonly IOrganizationRepository _organizationRepository;
+ private readonly IProjectRepository _projectRepository;
+ private readonly PersistentEventQueryValidator _validator;
+ private readonly ITextSerializer _serializer;
+ private readonly ExceptionlessElasticConfiguration _configuration;
+ private readonly AppOptions _appOptions;
+ private readonly BulkOperationOptions _options;
+ private readonly ILogger _logger;
+
+ public EventBulkSelectionResolver(IEventRepository eventRepository, IOrganizationRepository organizationRepository, IProjectRepository projectRepository,
+ PersistentEventQueryValidator validator, ITextSerializer serializer, ExceptionlessElasticConfiguration configuration, AppOptions appOptions, BulkOperationOptions options)
+ {
+ _eventRepository = eventRepository;
+ _organizationRepository = organizationRepository;
+ _projectRepository = projectRepository;
+ _validator = validator;
+ _serializer = serializer;
+ _configuration = configuration;
+ _appOptions = appOptions;
+ _options = options;
+ _logger = configuration.LoggerFactory.CreateLogger();
+ }
+
+ private ValueTask IsDateFieldAsync(string field)
+ {
+ return ValueTask.FromResult(_configuration.Events.MappingResolver.GetMappingProperty(field, followAlias: true) is DateProperty);
+ }
+
+ public BulkOperationEntity Entity => BulkOperationEntity.Events;
+
+ public async Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection)
+ {
+ var result = await _validator.ValidateQueryAsync(selection.Filter);
+ if (!result.IsValid)
+ return BulkOperationCheckResult.Invalid("selection.filter", result.Message ?? "Invalid filter.");
+
+ if (result.UsesPremiumFeatures && !organization.HasPremiumFeatures)
+ return BulkOperationCheckResult.PlanLimit(PremiumSearchUpgradeMessage);
+
+ return BulkOperationCheckResult.Ok;
+ }
+
+ public async Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken)
+ {
+ var (appFilter, check) = await GetAppFilterAsync(task);
+ if (!check.IsOk)
+ return BulkSelectionPreparation.Rejected(check);
+
+ if (appFilter is null)
+ return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, 0,
+ _serializer.SerializeToString(new PinnedEventSelection { Version = 1, IsEmpty = true }));
+
+ // Counting and paging must use the same relative-date anchor, even when preparation runs after the selected day.
+ string? frozenFilter = await RelativeDateFreezer.FreezeAsync(task.Bulk!.Selection.Filter,
+ task.Bulk.Selection.UtcCreatedBefore ?? task.CreatedUtc, IsDateFieldAsync);
+
+ var pinned = CreatePinnedSelection(appFilter);
+ // Resolve stack terms once and open an event snapshot in the same search that counts the selection. A created-at
+ // cutoff alone cannot pin mutable event fields such as the value of an active session or its user description.
+ var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection, pinned)
+ .FilterExpression(frozenFilter)
+ .Include(e => e.Id)
+ .SortExpression($"-{EventIndex.Alias.Date}")
+ .EnforceEventStackFilter();
+
+ string? pointInTimeId = null;
+ bool retainSnapshot = false;
+ try
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var results = await _eventRepository.FindAsync(query, SnapshotOptions(Math.Max(1, _options.BatchSize)).TrackTotalHits());
+ pointInTimeId = results.GetPointInTimeId();
+ cancellationToken.ThrowIfCancellationRequested();
+ if (String.IsNullOrEmpty(pointInTimeId))
+ return BulkSelectionPreparation.Rejected(SnapshotUnavailable());
+
+ pinned.EventFilter = query.GetFilterExpression();
+ pinned.ExcludedStackIds = query.GetExcludedStacks().ToArray();
+ pinned.StackIds = query.GetStacks().ToArray();
+ pinned.Total = results.Total;
+ pinned.IsEmpty = results.Total == 0;
+
+ string[] ids = GetIds(results);
+ if (results.Total <= ids.Length)
+ {
+ // The repository may already have closed a snapshot that fits on one page. Keep its complete, bounded
+ // membership instead of retaining a server resource for a small operation.
+ pinned.Ids = ids;
+ }
+ else
+ {
+ pinned.PointInTimeId = pointInTimeId;
+ }
+
+ string state = _serializer.SerializeToString(pinned);
+ retainSnapshot = pinned.PointInTimeId is not null;
+ return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, results.Total, state);
+ }
+ catch (DocumentLimitExceededException ex)
+ {
+ return BulkSelectionPreparation.Rejected(BulkOperationCheckResult.Invalid("selection.filter", ex.Message));
+ }
+ finally
+ {
+ if (!retainSnapshot)
+ await CloseSnapshotAsync(pointInTimeId, task.Id);
+ }
+ }
+
+ public async Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken)
+ {
+ var pinned = ReadPinnedSelection(pinnedState);
+ var position = String.IsNullOrEmpty(cursor) ? new SelectionCursor() : _serializer.Deserialize(cursor)
+ ?? throw new BulkSelectionRejectedException(SnapshotUnavailable());
+ if (position.Position < 0)
+ throw new BulkSelectionRejectedException(SnapshotUnavailable());
+
+ // A final checkpoint can survive a host restart before completion is recorded. There is no remaining action to
+ // authorize in that case, and a later plan change must not turn already completed work into a failed task.
+ if (position.Exhausted || pinned.IsEmpty || pinned.Ids is { Length: 0 })
+ return new BulkSelectionPage([], null);
+
+ var (appFilter, check) = await GetAppFilterAsync(task);
+ if (!check.IsOk)
+ throw new BulkSelectionRejectedException(check);
+
+ if (appFilter is null)
+ return new BulkSelectionPage([], null);
+
+ cancellationToken.ThrowIfCancellationRequested();
+ size = Math.Max(1, size);
+ var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection, pinned).Include(e => e.Id);
+
+ if (pinned.Ids is not null)
+ {
+ // Recheck current access policy, but never re-evaluate mutable user predicates. Search every remaining
+ // bounded id so a missing or newly excluded prefix does not hide eligible ids on a later page.
+ string[] remaining = pinned.Ids.Skip((int)Math.Min(position.Position, pinned.Ids.Length)).ToArray();
+ if (remaining.Length == 0)
+ return new BulkSelectionPage([], null);
+
+ var available = await _eventRepository.FindAsync(query.Id(remaining), new CommandOptions().PageLimit(remaining.Length).TrackTotalHits(false));
+ var allowed = GetIds(available).ToHashSet(StringComparer.Ordinal);
+ string[] ids = remaining.Where(allowed.Contains).Take(size).ToArray();
+ int next = ids.Length == 0 ? pinned.Ids.Length : Array.IndexOf(pinned.Ids, ids[^1]) + 1;
+ return new BulkSelectionPage(ids, _serializer.SerializeToString(new SelectionCursor { Position = next, Exhausted = next >= pinned.Ids.Length }));
+ }
+
+ query.FilterExpression(pinned.EventFilter).SortExpression($"-{EventIndex.Alias.Date}");
+
+ if (pinned.StackIds.Length > 0)
+ query.Stack(pinned.StackIds);
+
+ if (pinned.ExcludedStackIds.Length > 0)
+ query.ExcludeStack(pinned.ExcludedStackIds);
+
+ // Always supply the saved snapshot id. Missing or expired state must never open a new live selection.
+ string pointInTimeId = position.PointInTimeId ?? pinned.PointInTimeId!;
+ if (String.IsNullOrEmpty(pointInTimeId) || (position.Position > 0 && String.IsNullOrEmpty(position.After)))
+ throw new BulkSelectionRejectedException(SnapshotUnavailable());
+
+ var options = SnapshotOptions(size).PointInTimeId(pointInTimeId).SearchAfterToken(position.After, _serializer).TrackTotalHits(false);
+ var results = await _eventRepository.FindAsync(query, options);
+ // Foundatio represents a missing PIT (404) as an empty result. A valid PIT response includes its id even when
+ // current retention policy has narrowed the selection to no matches.
+ string? nextPointInTimeId = results.GetPointInTimeId();
+ if (String.IsNullOrEmpty(nextPointInTimeId))
+ throw new BulkSelectionRejectedException(SnapshotUnavailable());
+
+ string[] pageIds = GetIds(results);
+ long nextPosition = position.Position + pageIds.Length;
+ string? after = results.Hits.LastOrDefault()?.GetSortToken(_serializer);
+ bool exhausted = !results.HasMore || nextPosition >= pinned.Total;
+ if (!exhausted && String.IsNullOrEmpty(after))
+ throw new BulkSelectionRejectedException(SnapshotUnavailable());
+
+ // Retain the latest PIT id even on the last page, so terminal cleanup and a restart after the final checkpoint
+ // can close the correct resource without replaying the selection.
+ return new BulkSelectionPage(pageIds, _serializer.SerializeToString(new SelectionCursor
+ {
+ After = after,
+ PointInTimeId = nextPointInTimeId,
+ Position = nextPosition,
+ Exhausted = exhausted
+ }));
+ }
+
+ public async Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor)
+ {
+ if (pinnedState is null)
+ return;
+
+ try
+ {
+ var pinned = _serializer.Deserialize(pinnedState);
+ if (String.IsNullOrEmpty(pinned?.PointInTimeId))
+ return;
+
+ var position = String.IsNullOrEmpty(cursor) ? null : _serializer.Deserialize(cursor);
+ await CloseSnapshotAsync(position?.PointInTimeId ?? pinned.PointInTimeId, task.Id);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, "Could not release the event selection for task {TaskId}: {Message}", task.Id, ex.Message);
+ }
+ }
+
+ private async Task CloseSnapshotAsync(string? pointInTimeId, string taskId)
+ {
+ if (String.IsNullOrEmpty(pointInTimeId) || _eventRepository is not ISupportPointInTime repository)
+ return;
+
+ try
+ {
+ await repository.ClosePointInTimeAsync(pointInTimeId);
+ }
+ catch (Exception ex)
+ {
+ // Expiry is the bounded fallback if best-effort cleanup cannot reach Elasticsearch.
+ _logger.LogWarning(ex, "Could not close the event snapshot for task {TaskId}: {Message}", taskId, ex.Message);
+ }
+ }
+
+ private static CommandOptions SnapshotOptions(int size) => new CommandOptions()
+ .SearchAfterPaging(SearchAfterPagingMode.PointInTime)
+ // A queued task is stale after this idle period. Each page renews the same lease for active work and retries.
+ .SnapshotPagingLifetime(BackgroundTaskService.StaleAfter)
+ .PageLimit(size);
+
+ private static string[] GetIds(FindResults results) => results.Hits
+ .Select(hit => hit.Id ?? throw new BulkSelectionRejectedException(SnapshotUnavailable())).ToArray();
+
+ private static BulkOperationCheckResult SnapshotUnavailable() => BulkOperationCheckResult.Invalid("selection", SnapshotUnavailableMessage);
+
+ private PinnedEventSelection ReadPinnedSelection(string? state)
+ {
+ var pinned = state is null ? null : _serializer.Deserialize(state);
+ // Records created before snapshots were introduced contain only filters and stack ids. Replaying them would
+ // broaden the confirmed membership, so those tasks must be restarted instead of silently upgrading in place.
+ if (pinned is null || pinned.Version != 1 || (!pinned.IsEmpty && pinned.Ids is null && String.IsNullOrEmpty(pinned.PointInTimeId)))
+ throw new BulkSelectionRejectedException(SnapshotUnavailable());
+
+ return pinned;
+ }
+
+ private PinnedEventSelection CreatePinnedSelection(AppFilter appFilter)
+ {
+ var organization = appFilter.Organizations.Single();
+ var project = appFilter.Projects?.SingleOrDefault();
+ var timeProvider = _configuration.TimeProvider;
+ var utcNow = timeProvider.GetUtcNow().UtcDateTime;
+ var retentionDate = organization.GetRetentionUtcCutoff(_appOptions.MaximumRetentionDays,
+ project?.CreatedUtc.SafeSubtract(TimeSpan.FromDays(3)), timeProvider);
+ // Keep AppFilterQueryBuilder's date-math rounding, including the inclusive upper day's rounding. Pinning only
+ // the submitted time range could admit older events after a retention upgrade or project-scope change.
+ double retentionDays = Math.Max(Math.Round(Math.Abs(utcNow.Subtract(retentionDate).TotalDays), MidpointRounding.AwayFromZero), 1);
+ string anchor = utcNow.ToString("yyyy-MM-ddTHH:mm:ss.fff'Z'", CultureInfo.InvariantCulture) + "||";
+ return new PinnedEventSelection
+ {
+ Version = 1,
+ RetentionStart = FormattableString.Invariant($"{anchor}/d-{(int)retentionDays}d"),
+ RetentionEnd = $"{anchor}/d+1d"
+ };
+ }
+
+ private static RepositoryQuery ApplyScope(RepositoryQuery query, AppFilter appFilter, BulkOperationSelection selection, PinnedEventSelection pinned)
+ {
+ query.AppFilter(appFilter);
+ if (pinned.RetentionStart is not null && pinned.RetentionEnd is not null)
+ query.ElasticFilter(new DateRangeQuery { Field = EventIndex.Alias.Date, Gte = pinned.RetentionStart, Lte = pinned.RetentionEnd });
+
+ if (selection.UtcStart.HasValue || selection.UtcEnd.HasValue)
+ {
+ query.DateRange(selection.UtcStart, selection.UtcEnd, EventIndex.Alias.Date)
+ .Index(selection.UtcStart, selection.UtcEnd);
+ }
+
+ // The event date is chosen by the client, so an old-dated event can still arrive after the task was created.
+ if (selection.UtcCreatedBefore.HasValue)
+ query.DateRange(null, selection.UtcCreatedBefore, CreatedUtcField);
+
+ return query;
+ }
+
+ private async Task<(AppFilter? Filter, BulkOperationCheckResult Check)> GetAppFilterAsync(BackgroundTask task)
+ {
+ var organization = await _organizationRepository.GetByIdAsync(task.OrganizationId, o => o.Cache());
+ if (organization is null)
+ return (null, BulkOperationCheckResult.Ok);
+
+ var validation = await _validator.ValidateQueryAsync(task.Bulk!.Selection.Filter);
+ AppFilter appFilter;
+ if (!String.IsNullOrEmpty(task.ProjectId))
+ {
+ var project = await _projectRepository.GetByIdAsync(task.ProjectId, o => o.Cache());
+ if (project is null || !String.Equals(project.OrganizationId, organization.Id, StringComparison.Ordinal))
+ return (null, BulkOperationCheckResult.Ok);
+
+ appFilter = new AppFilter(project, organization);
+ }
+ else
+ {
+ appFilter = new AppFilter(organization);
+ }
+
+ // AppFilter turns an unavailable premium search into no matches. Report the plan change explicitly so the
+ // worker cannot mistake it for an exhausted selection and claim the deletion completed.
+ if (validation.UsesPremiumFeatures && !organization.HasPremiumFeatures)
+ return (null, BulkOperationCheckResult.PlanLimit(PremiumSearchUpgradeMessage));
+
+ appFilter.UsesPremiumFeatures = validation.UsesPremiumFeatures;
+ return (appFilter, BulkOperationCheckResult.Ok);
+ }
+
+ /// A bounded id set or the query and snapshot that fix the originally confirmed membership.
+ private sealed class PinnedEventSelection
+ {
+ public int Version { get; set; }
+ // Storage serialization omits empty arrays, so an intentionally empty membership needs its own marker.
+ public bool IsEmpty { get; set; }
+ public string[]? Ids { get; set; }
+ public string? PointInTimeId { get; set; }
+ public long Total { get; set; }
+ public string? RetentionStart { get; set; }
+ public string? RetentionEnd { get; set; }
+ public string? EventFilter { get; set; }
+ public string[] ExcludedStackIds { get; set; } = [];
+ public string[] StackIds { get; set; } = [];
+ }
+
+ private sealed class SelectionCursor
+ {
+ public string? After { get; set; }
+ public string? PointInTimeId { get; set; }
+ public long Position { get; set; }
+ public bool Exhausted { get; set; }
+ }
+}
diff --git a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs
new file mode 100644
index 0000000000..0cfd1f64eb
--- /dev/null
+++ b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs
@@ -0,0 +1,110 @@
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.BackgroundTasks;
+using Exceptionless.Core.Models.BulkOperations;
+
+namespace Exceptionless.Core.Services.BulkOperations;
+
+///
+/// One bulk action, e.g. events.delete. Adding a new bulk action means adding one implementation of this
+/// interface; handlers are discovered and registered automatically.
+///
+public interface IBulkOperationHandler
+{
+ /// The unique key that identifies the action in API requests and stored operations.
+ string Type { get; }
+
+ /// The kind of item the action targets. Selects the that pages ids.
+ BulkOperationEntity Entity { get; }
+
+ /// Destructive actions are limited to one active task per project.
+ bool IsDestructive { get; }
+
+ /// Checks that the actor may run this action against the organization and project.
+ Task AuthorizeAsync(BulkOperationRequest request);
+
+ /// Checks the action specific parameters and the selection shape.
+ Task ValidateAsync(BulkOperationRequest request);
+
+ /// A short, human readable description shown while the task runs.
+ string Describe(BackgroundTask task);
+
+ ///
+ /// Executes one batch of ids through the same code path as the single-item action, so side effects stay identical.
+ /// Ids that cannot be processed (missing, or outside the task's organization and project) must be reported as failed.
+ ///
+ ///
+ /// Handlers must tolerate at-least-once execution: a process can stop after the action but before its checkpoint is
+ /// saved, so the same ids may be delivered again. Missing ids must still be reported as failed.
+ ///
+ Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken);
+}
+
+/// Pages the ids that match a filter selection. One implementation per .
+public interface IBulkSelectionResolver
+{
+ BulkOperationEntity Entity { get; }
+
+ /// Checks that the filter of a selection can be run for the organization (syntax, plan limits).
+ Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection);
+
+ ///
+ /// Runs once when the task is created. Counts what matches and pins every part of the selection that could
+ /// otherwise change while the task runs, so pages never expand beyond what the user confirmed. Current access
+ /// policy may still narrow that membership. The pinned state is stored with the task for its lifetime.
+ ///
+ Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken);
+
+ ///
+ /// Returns the next page of ids after . The cursor is a value based position, so
+ /// paging stays correct while earlier items are being deleted or changed. Paging must not count totals.
+ ///
+ Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken);
+
+ ///
+ /// Best-effort release of selection resources when creation is rejected or a task is terminal. Active retries and
+ /// host shutdown keep their resources so they can resume. Implementations must tolerate duplicate cleanup.
+ ///
+ Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor) => Task.CompletedTask;
+}
+
+public record BulkSelectionPreparation(BulkOperationCheckResult Check, long Total = 0, string? PinnedState = null)
+{
+ public static BulkSelectionPreparation Rejected(BulkOperationCheckResult check) => new(check);
+}
+
+public record BulkSelectionPage(IReadOnlyList Ids, string? NextCursor);
+
+/// A previously accepted selection can no longer run with its saved membership or current access policy.
+public sealed class BulkSelectionRejectedException(BulkOperationCheckResult check) : InvalidOperationException(check.Message)
+{
+ public BulkOperationCheckResult Check { get; } = check;
+}
+
+public record BulkBatchResult(int Succeeded, int Failed, string? Message = null);
+
+/// The authenticated caller, independent of the transport the request arrived on.
+public record BulkOperationActor(string UserId, bool IsGlobalAdmin, IReadOnlyCollection OrganizationIds)
+{
+ public bool CanAccessOrganization(string organizationId) => IsGlobalAdmin || OrganizationIds.Contains(organizationId, StringComparer.Ordinal);
+}
+
+public record BulkOperationRequest(BulkOperationActor Actor, Organization Organization, Project? Project, string Type, BulkOperationSelection Selection, IReadOnlyDictionary Parameters);
+
+public enum BulkOperationCheckStatus
+{
+ Ok,
+ Denied,
+ Invalid,
+ PlanLimit
+}
+
+public record BulkOperationCheckResult(BulkOperationCheckStatus Status, string? Field = null, string? Message = null)
+{
+ public static readonly BulkOperationCheckResult Ok = new(BulkOperationCheckStatus.Ok);
+
+ public bool IsOk => Status is BulkOperationCheckStatus.Ok;
+
+ public static BulkOperationCheckResult Denied(string message) => new(BulkOperationCheckStatus.Denied, null, message);
+ public static BulkOperationCheckResult Invalid(string field, string message) => new(BulkOperationCheckStatus.Invalid, field, message);
+ public static BulkOperationCheckResult PlanLimit(string message) => new(BulkOperationCheckStatus.PlanLimit, null, message);
+}
diff --git a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs
new file mode 100644
index 0000000000..efe88962de
--- /dev/null
+++ b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs
@@ -0,0 +1,141 @@
+using System.Globalization;
+using Exceptionless.Core.Repositories.Queries;
+using Foundatio.Parsers.LuceneQueries;
+using Foundatio.Parsers.LuceneQueries.Nodes;
+using Foundatio.Parsers.LuceneQueries.Visitors;
+
+namespace Exceptionless.Core.Services.BulkOperations;
+
+///
+/// Pins relative dates in a filter (for example data.date:[now/d-4d TO now/d+1d}) to a fixed time, so a bulk
+/// operation keeps selecting the same events no matter when each batch runs. Only date fields are changed: a string range
+/// such as environment:[now TO z] is left alone.
+///
+public static class RelativeDateFreezer
+{
+ /// The filter to pin.
+ /// The time that now stands for.
+ /// Whether a built-in (non custom data) field is mapped as a date.
+ public static async Task FreezeAsync(string? filter, DateTime utcNow, Func> isDateFieldAsync)
+ {
+ if (String.IsNullOrWhiteSpace(filter) || filter.IndexOf("now", StringComparison.OrdinalIgnoreCase) < 0)
+ return filter;
+
+ var result = await new LuceneQueryParser().ParseAsync(filter);
+ if (result is null)
+ return filter;
+
+ await new FreezeRelativeDatesVisitor(utcNow, isDateFieldAsync).AcceptAsync(result, new QueryVisitorContext());
+ return result.ToString();
+ }
+
+ private sealed class FreezeRelativeDatesVisitor(DateTime utcNow, Func> isDateFieldAsync) : ChainableQueryVisitor
+ {
+ // Elasticsearch date math anchors an expression on a date with ||, for example 2026-11-03T04:00:00.000Z||/d-4d.
+ private readonly string _anchor = utcNow.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ", CultureInfo.InvariantCulture) + "||";
+
+ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext context)
+ {
+ // A custom field on a group is stored by the type of the group's terms, as the events index decides it. When that
+ // is a date, name the stored date field so the frozen bounds below keep the date type.
+ string? storedField = EventFieldsQueryVisitor.GetCustomFieldName(node.Field, EventFieldsQueryVisitor.GetGroupChildTerms(node));
+ if (IsStoredDateField(storedField))
+ node.Field = storedField;
+
+ foreach (var child in node.Children)
+ await child.AcceptAsync(this, context);
+ }
+
+ public override async Task VisitAsync(TermRangeNode node, IQueryVisitorContext context)
+ {
+ if (!IsRelative(node.Min) && !IsRelative(node.Max))
+ return;
+
+ if (!await EnsureDateFieldAsync(node, [node.Min, node.Max]))
+ return;
+
+ if (IsRelative(node.Min))
+ {
+ node.Min = _anchor + node.Min![3..];
+ node.IsMinQuotedTerm = true;
+ }
+
+ if (IsRelative(node.Max))
+ {
+ node.Max = _anchor + node.Max![3..];
+ node.IsMaxQuotedTerm = true;
+ }
+ }
+
+ public override async Task VisitAsync(TermNode node, IQueryVisitorContext context)
+ {
+ if (node.IsRegexTerm || node.Proximity is not null || !IsRelative(node.Term) || node.Term!.IndexOfAny(['*', '?']) >= 0
+ || !await EnsureDateFieldAsync(node, [node.Term]))
+ return;
+
+ // Elasticsearch treats a date term such as now/d as an inclusive date range, so it needs the same anchor.
+ node.Term = _anchor + node.Term![3..];
+ node.IsQuotedTerm = true;
+ }
+
+ private async ValueTask EnsureDateFieldAsync(IFieldQueryNode node, string?[] terms)
+ {
+ // An unfielded term or range inside a group takes its field from the group.
+ string? field = node.Field ?? GetGroupField(node);
+ if (String.IsNullOrEmpty(field))
+ return false;
+
+ if (node.Field is null)
+ {
+ // The group already decided the stored field: a custom field left unchanged there is not a date.
+ if (!IsStoredDateField(field) && (IsCustomField(field) || !await isDateFieldAsync(field)))
+ return false;
+ }
+ else
+ {
+ // Custom data fields are stored by type. Freezing their original terms would hide the date type,
+ // so name the stored date field directly.
+ string? storedField = EventFieldsQueryVisitor.GetCustomFieldName(field, terms);
+ if (IsCustomField(field))
+ {
+ if (!IsStoredDateField(storedField))
+ return false;
+
+ node.Field = storedField;
+ }
+ else if (!IsStoredDateField(field) && !await isDateFieldAsync(field))
+ {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private static bool IsCustomField(string field)
+ {
+ return !String.Equals(EventFieldsQueryVisitor.GetCustomFieldName(field, []), field, StringComparison.Ordinal);
+ }
+
+ private static bool IsStoredDateField(string? field)
+ {
+ return field is not null && field.StartsWith("idx.", StringComparison.OrdinalIgnoreCase) && field.EndsWith("-d", StringComparison.Ordinal);
+ }
+
+ private static string? GetGroupField(IQueryNode node)
+ {
+ for (var parent = node.Parent; parent is not null; parent = parent.Parent)
+ {
+ if (parent is GroupNode { Field: { Length: > 0 } groupField })
+ return groupField;
+ }
+
+ return null;
+ }
+
+ private static bool IsRelative(string? value)
+ {
+ return value is not null && value.StartsWith("now", StringComparison.Ordinal) && (value.Length == 3 || value[3] is '+' or '-' or '/');
+ }
+ }
+}
diff --git a/src/Exceptionless.Core/Services/EventDeletionService.cs b/src/Exceptionless.Core/Services/EventDeletionService.cs
new file mode 100644
index 0000000000..382cb91db9
--- /dev/null
+++ b/src/Exceptionless.Core/Services/EventDeletionService.cs
@@ -0,0 +1,46 @@
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Repositories;
+using Foundatio.Repositories;
+using Microsoft.Extensions.Logging;
+
+namespace Exceptionless.Core.Services;
+
+///
+/// Removes events and records the deleted usage. Shared by the single-item delete endpoint and the bulk delete
+/// operation so both have identical side effects.
+///
+public class EventDeletionService
+{
+ private readonly IEventRepository _eventRepository;
+ private readonly UsageService _usageService;
+ private readonly ILogger _logger;
+
+ public EventDeletionService(IEventRepository eventRepository, UsageService usageService, ILogger logger)
+ {
+ _eventRepository = eventRepository;
+ _usageService = usageService;
+ _logger = logger;
+ }
+
+ /// Deletes events the caller has already verified access to.
+ public async Task DeleteAsync(IReadOnlyCollection events)
+ {
+ if (events.Count == 0)
+ return;
+
+ await _eventRepository.RemoveAsync(events);
+
+ foreach (var projectGroup in events.GroupBy(ev => new { ev.OrganizationId, ev.ProjectId }))
+ {
+ try
+ {
+ await _usageService.IncrementDeletedAsync(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, projectGroup.Count());
+ }
+ catch (Exception ex)
+ {
+ // Accounting has no caller cancellation token. A transport timeout must not replay a completed delete.
+ _logger.LogWarning(ex, "Failed to increment deleted usage metrics for organization {OrganizationId} project {ProjectId}: {Message}", projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, ex.Message);
+ }
+ }
+ }
+}
diff --git a/src/Exceptionless.Core/Utility/SampleDataService.cs b/src/Exceptionless.Core/Utility/SampleDataService.cs
index 859540416c..7028801175 100644
--- a/src/Exceptionless.Core/Utility/SampleDataService.cs
+++ b/src/Exceptionless.Core/Utility/SampleDataService.cs
@@ -2,8 +2,10 @@
using Exceptionless.Core.Billing;
using Exceptionless.Core.Extensions;
using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.BackgroundTasks;
using Exceptionless.Core.Models.WorkItems;
using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Services.BackgroundTasks;
using Foundatio.Jobs;
using Foundatio.Queues;
using Foundatio.Repositories;
@@ -21,6 +23,7 @@ public class SampleDataService
private readonly BillingPlans _billingPlans;
private readonly TimeProvider _timeProvider;
private readonly IUserRepository _userRepository;
+ private readonly BackgroundTaskService _backgroundTaskService;
private readonly ILogger _logger;
public const string TEST_USER_EMAIL = "admin@exceptionless.test";
@@ -48,6 +51,7 @@ public SampleDataService(
IUserRepository userRepository,
ITokenRepository tokenRepository,
IQueue workItemQueue,
+ BackgroundTaskService backgroundTaskService,
BillingManager billingManager,
BillingPlans billingPlans,
TimeProvider timeProvider,
@@ -59,6 +63,7 @@ ILoggerFactory loggerFactory
_userRepository = userRepository;
_tokenRepository = tokenRepository;
_workItemQueue = workItemQueue;
+ _backgroundTaskService = backgroundTaskService;
_billingManager = billingManager;
_billingPlans = billingPlans;
_timeProvider = timeProvider;
@@ -287,17 +292,28 @@ await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem
_logger.LogInformation("Enqueued sample event generation: {EventCount} events over {DaysBack} days", eventCount, daysBack);
}
- public async Task EnqueueSampleEventsAsync(string organizationId, string projectId, int eventCount = 250, int daysBack = 7)
+ ///
+ /// Queues sample event generation for one project and returns the id of the background task that tracks it.
+ ///
+ public async Task EnqueueSampleEventsAsync(Project project, string createdByUserId, int eventCount = 250, int daysBack = 7)
{
- string workItemId = await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem
+ var task = await _backgroundTaskService.CreateAndEnqueueAsync(new BackgroundTask
{
- OrganizationId = organizationId,
- ProjectId = projectId,
+ OrganizationId = project.OrganizationId,
+ ProjectId = project.Id,
+ Type = BackgroundTaskTypes.GenerateProjectSampleData,
+ Description = $"Generate sample data for {project.Name}",
+ CreatedByUserId = createdByUserId
+ }, t => new GenerateSampleEventsWorkItem
+ {
+ OrganizationId = project.OrganizationId,
+ ProjectId = project.Id,
EventCount = eventCount,
- DaysBack = daysBack
+ DaysBack = daysBack,
+ TaskId = t.Id
});
- _logger.LogInformation("Enqueued sample event generation for project {ProjectId}: {EventCount} events over {DaysBack} days", projectId, eventCount, daysBack);
- return workItemId;
+ _logger.LogInformation("Enqueued sample event generation for project {ProjectId}: {EventCount} events over {DaysBack} days", project.Id, eventCount, daysBack);
+ return task.Id;
}
}
diff --git a/src/Exceptionless.Web/Api/ApiEndpoints.cs b/src/Exceptionless.Web/Api/ApiEndpoints.cs
index 540b0abfaa..4063b21e59 100644
--- a/src/Exceptionless.Web/Api/ApiEndpoints.cs
+++ b/src/Exceptionless.Web/Api/ApiEndpoints.cs
@@ -16,6 +16,7 @@ public static WebApplication MapApiEndpoints(this WebApplication app)
app.MapWebHookEndpoints();
app.MapStripeEndpoints();
app.MapSavedViewEndpoints();
+ app.MapBackgroundTaskEndpoints();
app.MapUserEndpoints();
app.MapProjectEndpoints();
app.MapOrganizationEndpoints();
diff --git a/src/Exceptionless.Web/Api/Endpoints/BackgroundTaskEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/BackgroundTaskEndpoints.cs
new file mode 100644
index 0000000000..3af9e51308
--- /dev/null
+++ b/src/Exceptionless.Web/Api/Endpoints/BackgroundTaskEndpoints.cs
@@ -0,0 +1,102 @@
+using Exceptionless.Core.Authorization;
+using Exceptionless.Web.Api.Filters;
+using Exceptionless.Web.Api.Results;
+using Exceptionless.Web.Models;
+using Exceptionless.Web.Utility;
+using Exceptionless.Web.Utility.OpenApi;
+using Foundatio.Mediator;
+using Microsoft.AspNetCore.Mvc;
+using TaskMessages = Exceptionless.Web.Api.Messages;
+using HttpIResult = Microsoft.AspNetCore.Http.IResult;
+
+namespace Exceptionless.Web.Api.Endpoints;
+
+public static class BackgroundTaskEndpoints
+{
+ public static IEndpointRouteBuilder MapBackgroundTaskEndpoints(this IEndpointRouteBuilder endpoints)
+ {
+ var group = endpoints.MapGroup("api/v2")
+ .RequireAuthorization(AuthorizationRoles.UserPolicy)
+ .AddEndpointFilter()
+ .WithTags("BackgroundTask");
+
+ group.MapPost("bulk-operations", async (IMediator mediator, IMediatorResultMapper resultMapper, [FromBody] NewBulkOperation operation)
+ => (await mediator.InvokeAsync>>(new TaskMessages.CreateBulkOperation(operation))).ToHttpResult(resultMapper))
+ .Accepts("application/json", "application/*+json")
+ .Produces(StatusCodes.Status202Accepted)
+ .ProducesProblem(StatusCodes.Status403Forbidden)
+ .ProducesProblem(StatusCodes.Status404NotFound)
+ .ProducesProblem(StatusCodes.Status409Conflict)
+ .ProducesProblem(StatusCodes.Status422UnprocessableEntity)
+ .ProducesProblem(StatusCodes.Status426UpgradeRequired)
+ .WithSummary("Create")
+ .WithDescription("""
+ Starts an action, such as `events.delete`, on everything that matches a selection. The selection is either a list of ids
+ or a filter with a time range. The time range is resolved to absolute UTC dates when the operation is created, and events
+ stored after that moment are never included. The operation runs in the background and is returned as a task; poll
+ `GET api/v2/tasks/{id}` or listen for `BackgroundTaskUpdated` messages to follow its progress.
+ """)
+ .WithMetadata(new EndpointDocumentation {
+ RequestBodyDescription = "The bulk operation to start.",
+ ResponseDescriptions = new() {
+ ["202"] = "Accepted",
+ ["403"] = "The action is not permitted for this organization or project.",
+ ["404"] = "The organization or project could not be found.",
+ ["409"] = "A destructive operation is already running for this project.",
+ ["422"] = "The request is not valid.",
+ ["426"] = "The filter uses a feature that requires a plan upgrade.",
+ }
+ });
+
+ group.MapGet("tasks/{id:objectid}", async (string id, IMediator mediator, IMediatorResultMapper resultMapper)
+ => (await mediator.InvokeAsync>(new TaskMessages.GetBackgroundTaskById(id))).ToHttpResult(resultMapper))
+ .WithName("GetBackgroundTaskById")
+ .Produces()
+ .ProducesProblem(StatusCodes.Status404NotFound)
+ .WithSummary("Get by id")
+ .WithDescription("Returns the progress of a task, such as a bulk action, a project data reset or sample data generation. Tasks are kept for seven days.")
+ .WithMetadata(new EndpointDocumentation {
+ ParameterDescriptions = new() {
+ ["id"] = "The identifier of the task.",
+ },
+ ResponseDescriptions = new() {
+ ["404"] = "The task could not be found.",
+ }
+ });
+
+ group.MapGet("organizations/{organizationId:objectid}/tasks", async (string organizationId, IMediator mediator, IMediatorResultMapper resultMapper)
+ => (await mediator.InvokeAsync>>(new TaskMessages.GetBackgroundTasksByOrganization(organizationId))).ToHttpResult(resultMapper))
+ .Produces>()
+ .ProducesProblem(StatusCodes.Status404NotFound)
+ .WithSummary("Get by organization")
+ .WithDescription("Returns the active and recently finished tasks of the organization, newest first. Finished tasks are kept for seven days.")
+ .WithMetadata(new EndpointDocumentation {
+ ParameterDescriptions = new() {
+ ["organizationId"] = "The identifier of the organization.",
+ },
+ ResponseDescriptions = new() {
+ ["404"] = "The organization could not be found.",
+ }
+ });
+
+ group.MapPost("tasks/{id:objectid}/cancel", async (string id, IMediator mediator, IMediatorResultMapper resultMapper)
+ => (await mediator.InvokeAsync>>(new TaskMessages.CancelBackgroundTask(id))).ToHttpResult(resultMapper))
+ .Produces(StatusCodes.Status202Accepted)
+ .ProducesProblem(StatusCodes.Status404NotFound)
+ .ProducesProblem(StatusCodes.Status422UnprocessableEntity)
+ .WithSummary("Cancel")
+ .WithDescription("Requests cancellation. A running bulk action stops after the batch it is processing; work already done is not undone. Cancelling a finished task has no effect. Tasks that cannot be stopped, such as a project data reset, are rejected.")
+ .WithMetadata(new EndpointDocumentation {
+ ParameterDescriptions = new() {
+ ["id"] = "The identifier of the task.",
+ },
+ ResponseDescriptions = new() {
+ ["202"] = "Accepted",
+ ["404"] = "The task could not be found.",
+ ["422"] = "The task cannot be cancelled.",
+ }
+ });
+
+ return endpoints;
+ }
+}
diff --git a/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs
new file mode 100644
index 0000000000..e08c7d7b79
--- /dev/null
+++ b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs
@@ -0,0 +1,372 @@
+using System.Text.RegularExpressions;
+using Exceptionless.Core;
+using Exceptionless.Core.Extensions;
+using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.BackgroundTasks;
+using Exceptionless.Core.Models.BulkOperations;
+using Exceptionless.Core.Repositories;
+using Exceptionless.Core.Repositories.Configuration;
+using Exceptionless.Core.Services.BackgroundTasks;
+using Exceptionless.Core.Services.BulkOperations;
+using Exceptionless.DateTimeExtensions;
+using Exceptionless.DateTimeExtensions.FormatParsers;
+using Exceptionless.DateTimeExtensions.FormatParsers.PartParsers;
+using Exceptionless.Web.Api.Infrastructure;
+using Exceptionless.Web.Api.Messages;
+using Exceptionless.Web.Api.Results;
+using Exceptionless.Web.Extensions;
+using Exceptionless.Web.Models;
+using Foundatio.Mediator;
+using Foundatio.Repositories;
+using Foundatio.Repositories.Utility;
+
+namespace Exceptionless.Web.Api.Handlers;
+
+public class BackgroundTaskHandler(
+ BackgroundTaskService backgroundTaskService,
+ BulkOperationService bulkOperationService,
+ IEnumerable resolvers,
+ IOrganizationRepository organizationRepository,
+ IProjectRepository projectRepository,
+ AppOptions appOptions,
+ TimeProvider timeProvider,
+ IHttpContextAccessor httpContextAccessor,
+ ILogger logger)
+{
+ private const int MaxIds = 1000;
+
+ private HttpContext HttpContext => httpContextAccessor.HttpContext ?? throw new InvalidOperationException("HttpContext is unavailable.");
+
+ public async Task>> Handle(CreateBulkOperation message)
+ {
+ var request = HttpContext.Request;
+ var model = message.Operation;
+
+ var handler = bulkOperationService.GetHandler(model.Type);
+ if (handler is null)
+ return Result.Invalid(ValidationError.Create("type", $"Unknown bulk operation type '{model.Type}'."));
+
+ // Organization and project access mirrors the single-item actions: not accessible looks the same as not found.
+ if (!request.CanAccessOrganization(model.OrganizationId))
+ return Result.NotFound("Organization not found.");
+
+ var organization = await organizationRepository.GetByIdAsync(model.OrganizationId, o => o.Cache());
+ if (organization is null)
+ return Result.NotFound("Organization not found.");
+
+ Project? project = null;
+ if (!String.IsNullOrEmpty(model.ProjectId))
+ {
+ project = await projectRepository.GetByIdAsync(model.ProjectId, o => o.Cache());
+ if (project is null || !String.Equals(project.OrganizationId, organization.Id, StringComparison.Ordinal))
+ return Result.NotFound("Project not found.");
+ }
+
+ var user = request.GetUser();
+ var actor = new BulkOperationActor(user.Id, request.IsGlobalAdmin(), request.GetAssociatedOrganizationIds().ToList());
+ var parameters = model.Parameters ?? new Dictionary();
+
+ var selectionResult = ResolveSelection(model.Selection, organization);
+ if (selectionResult.Result is not null)
+ return selectionResult.Result;
+
+ var selection = selectionResult.Selection!;
+ var bulkRequest = new BulkOperationRequest(actor, organization, project, handler.Type, selection, parameters);
+
+ var authorized = await handler.AuthorizeAsync(bulkRequest);
+ if (!authorized.IsOk)
+ return ToResult>(authorized);
+
+ var validated = await handler.ValidateAsync(bulkRequest);
+ if (!validated.IsOk)
+ return ToResult>(validated);
+
+ var task = new BackgroundTask
+ {
+ OrganizationId = organization.Id,
+ ProjectId = project?.Id,
+ Type = handler.Type,
+ CreatedByUserId = user.Id,
+ Bulk = new BulkOperationDetails
+ {
+ Selection = selection,
+ Parameters = new Dictionary(parameters)
+ }
+ };
+
+ if (selection.Kind is BulkOperationSelectionKind.Filter)
+ {
+ var resolver = resolvers.FirstOrDefault(r => r.Entity == handler.Entity);
+ if (resolver is null)
+ return Result.Error($"No selection resolver is registered for {handler.Entity}.");
+
+ var selectionCheck = await resolver.ValidateSelectionAsync(organization, selection);
+ if (!selectionCheck.IsOk)
+ return ToResult>(selectionCheck);
+ }
+
+ // Counting and pinning the selection (and describing the operation) happens while it is created.
+ var created = await bulkOperationService.CreateAsync(task);
+ if (created.Rejection is not null)
+ return ToResult>(created.Rejection);
+
+ if (created.Conflict is not null)
+ return Result.Conflict("Another destructive operation is already running for this project. Wait for it to finish or cancel it first.");
+
+ if (created.Task is null)
+ return Result.Conflict("Another bulk operation is being started for this organization. Please try again.");
+
+ using var _ = logger.BeginScope(new ExceptionlessState().Organization(organization.Id).Tag("BulkOperation").Identity(user.EmailAddress));
+ logger.LogInformation("User {User} started bulk operation {TaskId} {Type} for {Total} items", user.Id, created.Task.Id, created.Task.Type, created.Task.Total);
+
+ return new AcceptedResponse(MapToViewModel(created.Task));
+ }
+
+ public async Task> Handle(GetBackgroundTaskById message)
+ {
+ var task = await GetAccessibleAsync(message.Id);
+ if (task is null)
+ return Result.NotFound("Task not found.");
+
+ return MapToViewModel(task);
+ }
+
+ public async Task>> Handle(GetBackgroundTasksByOrganization message)
+ {
+ if (!HttpContext.Request.CanAccessOrganization(message.OrganizationId))
+ return Result.NotFound("Organization not found.");
+
+ var tasks = await backgroundTaskService.GetByOrganizationAsync(message.OrganizationId);
+ return Result>.Success(tasks.Select(MapToViewModel).ToList());
+ }
+
+ public async Task>> Handle(CancelBackgroundTask message)
+ {
+ var task = await GetAccessibleAsync(message.Id);
+ if (task is null)
+ return Result.NotFound("Task not found.");
+
+ var cancelled = await backgroundTaskService.CancelAsync(task.Id);
+ if (cancelled.Task is null)
+ return Result.NotFound("Task not found.");
+
+ if (cancelled.IsNotCancellable)
+ return Result.Invalid(ValidationError.Create("task", "This task cannot be cancelled."));
+
+ if (!cancelled.Task.IsActive)
+ await bulkOperationService.ReleaseSelectionAsync(cancelled.Task);
+
+ logger.LogInformation("User {User} requested cancellation of task {TaskId}", HttpContext.Request.GetUser().Id, cancelled.Task.Id);
+ return new AcceptedResponse(MapToViewModel(cancelled.Task));
+ }
+
+ private async Task GetAccessibleAsync(string id)
+ {
+ var task = await backgroundTaskService.GetAsync(id);
+ if (task is null || !HttpContext.Request.CanAccessOrganization(task.OrganizationId))
+ return null;
+
+ return task;
+ }
+
+ private (BulkOperationSelection? Selection, Result? Result) ResolveSelection(NewBulkOperationSelection model, Organization organization)
+ {
+ var utcNow = timeProvider.GetUtcNow().UtcDateTime;
+ if (model.Kind is BulkOperationSelectionKind.Ids)
+ {
+ if (model.Filter is not null || model.Time is not null)
+ return (null, Result.Invalid(ValidationError.Create("selection", "An id selection cannot include a filter or time.")));
+
+ var ids = (model.Ids ?? []).Distinct(StringComparer.Ordinal).ToList();
+ if (ids.Count == 0)
+ return (null, Result.Invalid(ValidationError.Create("selection.ids", "At least one id is required.")));
+
+ if (ids.Count > MaxIds)
+ return (null, Result.Invalid(ValidationError.Create("selection.ids", $"At most {MaxIds} ids can be selected. Use a filter selection for more.")));
+
+ if (ids.Any(id => !ObjectId.TryParse(id, out _)))
+ return (null, Result.Invalid(ValidationError.Create("selection.ids", "Every id must be a valid identifier.")));
+
+ return (new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ids }, null);
+ }
+
+ // Only an explicit filter selection may resolve a filter. An unknown kind must never fall through to an empty filter,
+ // which would select everything in scope.
+ if (model.Kind is not BulkOperationSelectionKind.Filter)
+ return (null, Result.Invalid(ValidationError.Create("selection.kind", "The selection kind must be ids or filter.")));
+
+ if (model.Ids is { Count: > 0 })
+ return (null, Result.Invalid(ValidationError.Create("selection", "A filter selection cannot include ids.")));
+
+ if (!TryParseSelectionOffset(model.Offset, out var utcOffset))
+ return (null, Result.Invalid(ValidationError.Create("selection.offset", "The UTC offset must be a valid whole-minute offset between -14h and 14h.")));
+
+ // Validate the requested range before retention and open-end defaults can turn a parse failure into all history.
+ // Reuse the parsed range so validation and execution resolve relative dates at exactly the same instant.
+ if (!TryParseSelectionTime(model.Time, new DateTimeOffset(utcNow).ToOffset(utcOffset), out var range))
+ return (null, Result.Invalid(ValidationError.Create("selection.time", "The time range is not valid.")));
+
+ var timeInfo = new TimeInfo { Field = EventIndex.Alias.Date, Offset = utcOffset, Range = range! };
+ timeInfo.ApplyMinimumUtcStartDate(organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider));
+ timeInfo.AdjustEndTimeIfMaxValue(timeProvider);
+ var utcStart = timeInfo.Range.UtcStart;
+ var utcEnd = timeInfo.Range.UtcEnd > utcNow ? utcNow : timeInfo.Range.UtcEnd;
+ if (utcStart > utcEnd)
+ return (null, Result.Invalid(ValidationError.Create("selection.time", "The time range does not include any past time.")));
+
+ var selection = new BulkOperationSelection
+ {
+ Kind = BulkOperationSelectionKind.Filter,
+ Filter = model.Filter?.Trim() ?? String.Empty,
+ Time = model.Time,
+ UtcStart = utcStart,
+ UtcEnd = utcEnd,
+ UtcCreatedBefore = utcNow
+ };
+
+ return (selection, null);
+ }
+
+ internal static bool TryParseSelectionTime(string? time, DateTimeOffset relativeBaseTime, out DateTimeRange? range)
+ {
+ if (String.IsNullOrWhiteSpace(time))
+ {
+ range = DateTimeRange.Empty;
+ return true;
+ }
+
+ string expression = time.Trim();
+ if (expression.StartsWith($"{EventIndex.Alias.Date}|", StringComparison.Ordinal))
+ expression = expression[(EventIndex.Alias.Date.Length + 1)..];
+
+ // DateTimeRange.Parse uses the same unbounded sentinel for invalid input and a valid [* TO *] range.
+ // Its two-part parser also supplies missing bounds, so require both to have been parsed explicitly.
+ range = null;
+ try
+ {
+ foreach (var parser in DateTimeRange.FormatParsers)
+ {
+ if (parser is TwoPartFormatParser twoPartParser)
+ {
+ var bounds = new List(2);
+ var partParsers = twoPartParser.Parsers.Select(partParser => partParser is WildcardPartParser
+ ? new SelectionTimeWildcardPartParser(bounds)
+ : new SelectionTimePartParser(partParser, bounds));
+ range = new TwoPartFormatParser(partParsers).Parse(expression, relativeBaseTime);
+ if (range is not null && (bounds.Count != 2 || bounds[0] > bounds[1]))
+ {
+ range = null;
+ return false;
+ }
+ }
+ else
+ {
+ range = parser.Parse(expression, relativeBaseTime);
+ }
+
+ if (range is not null)
+ return true;
+ }
+ }
+ catch (Exception ex) when (ex is ArgumentException or FormatException or OverflowException)
+ {
+ range = null;
+ }
+
+ return false;
+ }
+
+ internal static bool TryParseSelectionOffset(string? offset, out TimeSpan utcOffset)
+ {
+ utcOffset = TimeSpan.Zero;
+ if (String.IsNullOrWhiteSpace(offset))
+ return true;
+
+ try
+ {
+ if (!TimeUnit.TryParse(offset, out var parsedOffset) || !parsedOffset.HasValue
+ || parsedOffset.Value < TimeSpan.FromHours(-14) || parsedOffset.Value > TimeSpan.FromHours(14)
+ || parsedOffset.Value.Ticks % TimeSpan.TicksPerMinute != 0)
+ return false;
+
+ utcOffset = parsedOffset.Value;
+ return true;
+ }
+ catch (Exception ex) when (ex is ArgumentException or FormatException or OverflowException)
+ {
+ return false;
+ }
+ }
+
+ private sealed class SelectionTimePartParser(IPartParser inner, List bounds) : IPartParser
+ {
+ public Regex Regex => inner.Regex;
+
+ public DateTimeOffset? Parse(Match match, DateTimeOffset relativeBaseTime, bool isUpperLimit)
+ {
+ var value = inner.Parse(match, relativeBaseTime, isUpperLimit);
+ if (value.HasValue)
+ bounds.Add(value.Value);
+
+ return value;
+ }
+ }
+
+ // The two-part parser checks this type to keep wildcard bounds open regardless of bracket inclusivity.
+ private sealed class SelectionTimeWildcardPartParser(List bounds) : WildcardPartParser, IPartParser
+ {
+ DateTimeOffset? IPartParser.Parse(Match match, DateTimeOffset relativeBaseTime, bool isUpperLimit)
+ {
+ var value = base.Parse(match, relativeBaseTime, isUpperLimit);
+ if (value.HasValue)
+ bounds.Add(value.Value);
+
+ return value;
+ }
+ }
+
+ private static Result ToResult(BulkOperationCheckResult check)
+ {
+ return check.Status switch
+ {
+ BulkOperationCheckStatus.Denied => Result.Forbidden(check.Message ?? "Forbidden"),
+ BulkOperationCheckStatus.PlanLimit => Result.Invalid(ValidationError.Create(ApiValidationErrorIdentifiers.PlanLimit, check.Message ?? "Please upgrade your plan.")),
+ _ => Result.Invalid(ValidationError.Create(check.Field ?? "operation", check.Message ?? "Invalid request."))
+ };
+ }
+
+ private static ViewBackgroundTask MapToViewModel(BackgroundTask task)
+ {
+ var selection = task.Bulk?.Selection;
+ return new ViewBackgroundTask
+ {
+ Id = task.Id,
+ OrganizationId = task.OrganizationId,
+ ProjectId = task.ProjectId,
+ Type = task.Type,
+ Description = task.Description,
+ Status = task.Status,
+ Progress = task.Progress,
+ Total = task.Total,
+ Processed = task.Processed,
+ Succeeded = task.Succeeded,
+ Failed = task.Failed,
+ CanCancel = task.CanCancel,
+ Selection = selection is null ? null : new ViewBulkOperationSelection
+ {
+ Kind = selection.Kind,
+ IdCount = selection.Kind is BulkOperationSelectionKind.Ids ? selection.Ids?.Count : null,
+ Filter = selection.Filter,
+ Time = selection.Time,
+ UtcStart = selection.UtcStart,
+ UtcEnd = selection.UtcEnd
+ },
+ CreatedByUserId = task.CreatedByUserId,
+ CreatedUtc = task.CreatedUtc,
+ StartedUtc = task.StartedUtc,
+ CompletedUtc = task.CompletedUtc,
+ UpdatedUtc = task.UpdatedUtc,
+ Message = task.Message
+ };
+ }
+}
diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs
index 1b9a8f4029..633c62dde5 100644
--- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs
+++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs
@@ -49,6 +49,7 @@ public class EventHandler(
AppOptions appOptions,
SystemSettingsService systemSettingsService,
UsageService usageService,
+ EventDeletionService eventDeletionService,
TimeProvider timeProvider,
LinkGenerator linkGenerator,
ILoggerFactory loggerFactory)
@@ -665,23 +666,7 @@ public async Task> Handle(DeleteEvents message)
_logger.LogInformation("User {User} deleted {RemovedCount} events in project ({ProjectId})", currentUser.Id, projectGroup.Count(), ev.ProjectId);
}
- await eventRepository.RemoveAsync(list);
-
- foreach (var projectGroup in projectGroups)
- {
- try
- {
- await usageService.IncrementDeletedAsync(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, projectGroup.Count());
- }
- catch (OperationCanceledException)
- {
- throw;
- }
- catch (Exception ex)
- {
- _logger.LogWarning(ex, "Failed to increment deleted usage metrics for org {OrganizationId} project {ProjectId}: {Message}", projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, ex.Message);
- }
- }
+ await eventDeletionService.DeleteAsync(list.ToList());
if (results.Failure.Count == 0)
return new WorkInProgressResult();
diff --git a/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs b/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs
index 1d9d84d45c..e36706dc1c 100644
--- a/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs
+++ b/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs
@@ -3,10 +3,12 @@
using Exceptionless.Core.Billing;
using Exceptionless.Core.Extensions;
using Exceptionless.Core.Models;
+using Exceptionless.Core.Models.BackgroundTasks;
using Exceptionless.Core.Models.WorkItems;
using Exceptionless.Core.Repositories;
using Exceptionless.Core.Repositories.Queries;
using Exceptionless.Core.Services;
+using Exceptionless.Core.Services.BackgroundTasks;
using Exceptionless.Core.Utility;
using Exceptionless.Web.Api.Infrastructure;
using Exceptionless.Web.Api.Messages;
@@ -37,6 +39,7 @@ public class ProjectHandler(
SlackService slackService,
IOAuthProviderClient oauthProviderClient,
SampleDataService sampleDataService,
+ BackgroundTaskService backgroundTaskService,
ApiMapper mapper,
ITextSerializer serializer,
AppOptions options,
@@ -228,8 +231,8 @@ public async Task> Handle(GenerateProjectSampleData
if (project is null)
return Result.NotFound("Project not found.");
- string workItemId = await sampleDataService.EnqueueSampleEventsAsync(project.OrganizationId, project.Id);
- return new WorkInProgressResult([workItemId]);
+ string taskId = await sampleDataService.EnqueueSampleEventsAsync(project, GetCurrentUser(message.Context).Id);
+ return new WorkInProgressResult([taskId]);
}
public async Task> Handle(ResetProjectData message)
@@ -238,13 +241,22 @@ public async Task> Handle(ResetProjectData message)
if (project is null)
return Result.NotFound("Project not found.");
- string workItemId = await workItemQueue.EnqueueAsync(new ResetProjectDataWorkItem
+ // The returned id is the background task that tracks the reset, so the progress panel can follow it.
+ var task = await backgroundTaskService.CreateAndEnqueueAsync(new BackgroundTask
{
OrganizationId = project.OrganizationId,
- ProjectId = project.Id
+ ProjectId = project.Id,
+ Type = BackgroundTaskTypes.ResetProjectData,
+ Description = $"Reset data for {project.Name}",
+ CreatedByUserId = GetCurrentUser(message.Context).Id
+ }, t => new ResetProjectDataWorkItem
+ {
+ OrganizationId = project.OrganizationId,
+ ProjectId = project.Id,
+ TaskId = t.Id
});
- return new WorkInProgressResult([workItemId]);
+ return new WorkInProgressResult([task.Id]);
}
public async Task>> Handle(GetProjectNotificationSettings message)
diff --git a/src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs b/src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs
new file mode 100644
index 0000000000..26024a5207
--- /dev/null
+++ b/src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs
@@ -0,0 +1,8 @@
+using Exceptionless.Web.Models;
+
+namespace Exceptionless.Web.Api.Messages;
+
+public record CreateBulkOperation(NewBulkOperation Operation);
+public record GetBackgroundTaskById(string Id);
+public record GetBackgroundTasksByOrganization(string OrganizationId);
+public record CancelBackgroundTask(string Id);
diff --git a/src/Exceptionless.Web/Api/Results/AcceptedResponse.cs b/src/Exceptionless.Web/Api/Results/AcceptedResponse.cs
new file mode 100644
index 0000000000..aeb034f154
--- /dev/null
+++ b/src/Exceptionless.Web/Api/Results/AcceptedResponse.cs
@@ -0,0 +1,12 @@
+namespace Exceptionless.Web.Api.Results;
+
+/// Marks a successful handler result that is returned with the 202 Accepted status and a body.
+public interface IAcceptedResponse
+{
+ object? Body { get; }
+}
+
+public record AcceptedResponse(T Value) : IAcceptedResponse
+{
+ object? IAcceptedResponse.Body => Value;
+}
diff --git a/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs b/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs
index 0a0ac78919..374aee1fa4 100644
--- a/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs
+++ b/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs
@@ -152,6 +152,9 @@ private static IResult MapSuccess(Foundatio.Mediator.IResult result)
if (value is WorkInProgressResult)
return HttpResults.Json(value, statusCode: StatusCodes.Status202Accepted);
+ if (value is IAcceptedResponse accepted)
+ return HttpResults.Json(accepted.Body, statusCode: StatusCodes.Status202Accepted);
+
return HttpResults.Ok(value);
}
diff --git a/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts b/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts
index d6493e25c0..758aeb9fd8 100644
--- a/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts
+++ b/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts
@@ -38,10 +38,12 @@ export function waitForEventDetailResponses(page: Page, eventId: string, stackId
const url = new URL(response.url());
return url.pathname === `/api/v2/events/${eventId}` && url.searchParams.get('expected_stack_id') === stackId;
});
- const projectCountResponse = page.waitForResponse((response) => {
+ return Promise.all([eventResponse, waitForProjectUserCountResponse(page, projectId)]);
+}
+
+export function waitForProjectUserCountResponse(page: Page, projectId: string): Promise {
+ return page.waitForResponse((response) => {
const url = new URL(response.url());
return url.pathname === `/api/v2/projects/${projectId}/events/count` && url.searchParams.get('aggregations') === 'cardinality:user';
});
-
- return Promise.all([eventResponse, projectCountResponse]);
}
diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts
new file mode 100644
index 0000000000..0d17da7145
--- /dev/null
+++ b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts
@@ -0,0 +1,163 @@
+import { BackgroundTaskStatus, type ViewBackgroundTask } from '#generated/api.ts';
+
+import { expect, test } from '../fixtures/e2e-test';
+import { createRepresentativeEvent } from '../support/synthetic-event';
+
+const EVENT_COUNT = 5;
+const PAGE_SIZE = 2;
+
+test('many running tasks stay within the viewport and the oldest task can be cancelled after scrolling', async ({ e2eScenario, page }) => {
+ // Arrange
+ const now = Date.now();
+ const tasks: ViewBackgroundTask[] = Array.from({ length: 12 }, (_, index) => ({
+ can_cancel: true,
+ created_by_user_id: '000000000000000000000001',
+ created_utc: new Date(now - index * 1000).toISOString(),
+ description: `Synthetic bulk task ${index + 1}`,
+ failed: 0,
+ id: (index + 1).toString(16).padStart(24, '0'),
+ organization_id: e2eScenario.organizationId,
+ processed: 25,
+ progress: 25,
+ project_id: e2eScenario.projectId,
+ status: BackgroundTaskStatus.Running,
+ succeeded: 25,
+ total: 100,
+ type: 'events.delete',
+ updated_utc: new Date(now).toISOString()
+ }));
+ const oldest = tasks.at(-1)!;
+ await page.route(
+ (url) => url.pathname === `/api/v2/organizations/${e2eScenario.organizationId}/tasks`,
+ (route) => route.fulfill({ json: tasks })
+ );
+ await page.route(`**/api/v2/tasks/${oldest.id}/cancel`, async (route) => {
+ oldest.status = BackgroundTaskStatus.Cancelled;
+ oldest.updated_utc = new Date().toISOString();
+ await route.fulfill({ json: oldest, status: 202 });
+ });
+ await page.setViewportSize({ height: 400, width: 390 });
+ const panel = page.getByRole('region', { name: 'Background tasks' });
+ const oldestCancel = panel.getByRole('button', { exact: true, name: `Cancel ${oldest.description}` });
+
+ // Act
+ await page.goto(`/event?project=${e2eScenario.projectId}&time=all`);
+
+ // Assert
+ await expect(panel.getByRole('button', { name: /^Cancel Synthetic bulk task / })).toHaveCount(tasks.length);
+ const metrics = await panel.evaluate((element) => {
+ const bounds = element.getBoundingClientRect();
+ return {
+ bottom: bounds.bottom,
+ clientHeight: element.clientHeight,
+ scrollHeight: element.scrollHeight,
+ top: bounds.top,
+ viewportHeight: window.innerHeight
+ };
+ });
+ expect(metrics.top).toBeGreaterThanOrEqual(0);
+ expect(metrics.bottom).toBeLessThanOrEqual(metrics.viewportHeight);
+ expect(metrics.scrollHeight).toBeGreaterThan(metrics.clientHeight);
+ await expect(oldestCancel).not.toBeInViewport();
+
+ // Act
+ await oldestCancel.scrollIntoViewIfNeeded();
+
+ // Assert
+ await expect(oldestCancel).toBeInViewport();
+ expect(await panel.evaluate((element) => element.scrollTop)).toBeGreaterThan(0);
+
+ // Act
+ await oldestCancel.click();
+
+ // Assert
+ await expect(panel.getByRole('button', { exact: true, name: `Dismiss ${oldest.description}` })).toBeVisible();
+});
+
+test('deleting everything matching the filter runs in the background and shows progress', async ({ e2eApi, e2eScenario, page }) => {
+ // Arrange: more events than fit on one page, all in this test's own project.
+ const references = Array.from({ length: EVENT_COUNT }, (_, index) => `${e2eScenario.referenceId}-bulk-${index}`);
+ for (const referenceId of references) {
+ await e2eApi.submitEvent(
+ e2eScenario.projectId,
+ e2eScenario.projectToken,
+ createRepresentativeEvent({
+ appUrl: e2eApi.environment.appUrl,
+ message: `${e2eScenario.message} ${referenceId}`,
+ referenceId,
+ runId: e2eApi.environment.runId
+ })
+ );
+ }
+
+ for (const referenceId of references) {
+ await e2eApi.pollForEventByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId);
+ }
+
+ await page.goto(`/event?project=${e2eScenario.projectId}&time=all&limit=${PAGE_SIZE}`);
+ const table = page.locator('table:has(thead [role="checkbox"])').first();
+ await expect(table.getByRole('checkbox', { name: 'Select row' })).toHaveCount(PAGE_SIZE, { timeout: 30_000 });
+
+ await test.step('selecting the page offers to select everything matching the filter', async () => {
+ // Act
+ await table.getByRole('checkbox', { name: 'Select all rows' }).click();
+
+ // Assert
+ await expect(page.getByRole('status').filter({ hasText: `All ${PAGE_SIZE} events on this page are selected.` })).toBeVisible();
+ await expect(page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` })).toBeVisible();
+ });
+
+ await test.step('selecting everything matching the filter shows the total', async () => {
+ // Act
+ await page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` }).click();
+
+ // Assert
+ await expect(page.getByRole('status').filter({ hasText: `All ${EVENT_COUNT} events are selected` })).toBeVisible();
+ await expect(page.getByRole('button', { name: 'Clear selection' })).toBeVisible();
+ });
+
+ await test.step('clearing the selection deselects the page', async () => {
+ // Act
+ await page.getByRole('button', { name: 'Clear selection' }).click();
+
+ // Assert
+ await expect(page.getByRole('button', { name: 'Clear selection' })).toBeHidden();
+ await expect(table.getByRole('checkbox', { name: 'Select all rows' })).not.toBeChecked();
+ });
+
+ await test.step('deleting against the filter asks to confirm the exact count', async () => {
+ // Arrange
+ await table.getByRole('checkbox', { name: 'Select all rows' }).click();
+ await page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` }).click();
+
+ // Act
+ await page.getByRole('button', { name: 'Actions' }).click();
+ await page.getByRole('menuitem', { name: 'Delete' }).click();
+
+ // Assert
+ await expect(page.getByRole('alertdialog')).toContainText(`Delete ${EVENT_COUNT} events matching this filter? This can't be undone.`);
+ });
+
+ await test.step('confirming runs in the background and the progress panel reaches completion', async () => {
+ // Act
+ await page
+ .getByRole('alertdialog')
+ .getByRole('button', { name: `Delete ${EVENT_COUNT} Events` })
+ .click();
+
+ // Assert
+ const panel = page.getByRole('region', { name: 'Background tasks' });
+ await expect(panel).toContainText(/Delete \d+ events matching filter/, { timeout: 30_000 });
+ await expect(panel).toContainText(`Completed · ${EVENT_COUNT} processed`, { timeout: 60_000 });
+ await expect(table.getByRole('checkbox', { name: 'Select row' })).toHaveCount(0, { timeout: 30_000 });
+ });
+
+ await test.step('every event is gone', async () => {
+ // Assert
+ for (const referenceId of references) {
+ await expect
+ .poll(async () => (await e2eApi.getEventsByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId)).length, { timeout: 30_000 })
+ .toBe(0);
+ }
+ });
+});
diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts
index 6ecf710fb7..07a85b557e 100644
--- a/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts
+++ b/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts
@@ -1,7 +1,7 @@
import type { ConsoleMessage, Page, Request, Response } from '@playwright/test';
import { expect, test } from '../fixtures/e2e-test';
-import { isApplicationApiRequest, trackPendingApiRequests, waitForEventDetailResponses } from '../support/api-request-monitor';
+import { isApplicationApiRequest, trackPendingApiRequests, waitForEventDetailResponses, waitForProjectUserCountResponse } from '../support/api-request-monitor';
import { ExceptionlessE2EJourney } from '../support/exceptionless-journey';
import { createRepresentativeEvent } from '../support/synthetic-event';
import {
@@ -206,9 +206,19 @@ test('event list and detail effects stay bounded through paging and background c
await measureAction(diagnostics, 'stack detail discovery route remounts', async () => {
for (let index = 0; index < 5; index++) {
+ // Arrange
await pendingApiRequests.waitForPendingRequests();
+ // Overview can render before the dependent project metric request starts.
+ const projectCountResponse = waitForProjectUserCountResponse(page, e2eScenario.projectId);
+
+ // Act
await page.goto(`/stack/${journey.stackId}`);
+
+ // Assert
await expect(page.getByRole('tab', { name: 'Overview' })).toBeVisible();
+ const response = await projectCountResponse;
+ expect(response.ok()).toBe(true);
+ expect(await response.finished()).toBeNull();
}
});
expect(actionSample(diagnostics, 'stack detail discovery route remounts').eventDetails).toBeLessThanOrEqual(10);
@@ -217,9 +227,18 @@ test('event list and detail effects stay bounded through paging and background c
await measureAction(diagnostics, 'canonical stack event route remounts', async () => {
for (let index = 0; index < 5; index++) {
+ // Arrange
await pendingApiRequests.waitForPendingRequests();
+ const projectCountResponse = waitForProjectUserCountResponse(page, e2eScenario.projectId);
+
+ // Act
await page.goto(`/stack/${journey.stackId}/event/${journey.eventId}`);
+
+ // Assert
await expect(page.getByRole('tab', { name: 'Overview' })).toBeVisible();
+ const response = await projectCountResponse;
+ expect(response.ok()).toBe(true);
+ expect(await response.finished()).toBeNull();
}
});
expect(actionSample(diagnostics, 'canonical stack event route remounts').eventDetails).toBeLessThanOrEqual(10);
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts
index f14e187745..b498e6c4cc 100644
--- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts
+++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts
@@ -99,7 +99,7 @@ export async function invalidatePersistentEventQueries(queryClient: QueryClient,
}
export const queryKeys = {
- deleteEvent: (ids: string[] | undefined) => [...queryKeys.type, 'delete', ...(ids ?? [])] as const,
+ deleteEvent: () => [...queryKeys.type, 'delete'] as const,
eventsByReference: (referenceId: string | undefined, projectId?: string | undefined, params?: GetEventsByReferenceRequest['params']) =>
[...queryKeys.type, 'by-ref', referenceId, projectId, params] as const,
id: (id: string | undefined) => [...queryKeys.type, id] as const,
@@ -127,12 +127,6 @@ export const PERSISTENT_EVENT_DELETE_RECONCILE_RETRY_DELAY = 5000;
export const ORGANIZATION_EVENT_QUERY_STALE_TIME_MS = 60 * 1000;
export const ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS = 5 * 1000;
-export interface DeleteEventsRequest {
- route: {
- ids: string[] | undefined;
- };
-}
-
export interface EventNavigation {
nextId: null | string;
previousId: null | string;
@@ -335,26 +329,26 @@ export function createEventWithNavigationQueryOptions(request: GetEventRequest,
};
}
-export function deleteEvent(request: DeleteEventsRequest) {
+export function deleteEvent() {
const queryClient = useQueryClient();
- return createMutation(() => ({
- enabled: () => !!accessToken.current && !!request.route.ids?.length,
- mutationFn: async () => {
+ return createMutation(() => ({
+ enabled: () => !!accessToken.current,
+ mutationFn: async (ids) => {
const client = useFetchClient();
- const response = await client.deleteJSON(`events/${request.route.ids?.join(',')}`);
+ const response = await client.deleteJSON(`events/${ids.join(',')}`);
return response.data!;
},
- mutationKey: queryKeys.deleteEvent(request.route.ids),
- onError: () => {
- request.route.ids?.forEach((id) =>
+ mutationKey: queryKeys.deleteEvent(),
+ onError: (_, ids) => {
+ ids.forEach((id) =>
queryClient.invalidateQueries({
queryKey: queryKeys.id(id)
})
);
},
- onSuccess: () => {
- request.route.ids?.forEach((id) =>
+ onSuccess: (_, ids) => {
+ ids.forEach((id) =>
queryClient.invalidateQueries({
queryKey: queryKeys.id(id)
})
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts
index 26f88837ff..c692de219c 100644
--- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts
+++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts
@@ -1,6 +1,10 @@
-import { QueryClient } from '@tanstack/svelte-query';
+import type { ProblemDetails } from '@foundatiofx/fetchclient';
+
+import { MutationObserver, type MutationObserverOptions, onlineManager, QueryClient } from '@tanstack/svelte-query';
import { afterEach, describe, expect, it, vi } from 'vitest';
+import type { WorkInProgressResult } from '#shared/models/index.ts';
+
import { ChangeType } from '#features/websockets/models.ts';
import type { PersistentEvent } from './models';
@@ -9,6 +13,7 @@ import { queryKeys as stackQueryKeys } from '../stacks/api.svelte';
import {
createEventWithNavigationQueryOptions,
createOrganizationEventNotificationRefresher,
+ deleteEvent,
type GetEventRequest,
invalidatePersistentEventQueries,
ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS,
@@ -20,10 +25,19 @@ import {
schedulePersistentEventDeleteReconciliation
} from './api.svelte';
-const fetchClientMocks = vi.hoisted(() => ({ getJSON: vi.fn() }));
+const fetchClientMocks = vi.hoisted(() => ({ deleteJSON: vi.fn(), getJSON: vi.fn() }));
+const mutationMocks = vi.hoisted(() => ({
+ createMutation: vi.fn<(options: () => MutationObserverOptions) => void>(),
+ useQueryClient: vi.fn()
+}));
vi.mock('@foundatiofx/fetchclient', () => ({
- useFetchClient: () => ({ getJSON: fetchClientMocks.getJSON })
+ useFetchClient: () => fetchClientMocks
+}));
+vi.mock('@tanstack/svelte-query', async (importOriginal) => ({
+ ...(await importOriginal()),
+ createMutation: mutationMocks.createMutation,
+ useQueryClient: mutationMocks.useQueryClient
}));
afterEach(() => {
@@ -41,6 +55,46 @@ describe('retainPreviousOrganizationQueryData', () => {
});
});
+describe('deleteEvent', () => {
+ it('keeps the confirmed event IDs when a paused deletion resumes after the selection changes', async () => {
+ // Arrange
+ vi.useFakeTimers();
+ vi.stubGlobal('document', new EventTarget());
+ onlineManager.setOnline(false);
+ const client = new QueryClient();
+ const invalidate = vi.spyOn(client, 'invalidateQueries').mockResolvedValue(undefined);
+ mutationMocks.useQueryClient.mockReturnValue(client);
+ fetchClientMocks.deleteJSON.mockResolvedValue({ data: { workers: ['worker-id'] } });
+ const selection = ['event-a'];
+ deleteEvent();
+ const mutation = new MutationObserver(client, mutationMocks.createMutation.mock.calls.at(-1)![0]());
+
+ try {
+ // Act
+ const result = mutation.mutate([...selection]);
+
+ // Assert
+ expect(mutation.getCurrentResult().isPaused).toBe(true);
+ expect(fetchClientMocks.deleteJSON).not.toHaveBeenCalled();
+
+ // Act
+ selection.splice(0, selection.length, 'event-b');
+ onlineManager.setOnline(true);
+ await client.resumePausedMutations();
+ await result;
+
+ // Assert
+ expect(fetchClientMocks.deleteJSON).toHaveBeenCalledExactlyOnceWith('events/event-a');
+ expect(invalidate).toHaveBeenCalledWith({ queryKey: queryKeys.id('event-a') });
+ expect(invalidate).not.toHaveBeenCalledWith({ queryKey: queryKeys.id('event-b') });
+ } finally {
+ onlineManager.setOnline(true);
+ client.clear();
+ vi.unstubAllGlobals();
+ }
+ });
+});
+
describe('createOrganizationEventNotificationRefresher', () => {
it('defers removal reconciliation without a leading refetch', async () => {
// Arrange
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte
index 115a40f3de..e8e2855fe1 100644
--- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte
+++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte
@@ -5,11 +5,13 @@
interface Props {
count?: number;
+ /** The delete runs against everything matching the current filter instead of the selected rows. */
+ matchingFilter?: boolean;
open: boolean;
remove: () => Promise;
}
- let { count = 1, open = $bindable(), remove }: Props = $props();
+ let { count = 1, matchingFilter = false, open = $bindable(), remove }: Props = $props();
async function onSubmit() {
await remove();
@@ -29,12 +31,17 @@
{/if}
- Are you sure you want to delete
- {#if count === 1}
- this event
+ {#if matchingFilter}
+ Delete
+ {count === 1 ? 'event' : 'events'} matching this filter? This can't be undone.
{:else}
- events
- {/if}?
+ Are you sure you want to delete
+ {#if count === 1}
+ this event
+ {:else}
+ events
+ {/if}?
+ {/if}
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts
new file mode 100644
index 0000000000..7800dd442f
--- /dev/null
+++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts
@@ -0,0 +1,107 @@
+import { createTable, stockFeatures, type StockFeatures, type Table, tableFeatures } from '@tanstack/svelte-table';
+import { describe, expect, it } from 'vitest';
+
+import { createAllMatchingSelection, getEventsQueryIdentity } from './all-matching-selection.svelte';
+
+describe('createAllMatchingSelection', () => {
+ it('is active only for the query it was made for', () => {
+ // Arrange
+ const identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id', time: '[now-7d TO now]' });
+ const selection = createAllMatchingSelection(() => identity);
+
+ // Act
+ selection.select(1234);
+
+ // Assert
+ expect(selection.active).toEqual({ count: 1234, identity });
+ });
+
+ it.each([
+ ['filter', { filter: 'type:log', organizationId: 'organization-id', time: '[now-7d TO now]' }],
+ ['time range', { filter: 'type:error', organizationId: 'organization-id', time: '[now-30d TO now]' }],
+ ['project filter', { filter: 'type:error project:project-b', organizationId: 'organization-id', time: '[now-7d TO now]' }],
+ ['saved view', { filter: 'type:error', organizationId: 'organization-id', savedViewId: 'view-2', time: '[now-7d TO now]' }],
+ ['organization', { filter: 'type:error', organizationId: 'another-organization', time: '[now-7d TO now]' }]
+ ])('is dropped when the %s changes', (_, changed) => {
+ // Arrange
+ let identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id', time: '[now-7d TO now]' });
+ const selection = createAllMatchingSelection(() => identity);
+ selection.select(1234);
+
+ // Act
+ identity = getEventsQueryIdentity(changed);
+
+ // Assert
+ expect(selection.active).toBeUndefined();
+ });
+
+ it('ignores a count that was fetched for a filter that is no longer showing', () => {
+ // Arrange
+ let identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' });
+ const selection = createAllMatchingSelection(() => identity);
+ const selectionWhenCountingStarted = selection.snapshot;
+
+ // Act
+ identity = getEventsQueryIdentity({ filter: 'type:log', organizationId: 'organization-id' });
+ const selected = selection.select(1234, selectionWhenCountingStarted);
+
+ // Assert
+ expect(selected).toBe(false);
+ expect(selection.active).toBeUndefined();
+ });
+
+ it('does not come back when the original query is restored after being cleared', () => {
+ // Arrange
+ const original = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' });
+ let identity = original;
+ const selection = createAllMatchingSelection(() => identity);
+ selection.select(10);
+
+ // Act
+ identity = getEventsQueryIdentity({ filter: 'type:log', organizationId: 'organization-id' });
+ selection.clear();
+ identity = original;
+
+ // Assert
+ expect(selection.active).toBeUndefined();
+ });
+
+ it.each([
+ ['unchanged', false],
+ ['cleared and selected again', true]
+ ])('handles a deferred count when the page selection is %s', async (_, changeSelection) => {
+ // Arrange
+ const identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' });
+ let table: Table;
+ const dispose = $effect.root(() => {
+ table = createTable({
+ columns: [{ accessorKey: 'id' }],
+ data: [{ id: 'event-1' }, { id: 'event-2' }],
+ features: tableFeatures(stockFeatures),
+ getRowId: (row) => row.id
+ });
+ });
+ table!.toggleAllRowsSelected(true);
+ const selection = createAllMatchingSelection(
+ () => identity,
+ () => table!.store.state.rowSelection
+ );
+ const counting = Promise.withResolvers();
+ const selectionWhenCountingStarted = selection.snapshot;
+ const selecting = counting.promise.then((count) => selection.select(count, selectionWhenCountingStarted));
+
+ // Act
+ if (changeSelection) {
+ table!.resetRowSelection();
+ table!.toggleAllRowsSelected(true);
+ }
+ counting.resolve(1234);
+ const selected = await selecting;
+ dispose();
+
+ // Assert
+ expect(table!.getIsAllRowsSelected()).toBe(true);
+ expect(selected).toBe(!changeSelection);
+ expect(selection.active).toEqual(changeSelection ? undefined : { count: 1234, identity });
+ });
+});
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts
new file mode 100644
index 0000000000..3c41f18933
--- /dev/null
+++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts
@@ -0,0 +1,67 @@
+import type { RowSelectionState } from '@tanstack/svelte-table';
+
+export interface AllMatchingSelection {
+ /** How many events matched when the user chose "select all matching". */
+ count: number;
+ /** The query the selection was made for. */
+ identity: string;
+}
+
+export interface AllMatchingSelectionSnapshot {
+ identity: string;
+ rowSelection: RowSelectionState | undefined;
+}
+
+/** Everything matching the current filter, as passed to an action menu. */
+export interface AllMatchingTarget {
+ count: number;
+ filter?: string;
+ offset?: string;
+ organizationId: string;
+ time?: string;
+}
+
+/**
+ * Tracks the Gmail-style "everything matching this filter" selection. The selection belongs to the query it was made
+ * for: as soon as the query identity (organization, filter, time range, saved view) changes it is no longer active, so
+ * an action can never run against a different set of events than the user confirmed.
+ */
+export function createAllMatchingSelection(getIdentity: () => string, getRowSelection?: () => RowSelectionState) {
+ let selection = $state();
+
+ return {
+ get active(): AllMatchingSelection | undefined {
+ return selection && selection.identity === getIdentity() ? selection : undefined;
+ },
+ clear() {
+ selection = undefined;
+ },
+ /**
+ * Selects everything matching the current query. A pending count must not replace a newer filter or page
+ * selection, including clearing and then selecting the same rows again.
+ */
+ select(count: number, expected?: AllMatchingSelectionSnapshot): boolean {
+ const identity = getIdentity();
+ if (expected && (expected.identity !== identity || expected.rowSelection !== getRowSelection?.())) {
+ return false;
+ }
+
+ selection = {
+ count,
+ identity
+ };
+ return true;
+ },
+ /** Capture the query and page selection before awaiting a count, then pass the snapshot to select. */
+ get snapshot(): AllMatchingSelectionSnapshot {
+ return {
+ identity: getIdentity(),
+ rowSelection: getRowSelection?.()
+ };
+ }
+ };
+}
+
+export function getEventsQueryIdentity(parts: { filter?: null | string; organizationId?: string; savedViewId?: string; time?: null | string }): string {
+ return JSON.stringify([parts.organizationId ?? null, parts.filter ?? null, parts.time ?? null, parts.savedViewId ?? null]);
+}
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte
index fec3c8e401..046a9d8728 100644
--- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte
+++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte
@@ -3,6 +3,8 @@
@@ -56,10 +122,10 @@
{#snippet child({ props })}