From a59ac4f238475a2cf16bf2d1c8fbe21b3f8c671d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 18:31:08 -0600 Subject: [PATCH 01/42] Run bulk actions on everything matching a filter Add a generic bulk operation framework and use it for events.delete. The events list can now select everything matching the current filter (Gmail style) and delete it in the background with a progress panel. - Operation record in the cache (7 days) with a per-organization index. - IBulkOperationHandler and IBulkSelectionResolver, discovered by DI. - BulkOperationWorkItem job: batches, saved cursor, cancellation, one active destructive operation per project. - Endpoints under /api/v2/bulk-operations and BulkOperationUpdated push. - Single-item delete and the bulk batch share EventDeletionService. Co-Authored-By: Claude Opus 5.5 --- src/Exceptionless.Core/Bootstrapper.cs | 7 + .../BulkOperationWorkItemHandler.cs | 183 +++++ .../Models/BulkOperations/BulkOperation.cs | 86 +++ .../Models/Messaging/BulkOperationUpdated.cs | 23 + .../Models/WorkItems/BulkOperationWorkItem.cs | 6 + .../BulkOperations/BulkOperationService.cs | 256 +++++++ .../DeleteEventsBulkOperationHandler.cs | 70 ++ .../EventBulkSelectionResolver.cs | 119 +++ .../BulkOperations/IBulkOperationHandler.cs | 84 ++ .../Services/EventDeletionService.cs | 49 ++ src/Exceptionless.Web/Api/ApiEndpoints.cs | 1 + .../Api/Endpoints/BulkOperationEndpoints.cs | 99 +++ .../Api/Handlers/BulkOperationHandler.cs | 248 ++++++ .../Api/Handlers/EventHandler.cs | 19 +- .../Api/Messages/BulkOperationMessages.cs | 8 + .../Api/Results/AcceptedResponse.cs | 12 + .../Api/Results/ApiResultMapper.cs | 3 + .../e2e/tests/bulk-delete-matching.e2e.ts | 84 ++ .../features/bulk-operations/api.svelte.ts | 114 +++ .../lib/features/bulk-operations/api.test.ts | 93 +++ .../components/bulk-operation-item.svelte | 69 ++ .../bulk-operation-item.svelte.test.ts | 106 +++ .../components/bulk-operation-panel.svelte | 93 +++ .../bulk-operation-panel.svelte.test.ts | 150 ++++ .../features/bulk-operations/models.test.ts | 134 ++++ .../lib/features/bulk-operations/models.ts | 67 ++ .../dialogs/remove-event-dialog.svelte | 19 +- .../all-matching-selection.svelte.test.ts | 52 ++ .../table/all-matching-selection.svelte.ts | 43 ++ .../events-bulk-actions-dropdown-menu.svelte | 55 +- ...-bulk-actions-dropdown-menu.svelte.test.ts | 75 +- .../table/events-select-all-banner.svelte | 35 + .../events-select-all-banner.svelte.test.ts | 53 ++ .../events/components/table/options.svelte.ts | 1 + .../shared/components/ui/progress/index.ts | 7 + .../components/ui/progress/progress.svelte | 27 + .../src/lib/features/websockets/models.ts | 46 +- .../ClientApp/src/lib/generated/api.ts | 102 +++ .../ClientApp/src/lib/generated/schemas.ts | 92 +++ .../ClientApp/src/routes/(app)/+layout.svelte | 2 + .../src/routes/(app)/event/+page.svelte | 79 +- .../Hubs/MessageBusBroker.cs | 12 + .../Models/BulkOperation/NewBulkOperation.cs | 51 ++ .../Models/BulkOperation/ViewBulkOperation.cs | 52 ++ .../Api/Data/endpoint-manifest.json | 56 ++ .../Exceptionless.Tests/Api/Data/openapi.json | 488 ++++++++++++ .../Endpoints/BulkOperationEndpointTests.cs | 717 ++++++++++++++++++ .../Extensions/RequestExtensions.cs | 5 + 48 files changed, 4207 insertions(+), 45 deletions(-) create mode 100644 src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs create mode 100644 src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs create mode 100644 src/Exceptionless.Core/Models/Messaging/BulkOperationUpdated.cs create mode 100644 src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs create mode 100644 src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs create mode 100644 src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs create mode 100644 src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs create mode 100644 src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs create mode 100644 src/Exceptionless.Core/Services/EventDeletionService.cs create mode 100644 src/Exceptionless.Web/Api/Endpoints/BulkOperationEndpoints.cs create mode 100644 src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs create mode 100644 src/Exceptionless.Web/Api/Messages/BulkOperationMessages.cs create mode 100644 src/Exceptionless.Web/Api/Results/AcceptedResponse.cs create mode 100644 src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.svelte.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/index.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/progress.svelte create mode 100644 src/Exceptionless.Web/Models/BulkOperation/NewBulkOperation.cs create mode 100644 src/Exceptionless.Web/Models/BulkOperation/ViewBulkOperation.cs create mode 100644 tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 0d45dc5823..d6f73449a1 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -26,6 +26,7 @@ using Exceptionless.Core.Seed; using Exceptionless.Core.Serialization; using Exceptionless.Core.Services; +using Exceptionless.Core.Services.BulkOperations; using Exceptionless.Core.Services.SourceMaps; using Exceptionless.Core.Utility; using Exceptionless.Core.Validation; @@ -93,6 +94,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(s => { var handlers = new WorkItemHandlers(); + handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); @@ -234,6 +236,11 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO }); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(typeof(IBulkOperationHandler), typeof(Bootstrapper).Assembly); + services.AddSingleton(typeof(IBulkSelectionResolver), typeof(Bootstrapper).Assembly); services.AddSingleton(provider => provider.GetRequiredService()); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs new file mode 100644 index 0000000000..471466481c --- /dev/null +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -0,0 +1,183 @@ +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BulkOperations; +using Foundatio.Jobs; +using Foundatio.Lock; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs.WorkItemHandlers; + +/// +/// Runs a stored bulk operation in batches. After every batch the counts and the resume cursor are saved and a progress +/// message is published; between batches it checks for cancellation. A restarted job continues from the saved cursor. +/// +public class BulkOperationWorkItemHandler : WorkItemHandlerBase +{ + private readonly BulkOperationService _service; + private readonly IReadOnlyDictionary _resolvers; + private readonly BulkOperationOptions _options; + private readonly ILockProvider _lockProvider; + private readonly TimeProvider _timeProvider; + + public BulkOperationWorkItemHandler(BulkOperationService service, IEnumerable resolvers, BulkOperationOptions options, + ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory) + { + _service = service; + _resolvers = resolvers.ToDictionary(r => r.Entity); + _options = options; + _lockProvider = lockProvider; + _timeProvider = timeProvider; + } + + public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) + { + string cacheKey = $"{nameof(BulkOperationWorkItem)}:{((BulkOperationWorkItem)workItem).OperationId}"; + return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(15), cancellationToken); + } + + public override async Task HandleItemAsync(WorkItemContext context) + { + var workItem = context.GetData()!; + ArgumentException.ThrowIfNullOrWhiteSpace(workItem.OperationId); + + var operation = await _service.GetAsync(workItem.OperationId); + if (operation is null) + { + Log.LogWarning("Bulk operation {OperationId} no longer exists", workItem.OperationId); + return; + } + + using var _ = Log.BeginScope(new ExceptionlessState().Organization(operation.OrganizationId).Tag("BulkOperation")); + if (!operation.IsActive) + { + Log.LogInformation("Skipping bulk operation {OperationId} because it is already {Status}", operation.Id, operation.Status); + return; + } + + var handler = _service.GetHandler(operation.Type); + if (handler is null || !_resolvers.TryGetValue(handler.Entity, out var resolver)) + { + await FinishAsync(operation.Id, BulkOperationStatus.Failed, $"Unknown bulk operation type '{operation.Type}'."); + return; + } + + if (await _service.IsCancelRequestedAsync(operation.Id)) + { + await FinishAsync(operation.Id, BulkOperationStatus.Cancelled, "Cancelled by user."); + return; + } + + operation = await _service.UpdateAsync(operation.Id, current => + { + if (!current.IsActive) + return false; + + current.Status = BulkOperationStatus.Running; + current.StartedUtc ??= _timeProvider.GetUtcNow().UtcDateTime; + return true; + }); + + if (operation is null || operation.Status is not BulkOperationStatus.Running) + return; + + Log.LogInformation("Running bulk operation {OperationId} {Type} from cursor {Cursor}", operation.Id, operation.Type, operation.Cursor); + await context.ReportProgressAsync(GetPercent(operation), operation.Description); + + try + { + while (true) + { + if (context.CancellationToken.IsCancellationRequested) + { + // The host is shutting down. Leave the operation running so the queue retry resumes it from the cursor. + context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {operation.Id} for shutdown"); + return; + } + + if (await _service.IsCancelRequestedAsync(operation.Id)) + { + await FinishAsync(operation.Id, BulkOperationStatus.Cancelled, $"Cancelled by user after {operation.Processed:N0} of {operation.Total:N0}."); + return; + } + + var page = await GetPageAsync(operation, resolver, context.CancellationToken); + if (page.Ids.Count == 0) + break; + + var result = await handler.ExecuteBatchAsync(operation, page.Ids, context.CancellationToken); + + operation = await _service.UpdateAsync(operation.Id, current => + { + if (!current.IsActive) + return false; + + current.Processed += page.Ids.Count; + current.Succeeded += result.Succeeded; + current.Failed += result.Failed; + current.Cursor = page.NextCursor; + current.Total = Math.Max(current.Total, current.Processed); + current.Message = result.Message; + return true; + }); + + if (operation is null || !operation.IsActive) + return; + + await context.ReportProgressAsync(GetPercent(operation), $"{operation.Processed:N0} of {operation.Total:N0}"); + + if (page.NextCursor is null) + break; + } + + await _service.UpdateAsync(operation.Id, current => + { + if (!current.IsActive) + return false; + + current.Total = current.Processed; + _service.Finish(current, BulkOperationStatus.Completed, current.Failed > 0 ? $"Completed with {current.Failed:N0} failures." : "Completed."); + return true; + }); + } + catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested) + { + context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {operation?.Id} for shutdown"); + } + catch (Exception ex) + { + Log.LogError(ex, "Bulk operation {OperationId} failed: {Message}", workItem.OperationId, ex.Message); + await FinishAsync(workItem.OperationId, BulkOperationStatus.Failed, "The operation stopped because of an unexpected error."); + } + } + + private async Task GetPageAsync(BulkOperation operation, IBulkSelectionResolver resolver, CancellationToken cancellationToken) + { + int size = Math.Max(1, _options.BatchSize); + if (operation.Selection.Kind is not BulkOperationSelectionKind.Ids) + return await resolver.GetPageAsync(operation, operation.Cursor, size, cancellationToken); + + // The cursor of an id selection is the index of the next id to process. + var ids = operation.Selection.Ids ?? []; + int start = Int32.TryParse(operation.Cursor, out int parsed) ? parsed : 0; + var batch = ids.Skip(start).Take(size).ToList(); + int next = start + batch.Count; + return new BulkSelectionPage(batch, next < ids.Count ? next.ToString() : null); + } + + private Task FinishAsync(string operationId, BulkOperationStatus status, string message) + { + return _service.UpdateAsync(operationId, current => + { + if (!current.IsActive) + return false; + + _service.Finish(current, status, message); + return true; + }); + } + + private static int GetPercent(BulkOperation operation) + { + return operation.Total <= 0 ? 0 : (int)Math.Min(99, operation.Processed * 100 / operation.Total); + } +} diff --git a/src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs b/src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs new file mode 100644 index 0000000000..819f967040 --- /dev/null +++ b/src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs @@ -0,0 +1,86 @@ +using System.Text.Json.Serialization; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Core.Models.BulkOperations; + +/// +/// A long-running action applied to every item that matches a selection. Stored in the cache with a limited +/// lifetime and indexed per organization; it is intentionally not persisted in Elasticsearch. +/// +public class BulkOperation : IIdentity, IOwnedByOrganization +{ + public string Id { get; set; } = null!; + public string OrganizationId { get; set; } = null!; + public string? ProjectId { get; set; } + + /// The handler key, e.g. events.delete. + public string Type { get; set; } = null!; + + public string Description { get; set; } = null!; + public BulkOperationSelection Selection { get; set; } = null!; + public Dictionary Parameters { get; set; } = new(); + public BulkOperationStatus Status { get; set; } = BulkOperationStatus.Queued; + + public long Total { get; set; } + public long Processed { get; set; } + public long Succeeded { get; set; } + public long Failed { get; set; } + + public string CreatedByUserId { get; set; } = null!; + public DateTime CreatedUtc { get; set; } + public DateTime? StartedUtc { get; set; } + public DateTime? CompletedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } + + public string? Message { get; set; } + + /// Opaque position of the last batch that finished; the job resumes after it. + public string? Cursor { get; set; } + + [JsonIgnore] + public bool IsActive => Status is BulkOperationStatus.Queued or BulkOperationStatus.Running; +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum BulkOperationStatus +{ + Queued, + Running, + Completed, + Failed, + Cancelled +} + +/// The kind of item an operation targets. Each entity has one selection resolver. +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum BulkOperationEntity +{ + Events +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum BulkOperationSelectionKind +{ + Ids, + Filter +} + +/// +/// Either an explicit list of ids or a filter. A filter selection always carries the absolute UTC window that was +/// resolved when the operation was created, so items that arrive afterwards are never included. +/// +public class BulkOperationSelection +{ + public BulkOperationSelectionKind Kind { get; set; } + public List? Ids { get; set; } + public string? Filter { get; set; } + + /// The relative time expression the window was resolved from. Informational only. + public string? Time { get; set; } + + public DateTime? UtcStart { get; set; } + public DateTime? UtcEnd { get; set; } + + /// Items stored after this instant (UTC) are never part of a filter selection. + public DateTime? UtcCreatedBefore { get; set; } +} diff --git a/src/Exceptionless.Core/Models/Messaging/BulkOperationUpdated.cs b/src/Exceptionless.Core/Models/Messaging/BulkOperationUpdated.cs new file mode 100644 index 0000000000..27d7c68f3f --- /dev/null +++ b/src/Exceptionless.Core/Models/Messaging/BulkOperationUpdated.cs @@ -0,0 +1,23 @@ +using Exceptionless.Core.Models.BulkOperations; + +namespace Exceptionless.Core.Messaging.Models; + +/// +/// Published after every batch and every status change of a bulk operation. Intentionally independent of the +/// transport that delivers it to clients, and free of the selection and parameters. +/// +public record BulkOperationUpdated +{ + public required string Id { get; set; } + public required string OrganizationId { get; set; } + public string? ProjectId { get; set; } + public required string Type { get; set; } + public required string Description { get; set; } + public BulkOperationStatus Status { get; set; } + public long Total { get; set; } + public long Processed { get; set; } + public long Succeeded { get; set; } + public long Failed { get; set; } + public string? Message { get; set; } + public DateTime UpdatedUtc { get; set; } +} diff --git a/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs new file mode 100644 index 0000000000..d4702f6c5c --- /dev/null +++ b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs @@ -0,0 +1,6 @@ +namespace Exceptionless.Core.Models.WorkItems; + +public record BulkOperationWorkItem +{ + public required string OperationId { get; init; } +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs new file mode 100644 index 0000000000..2584cbcb32 --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -0,0 +1,256 @@ +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Repositories.Utility; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services.BulkOperations; + +public class BulkOperationOptions +{ + /// How many ids are resolved and executed per batch. + public int BatchSize { get; set; } = 500; +} + +/// +/// Stores bulk operations in the cache (with a limited lifetime), keeps a per-organization index of operation ids, +/// enforces the one-active-destructive-operation-per-project rule, and publishes every change. +/// +public class BulkOperationService +{ + public static readonly TimeSpan OperationLifetime = TimeSpan.FromDays(7); + + /// An active operation that has not reported progress for this long is considered abandoned. + public static readonly TimeSpan StaleAfter = TimeSpan.FromHours(2); + + private const int MaxIndexedOperations = 500; + private readonly ICacheClient _cache; + private readonly ILockProvider _lockProvider; + private readonly IMessagePublisher _publisher; + private readonly IQueue _workItemQueue; + private readonly IReadOnlyDictionary _handlers; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public BulkOperationService(ICacheClient cache, ILockProvider lockProvider, IMessagePublisher publisher, IQueue workItemQueue, + IEnumerable handlers, TimeProvider timeProvider, ILogger logger) + { + _cache = cache; + _lockProvider = lockProvider; + _publisher = publisher; + _workItemQueue = workItemQueue; + _handlers = handlers.ToDictionary(h => h.Type, StringComparer.Ordinal); + _timeProvider = timeProvider; + _logger = logger; + } + + public IBulkOperationHandler? GetHandler(string type) => _handlers.GetValueOrDefault(type); + + public async Task GetAsync(string id) + { + if (String.IsNullOrEmpty(id)) + return null; + + var cached = await _cache.GetAsync(GetOperationKey(id)); + return cached.HasValue ? cached.Value : null; + } + + /// All operations of the organization that are still stored, newest first. + public async Task> GetByOrganizationAsync(string organizationId) + { + var ids = new List(); + for (int page = 1; page <= MaxIndexedOperations / 100; page++) + { + var batch = await _cache.GetListAsync(GetIndexKey(organizationId), page, 100); + if (!batch.HasValue || batch.Value.Count == 0) + break; + + ids.AddRange(batch.Value); + if (batch.Value.Count < 100) + break; + } + + var operations = new List(); + var expired = new List(); + foreach (string id in ids.Distinct(StringComparer.Ordinal)) + { + var operation = await GetAsync(id); + if (operation is null) + expired.Add(id); + else if (String.Equals(operation.OrganizationId, organizationId, StringComparison.Ordinal)) + operations.Add(operation); + } + + if (expired.Count > 0) + await _cache.ListRemoveAsync(GetIndexKey(organizationId), expired); + + return operations.OrderByDescending(o => o.CreatedUtc).ToList(); + } + + /// + /// Stores and enqueues the operation unless it conflicts with an active destructive operation on the same project. + /// The check and the write happen under a per-organization lock so two requests cannot both pass it. + /// + public async Task CreateAsync(BulkOperation operation) + { + var handler = GetHandler(operation.Type) ?? throw new ArgumentException($"Unknown bulk operation type '{operation.Type}'.", nameof(operation)); + + await using var createLock = await _lockProvider.TryAcquireAsync($"bulk-operations:create:{operation.OrganizationId}", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); + if (createLock is null) + return BulkOperationCreateResult.Busy(); + + if (handler.IsDestructive) + { + var conflict = await FindConflictingOperationAsync(operation); + if (conflict is not null) + return BulkOperationCreateResult.Conflicting(conflict); + } + + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + operation.Id = ObjectId.GenerateNewId(utcNow).ToString(); + operation.Status = BulkOperationStatus.Queued; + operation.CreatedUtc = utcNow; + operation.UpdatedUtc = utcNow; + + await SaveAsync(operation); + await _cache.ListAddAsync(GetIndexKey(operation.OrganizationId), [operation.Id], OperationLifetime); + await _workItemQueue.EnqueueAsync(new BulkOperationWorkItem { OperationId = operation.Id }); + await PublishAsync(operation); + + return BulkOperationCreateResult.Created(operation); + } + + /// + /// Loads the operation, lets change it, then saves and publishes it. The change is skipped + /// when returns false. Returns the stored operation, or null when it no longer exists. + /// + public async Task UpdateAsync(string id, Func update) + { + await using var updateLock = await _lockProvider.TryAcquireAsync($"bulk-operations:update:{id}", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); + if (updateLock is null) + throw new InvalidOperationException($"Unable to lock bulk operation {id} for update."); + + var operation = await GetAsync(id); + if (operation is null) + return null; + + if (!update(operation)) + return operation; + + operation.UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime; + await SaveAsync(operation); + await PublishAsync(operation); + return operation; + } + + /// + /// Requests cancellation. A queued (or abandoned) operation is cancelled right away; a running one stops at the + /// next batch boundary, after the batch that is in flight finishes. + /// + public async Task CancelAsync(string id) + { + var operation = await GetAsync(id); + if (operation is null || !operation.IsActive) + return operation; + + await _cache.SetAsync(GetCancelKey(id), true, OperationLifetime); + + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + return await UpdateAsync(id, current => + { + if (!current.IsActive) + return false; + + bool isRunningNow = current.Status is BulkOperationStatus.Running && utcNow - current.UpdatedUtc < StaleAfter; + if (isRunningNow) + return false; + + Finish(current, BulkOperationStatus.Cancelled, "Cancelled before it started.", utcNow); + return true; + }); + } + + public async Task IsCancelRequestedAsync(string id) + { + return (await _cache.GetAsync(GetCancelKey(id))).HasValue; + } + + public void Finish(BulkOperation operation, BulkOperationStatus status, string? message, DateTime? utcNow = null) + { + operation.Status = status; + operation.Message = message; + operation.CompletedUtc = utcNow ?? _timeProvider.GetUtcNow().UtcDateTime; + } + + private async Task FindConflictingOperationAsync(BulkOperation candidate) + { + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + foreach (var existing in await GetByOrganizationAsync(candidate.OrganizationId)) + { + if (!existing.IsActive || utcNow - existing.UpdatedUtc >= StaleAfter) + continue; + + if (GetHandler(existing.Type) is not { IsDestructive: true }) + continue; + + // An organization wide operation can touch every project, so it conflicts with any project's operation. + bool overlaps = existing.ProjectId is null || candidate.ProjectId is null || String.Equals(existing.ProjectId, candidate.ProjectId, StringComparison.Ordinal); + if (overlaps) + return existing; + } + + return null; + } + + private Task SaveAsync(BulkOperation operation) => _cache.SetAsync(GetOperationKey(operation.Id), operation, OperationLifetime); + + private async Task PublishAsync(BulkOperation operation) + { + try + { + await _publisher.PublishAsync(new BulkOperationUpdated + { + Id = operation.Id, + OrganizationId = operation.OrganizationId, + ProjectId = operation.ProjectId, + Type = operation.Type, + Description = operation.Description, + Status = operation.Status, + Total = operation.Total, + Processed = operation.Processed, + Succeeded = operation.Succeeded, + Failed = operation.Failed, + Message = operation.Message, + UpdatedUtc = operation.UpdatedUtc + }); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to publish bulk operation update for {OperationId}: {Message}", operation.Id, ex.Message); + } + } + + private static string GetOperationKey(string id) => $"bulk-operations:{id}"; + private static string GetCancelKey(string id) => $"bulk-operations:{id}:cancel"; + private static string GetIndexKey(string organizationId) => $"bulk-operations:organization:{organizationId}"; +} + +public record BulkOperationCreateResult +{ + public BulkOperation? Operation { get; init; } + public BulkOperation? Conflict { get; init; } + public bool IsBusy { get; init; } + + public static BulkOperationCreateResult Created(BulkOperation operation) => new() { Operation = operation }; + public static BulkOperationCreateResult Conflicting(BulkOperation conflict) => new() { Conflict = conflict }; + public static BulkOperationCreateResult Busy() => new() { IsBusy = true }; +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs new file mode 100644 index 0000000000..323379cd26 --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs @@ -0,0 +1,70 @@ +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Foundatio.Repositories; + +namespace Exceptionless.Core.Services.BulkOperations; + +/// +/// Deletes events. Each batch goes through , the same code the single-item +/// DELETE /events/{ids} endpoint uses, so usage accounting and notifications are identical. +/// +public class DeleteEventsBulkOperationHandler : IBulkOperationHandler +{ + public const string OperationType = "events.delete"; + + private readonly IEventRepository _eventRepository; + private readonly EventDeletionService _eventDeletionService; + + public DeleteEventsBulkOperationHandler(IEventRepository eventRepository, EventDeletionService eventDeletionService) + { + _eventRepository = eventRepository; + _eventDeletionService = eventDeletionService; + } + + public string Type => OperationType; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => true; + + public Task AuthorizeAsync(BulkOperationRequest request) + { + // Same rule as DELETE /events/{ids}: any user of the organization (or a global admin) may delete its events. + if (!request.Actor.CanAccessOrganization(request.Organization.Id)) + return Task.FromResult(BulkOperationCheckResult.Denied("Organization not found.")); + + if (request.Project is not null && !String.Equals(request.Project.OrganizationId, request.Organization.Id, StringComparison.Ordinal)) + return Task.FromResult(BulkOperationCheckResult.Denied("Project not found.")); + + return Task.FromResult(BulkOperationCheckResult.Ok); + } + + public Task ValidateAsync(BulkOperationRequest request) + { + if (request.Parameters.Count > 0) + return Task.FromResult(BulkOperationCheckResult.Invalid("parameters", $"The {Type} operation does not take parameters.")); + + return Task.FromResult(BulkOperationCheckResult.Ok); + } + + public string Describe(BulkOperation operation) + { + return operation.Selection.Kind is BulkOperationSelectionKind.Ids + ? $"Delete {operation.Total:N0} selected events" + : $"Delete {operation.Total:N0} events matching filter"; + } + + public async Task ExecuteBatchAsync(BulkOperation operation, IReadOnlyCollection ids, CancellationToken cancellationToken) + { + var events = await _eventRepository.GetByIdsAsync(ids.ToArray(), o => o.Cache(false)); + + // The operation may only touch its own organization (and project), no matter what ids it was given. + var inScope = events + .Where(e => String.Equals(e.OrganizationId, operation.OrganizationId, StringComparison.Ordinal) + && (operation.ProjectId is null || String.Equals(e.ProjectId, operation.ProjectId, StringComparison.Ordinal))) + .ToList(); + + await _eventDeletionService.DeleteAsync(inScope); + + int failed = ids.Count - inScope.Count; + return new BulkBatchResult(inScope.Count, failed, failed > 0 ? $"{failed:N0} events were not found." : null); + } +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs new file mode 100644 index 0000000000..7d7392acd7 --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -0,0 +1,119 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Repositories.Queries; +using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Extensions; +using Foundatio.Serializer; + +namespace Exceptionless.Core.Services.BulkOperations; + +/// +/// Pages the ids of events that match an organization or project scope, a filter and an absolute UTC window, using the +/// same query building as the events list so the selection is exactly what the user was looking at. +/// +public class EventBulkSelectionResolver : IBulkSelectionResolver +{ + private const string CreatedUtcField = "created_utc"; + private readonly IEventRepository _eventRepository; + private readonly IOrganizationRepository _organizationRepository; + private readonly IProjectRepository _projectRepository; + private readonly PersistentEventQueryValidator _validator; + private readonly ITextSerializer _serializer; + + public EventBulkSelectionResolver(IEventRepository eventRepository, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, + PersistentEventQueryValidator validator, ITextSerializer serializer) + { + _eventRepository = eventRepository; + _organizationRepository = organizationRepository; + _projectRepository = projectRepository; + _validator = validator; + _serializer = serializer; + } + + public BulkOperationEntity Entity => BulkOperationEntity.Events; + + public async Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection) + { + var result = await _validator.ValidateQueryAsync(selection.Filter); + if (!result.IsValid) + return BulkOperationCheckResult.Invalid("selection.filter", result.Message ?? "Invalid filter."); + + if (result.UsesPremiumFeatures && !organization.HasPremiumFeatures) + return BulkOperationCheckResult.PlanLimit("Please upgrade your plan to use premium search features."); + + return BulkOperationCheckResult.Ok; + } + + public async Task CountAsync(BulkOperation operation, CancellationToken cancellationToken) + { + var scope = await GetScopeAsync(operation); + if (scope is null) + return 0; + + var result = await _eventRepository.CountAsync(q => BuildQuery(q, scope.Value.AppFilter, operation.Selection)); + return result.Total; + } + + public async Task GetPageAsync(BulkOperation operation, string? cursor, int size, CancellationToken cancellationToken) + { + var scope = await GetScopeAsync(operation); + if (scope is null) + return new BulkSelectionPage([], null); + + var results = await _eventRepository.FindAsync( + q => BuildQuery(q, scope.Value.AppFilter, operation.Selection).Include(e => e.Id).SortExpression($"-{EventIndex.Alias.Date}"), + o => o.SearchAfterToken(cursor, _serializer).PageLimit(size)); + + string[] ids = results.Hits.Select(h => h.Id).Where(id => !String.IsNullOrEmpty(id)).Select(id => id!).ToArray(); + string? nextCursor = results.Hits.LastOrDefault()?.GetSortToken(_serializer); + return new BulkSelectionPage(ids, ids.Length > 0 ? nextCursor : null); + } + + private static T BuildQuery(T query, AppFilter appFilter, BulkOperationSelection selection) where T : class, IRepositoryQuery + { + query.AppFilter(appFilter) + .FilterExpression(selection.Filter) + .EnforceEventStackFilter(); + + if (selection.UtcStart.HasValue || selection.UtcEnd.HasValue) + { + query.DateRange(selection.UtcStart, selection.UtcEnd, EventIndex.Alias.Date) + .Index(selection.UtcStart, selection.UtcEnd); + } + + // The event date is chosen by the client, so an old-dated event can still arrive after the operation was created. + if (selection.UtcCreatedBefore.HasValue) + query.DateRange(null, selection.UtcCreatedBefore, CreatedUtcField); + + return query; + } + + private async Task<(AppFilter AppFilter, Organization Organization)?> GetScopeAsync(BulkOperation operation) + { + var organization = await _organizationRepository.GetByIdAsync(operation.OrganizationId, o => o.Cache()); + if (organization is null) + return null; + + var validation = await _validator.ValidateQueryAsync(operation.Selection.Filter); + AppFilter appFilter; + if (!String.IsNullOrEmpty(operation.ProjectId)) + { + var project = await _projectRepository.GetByIdAsync(operation.ProjectId, o => o.Cache()); + if (project is null || !String.Equals(project.OrganizationId, organization.Id, StringComparison.Ordinal)) + return null; + + appFilter = new AppFilter(project, organization); + } + else + { + appFilter = new AppFilter(organization); + } + + appFilter.UsesPremiumFeatures = validation.UsesPremiumFeatures; + return (appFilter, organization); + } +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs new file mode 100644 index 0000000000..7d1d31e459 --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs @@ -0,0 +1,84 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BulkOperations; + +namespace Exceptionless.Core.Services.BulkOperations; + +/// +/// One bulk action, e.g. events.delete. Adding a new bulk action means adding one implementation of this +/// interface; handlers are discovered and registered automatically. +/// +public interface IBulkOperationHandler +{ + /// The unique key that identifies the action in API requests and stored operations. + string Type { get; } + + /// The kind of item the action targets. Selects the that pages ids. + BulkOperationEntity Entity { get; } + + /// Destructive actions are limited to one active operation per project. + bool IsDestructive { get; } + + /// Checks that the actor may run this action against the organization and project. + Task AuthorizeAsync(BulkOperationRequest request); + + /// Checks the action specific parameters and the selection shape. + Task ValidateAsync(BulkOperationRequest request); + + /// A short, human readable description shown while the operation runs. + string Describe(BulkOperation operation); + + /// + /// Executes one batch of ids through the same code path as the single-item action, so side effects stay identical. + /// Ids that cannot be processed (missing, or outside the operation's organization and project) must be reported as failed. + /// + Task ExecuteBatchAsync(BulkOperation operation, IReadOnlyCollection ids, CancellationToken cancellationToken); +} + +/// Pages the ids that match a filter selection. One implementation per . +public interface IBulkSelectionResolver +{ + BulkOperationEntity Entity { get; } + + /// Checks that the filter of a selection can be run for the organization (syntax, plan limits). + Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection); + + /// Counts everything that currently matches the selection. + Task CountAsync(BulkOperation operation, CancellationToken cancellationToken); + + /// + /// Returns the next page of ids after . The cursor is a value based position, so + /// paging stays correct while earlier items are being deleted or changed. + /// + Task GetPageAsync(BulkOperation operation, string? cursor, int size, CancellationToken cancellationToken); +} + +public record BulkSelectionPage(IReadOnlyList Ids, string? NextCursor); + +public record BulkBatchResult(int Succeeded, int Failed, string? Message = null); + +/// The authenticated caller, independent of the transport the request arrived on. +public record BulkOperationActor(string UserId, bool IsGlobalAdmin, IReadOnlyCollection OrganizationIds) +{ + public bool CanAccessOrganization(string organizationId) => IsGlobalAdmin || OrganizationIds.Contains(organizationId, StringComparer.Ordinal); +} + +public record BulkOperationRequest(BulkOperationActor Actor, Organization Organization, Project? Project, string Type, BulkOperationSelection Selection, IReadOnlyDictionary Parameters); + +public enum BulkOperationCheckStatus +{ + Ok, + Denied, + Invalid, + PlanLimit +} + +public record BulkOperationCheckResult(BulkOperationCheckStatus Status, string? Field = null, string? Message = null) +{ + public static readonly BulkOperationCheckResult Ok = new(BulkOperationCheckStatus.Ok); + + public bool IsOk => Status is BulkOperationCheckStatus.Ok; + + public static BulkOperationCheckResult Denied(string message) => new(BulkOperationCheckStatus.Denied, null, message); + public static BulkOperationCheckResult Invalid(string field, string message) => new(BulkOperationCheckStatus.Invalid, field, message); + public static BulkOperationCheckResult PlanLimit(string message) => new(BulkOperationCheckStatus.PlanLimit, null, message); +} diff --git a/src/Exceptionless.Core/Services/EventDeletionService.cs b/src/Exceptionless.Core/Services/EventDeletionService.cs new file mode 100644 index 0000000000..d104e78187 --- /dev/null +++ b/src/Exceptionless.Core/Services/EventDeletionService.cs @@ -0,0 +1,49 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Foundatio.Repositories; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services; + +/// +/// Removes events and records the deleted usage. Shared by the single-item delete endpoint and the bulk delete +/// operation so both have identical side effects. +/// +public class EventDeletionService +{ + private readonly IEventRepository _eventRepository; + private readonly UsageService _usageService; + private readonly ILogger _logger; + + public EventDeletionService(IEventRepository eventRepository, UsageService usageService, ILogger logger) + { + _eventRepository = eventRepository; + _usageService = usageService; + _logger = logger; + } + + /// Deletes events the caller has already verified access to. + public async Task DeleteAsync(IReadOnlyCollection events) + { + if (events.Count == 0) + return; + + await _eventRepository.RemoveAsync(events); + + foreach (var projectGroup in events.GroupBy(ev => new { ev.OrganizationId, ev.ProjectId })) + { + try + { + await _usageService.IncrementDeletedAsync(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, projectGroup.Count()); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to increment deleted usage metrics for organization {OrganizationId} project {ProjectId}: {Message}", projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, ex.Message); + } + } + } +} diff --git a/src/Exceptionless.Web/Api/ApiEndpoints.cs b/src/Exceptionless.Web/Api/ApiEndpoints.cs index 540b0abfaa..cd547e9e34 100644 --- a/src/Exceptionless.Web/Api/ApiEndpoints.cs +++ b/src/Exceptionless.Web/Api/ApiEndpoints.cs @@ -16,6 +16,7 @@ public static WebApplication MapApiEndpoints(this WebApplication app) app.MapWebHookEndpoints(); app.MapStripeEndpoints(); app.MapSavedViewEndpoints(); + app.MapBulkOperationEndpoints(); app.MapUserEndpoints(); app.MapProjectEndpoints(); app.MapOrganizationEndpoints(); diff --git a/src/Exceptionless.Web/Api/Endpoints/BulkOperationEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/BulkOperationEndpoints.cs new file mode 100644 index 0000000000..c320374218 --- /dev/null +++ b/src/Exceptionless.Web/Api/Endpoints/BulkOperationEndpoints.cs @@ -0,0 +1,99 @@ +using Exceptionless.Core.Authorization; +using Exceptionless.Web.Api.Filters; +using Exceptionless.Web.Api.Results; +using Exceptionless.Web.Models; +using Exceptionless.Web.Utility; +using Exceptionless.Web.Utility.OpenApi; +using Foundatio.Mediator; +using Microsoft.AspNetCore.Mvc; +using BulkOperationMessages = Exceptionless.Web.Api.Messages; +using HttpIResult = Microsoft.AspNetCore.Http.IResult; + +namespace Exceptionless.Web.Api.Endpoints; + +public static class BulkOperationEndpoints +{ + public static IEndpointRouteBuilder MapBulkOperationEndpoints(this IEndpointRouteBuilder endpoints) + { + var group = endpoints.MapGroup("api/v2") + .RequireAuthorization(AuthorizationRoles.UserPolicy) + .AddEndpointFilter() + .WithTags("BulkOperation"); + + group.MapPost("bulk-operations", async (IMediator mediator, IMediatorResultMapper resultMapper, [FromBody] NewBulkOperation operation) + => (await mediator.InvokeAsync>>(new BulkOperationMessages.CreateBulkOperation(operation))).ToHttpResult(resultMapper)) + .Accepts("application/json", "application/*+json") + .Produces(StatusCodes.Status202Accepted) + .ProducesProblem(StatusCodes.Status403Forbidden) + .ProducesProblem(StatusCodes.Status404NotFound) + .ProducesProblem(StatusCodes.Status409Conflict) + .ProducesProblem(StatusCodes.Status422UnprocessableEntity) + .ProducesProblem(StatusCodes.Status426UpgradeRequired) + .WithSummary("Create") + .WithDescription(""" + Starts an action, such as `events.delete`, on everything that matches a selection. The selection is either a list of ids + or a filter with a time range. The time range is resolved to absolute UTC dates when the operation is created, and events + stored after that moment are never included. The operation runs in the background; poll it or listen for + `BulkOperationUpdated` messages to follow its progress. + """) + .WithMetadata(new EndpointDocumentation { + RequestBodyDescription = "The bulk operation to start.", + ResponseDescriptions = new() { + ["202"] = "Accepted", + ["403"] = "The action is not permitted for this organization or project.", + ["404"] = "The organization or project could not be found.", + ["409"] = "A destructive operation is already running for this project.", + ["422"] = "The request is not valid.", + ["426"] = "The filter uses a feature that requires a plan upgrade.", + } + }); + + group.MapGet("bulk-operations/{id:objectid}", async (string id, IMediator mediator, IMediatorResultMapper resultMapper) + => (await mediator.InvokeAsync>(new BulkOperationMessages.GetBulkOperationById(id))).ToHttpResult(resultMapper)) + .WithName("GetBulkOperationById") + .Produces() + .ProducesProblem(StatusCodes.Status404NotFound) + .WithSummary("Get by id") + .WithMetadata(new EndpointDocumentation { + ParameterDescriptions = new() { + ["id"] = "The identifier of the bulk operation.", + }, + ResponseDescriptions = new() { + ["404"] = "The bulk operation could not be found.", + } + }); + + group.MapGet("organizations/{organizationId:objectid}/bulk-operations", async (string organizationId, IMediator mediator, IMediatorResultMapper resultMapper) + => (await mediator.InvokeAsync>>(new BulkOperationMessages.GetBulkOperationsByOrganization(organizationId))).ToHttpResult(resultMapper)) + .Produces>() + .ProducesProblem(StatusCodes.Status404NotFound) + .WithSummary("Get by organization") + .WithDescription("Returns the active and recently finished bulk operations of the organization, newest first. Finished operations are kept for seven days.") + .WithMetadata(new EndpointDocumentation { + ParameterDescriptions = new() { + ["organizationId"] = "The identifier of the organization.", + }, + ResponseDescriptions = new() { + ["404"] = "The organization could not be found.", + } + }); + + group.MapPost("bulk-operations/{id:objectid}/cancel", async (string id, IMediator mediator, IMediatorResultMapper resultMapper) + => (await mediator.InvokeAsync>>(new BulkOperationMessages.CancelBulkOperation(id))).ToHttpResult(resultMapper)) + .Produces(StatusCodes.Status202Accepted) + .ProducesProblem(StatusCodes.Status404NotFound) + .WithSummary("Cancel") + .WithDescription("Requests cancellation. A running operation stops after the batch it is processing; work already done is not undone. Cancelling a finished operation has no effect.") + .WithMetadata(new EndpointDocumentation { + ParameterDescriptions = new() { + ["id"] = "The identifier of the bulk operation.", + }, + ResponseDescriptions = new() { + ["202"] = "Accepted", + ["404"] = "The bulk operation could not be found.", + } + }); + + return endpoints; + } +} diff --git a/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs b/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs new file mode 100644 index 0000000000..5149567c9d --- /dev/null +++ b/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs @@ -0,0 +1,248 @@ +using Exceptionless.Core; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Web.Api.Infrastructure; +using Exceptionless.Web.Api.Messages; +using Exceptionless.Web.Api.Results; +using Exceptionless.Web.Extensions; +using Exceptionless.Web.Models; +using Foundatio.Mediator; +using Foundatio.Repositories; +using Foundatio.Repositories.Utility; + +namespace Exceptionless.Web.Api.Handlers; + +public class BulkOperationHandler( + BulkOperationService bulkOperationService, + IEnumerable resolvers, + IOrganizationRepository organizationRepository, + IProjectRepository projectRepository, + AppOptions appOptions, + TimeProvider timeProvider, + IHttpContextAccessor httpContextAccessor, + ILogger logger) +{ + private const int MaxIds = 1000; + private static readonly ICollection _allowedDateFields = new List { EventIndex.Alias.Date }; + + private HttpContext HttpContext => httpContextAccessor.HttpContext ?? throw new InvalidOperationException("HttpContext is unavailable."); + + public async Task>> Handle(CreateBulkOperation message) + { + var request = HttpContext.Request; + var model = message.Operation; + + var handler = bulkOperationService.GetHandler(model.Type); + if (handler is null) + return Result.Invalid(ValidationError.Create("type", $"Unknown bulk operation type '{model.Type}'.")); + + // Organization and project access mirrors the single-item actions: not accessible looks the same as not found. + if (!request.CanAccessOrganization(model.OrganizationId)) + return Result.NotFound("Organization not found."); + + var organization = await organizationRepository.GetByIdAsync(model.OrganizationId, o => o.Cache()); + if (organization is null) + return Result.NotFound("Organization not found."); + + Project? project = null; + if (!String.IsNullOrEmpty(model.ProjectId)) + { + project = await projectRepository.GetByIdAsync(model.ProjectId, o => o.Cache()); + if (project is null || !String.Equals(project.OrganizationId, organization.Id, StringComparison.Ordinal)) + return Result.NotFound("Project not found."); + } + + var user = request.GetUser(); + var actor = new BulkOperationActor(user.Id, request.IsGlobalAdmin(), request.GetAssociatedOrganizationIds().ToList()); + var parameters = model.Parameters ?? new Dictionary(); + + var selectionResult = ResolveSelection(model.Selection, organization); + if (selectionResult.Result is not null) + return selectionResult.Result; + + var selection = selectionResult.Selection!; + var bulkRequest = new BulkOperationRequest(actor, organization, project, handler.Type, selection, parameters); + + var authorized = await handler.AuthorizeAsync(bulkRequest); + if (!authorized.IsOk) + return ToResult>(authorized); + + var validated = await handler.ValidateAsync(bulkRequest); + if (!validated.IsOk) + return ToResult>(validated); + + var operation = new BulkOperation + { + OrganizationId = organization.Id, + ProjectId = project?.Id, + Type = handler.Type, + Selection = selection, + Parameters = new Dictionary(parameters), + CreatedByUserId = user.Id + }; + + if (selection.Kind is BulkOperationSelectionKind.Filter) + { + var resolver = resolvers.FirstOrDefault(r => r.Entity == handler.Entity); + if (resolver is null) + return Result.Error($"No selection resolver is registered for {handler.Entity}."); + + var selectionCheck = await resolver.ValidateSelectionAsync(organization, selection); + if (!selectionCheck.IsOk) + return ToResult>(selectionCheck); + + operation.Total = await resolver.CountAsync(operation, HttpContext.RequestAborted); + } + else + { + operation.Total = selection.Ids!.Count; + } + + operation.Description = handler.Describe(operation); + + var created = await bulkOperationService.CreateAsync(operation); + if (created.Conflict is not null) + return Result.Conflict("Another destructive operation is already running for this project. Wait for it to finish or cancel it first."); + + if (created.Operation is null) + return Result.Conflict("Another bulk operation is being started for this organization. Please try again."); + + using var _ = logger.BeginScope(new ExceptionlessState().Organization(organization.Id).Tag("BulkOperation").Identity(user.EmailAddress)); + logger.LogInformation("User {User} started bulk operation {OperationId} {Type} for {Total} items", user.Id, created.Operation.Id, created.Operation.Type, created.Operation.Total); + + return new AcceptedResponse(MapToViewModel(created.Operation)); + } + + public async Task> Handle(GetBulkOperationById message) + { + var operation = await GetAccessibleAsync(message.Id); + if (operation is null) + return Result.NotFound("Bulk operation not found."); + + return MapToViewModel(operation); + } + + public async Task>> Handle(GetBulkOperationsByOrganization message) + { + if (!HttpContext.Request.CanAccessOrganization(message.OrganizationId)) + return Result.NotFound("Organization not found."); + + var operations = await bulkOperationService.GetByOrganizationAsync(message.OrganizationId); + return Result>.Success(operations.Select(MapToViewModel).ToList()); + } + + public async Task>> Handle(CancelBulkOperation message) + { + var operation = await GetAccessibleAsync(message.Id); + if (operation is null) + return Result.NotFound("Bulk operation not found."); + + var cancelled = await bulkOperationService.CancelAsync(operation.Id); + if (cancelled is null) + return Result.NotFound("Bulk operation not found."); + + logger.LogInformation("User {User} requested cancellation of bulk operation {OperationId}", HttpContext.Request.GetUser().Id, cancelled.Id); + return new AcceptedResponse(MapToViewModel(cancelled)); + } + + private async Task GetAccessibleAsync(string id) + { + var operation = await bulkOperationService.GetAsync(id); + if (operation is null || !HttpContext.Request.CanAccessOrganization(operation.OrganizationId)) + return null; + + return operation; + } + + private (BulkOperationSelection? Selection, Result? Result) ResolveSelection(NewBulkOperationSelection model, Organization organization) + { + var utcNow = timeProvider.GetUtcNow().UtcDateTime; + if (model.Kind is BulkOperationSelectionKind.Ids) + { + if (model.Filter is not null || model.Time is not null) + return (null, Result.Invalid(ValidationError.Create("selection", "An id selection cannot include a filter or time."))); + + var ids = (model.Ids ?? []).Distinct(StringComparer.Ordinal).ToList(); + if (ids.Count == 0) + return (null, Result.Invalid(ValidationError.Create("selection.ids", "At least one id is required."))); + + if (ids.Count > MaxIds) + return (null, Result.Invalid(ValidationError.Create("selection.ids", $"At most {MaxIds} ids can be selected. Use a filter selection for more."))); + + if (ids.Any(id => !ObjectId.TryParse(id, out _))) + return (null, Result.Invalid(ValidationError.Create("selection.ids", "Every id must be a valid identifier."))); + + return (new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ids }, null); + } + + if (model.Ids is { Count: > 0 }) + return (null, Result.Invalid(ValidationError.Create("selection", "A filter selection cannot include ids."))); + + // Resolve the relative time range once, now. The end can never be later than this moment, so events that + // arrive after the operation is created are not part of it. + var timeInfo = TimeRangeParser.GetTimeInfo(model.Time, model.Offset, timeProvider, _allowedDateFields, EventIndex.Alias.Date, + organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); + var utcStart = timeInfo.Range.UtcStart; + var utcEnd = timeInfo.Range.UtcEnd > utcNow ? utcNow : timeInfo.Range.UtcEnd; + if (utcStart > utcEnd) + return (null, Result.Invalid(ValidationError.Create("selection.time", "The time range does not include any past time."))); + + var selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Filter, + Filter = model.Filter?.Trim() ?? String.Empty, + Time = model.Time, + UtcStart = utcStart, + UtcEnd = utcEnd, + UtcCreatedBefore = utcNow + }; + + return (selection, null); + } + + private static Result ToResult(BulkOperationCheckResult check) + { + return check.Status switch + { + BulkOperationCheckStatus.Denied => Result.Forbidden(check.Message ?? "Forbidden"), + BulkOperationCheckStatus.PlanLimit => Result.Invalid(ValidationError.Create(ApiValidationErrorIdentifiers.PlanLimit, check.Message ?? "Please upgrade your plan.")), + _ => Result.Invalid(ValidationError.Create(check.Field ?? "operation", check.Message ?? "Invalid request.")) + }; + } + + private static ViewBulkOperation MapToViewModel(BulkOperation operation) + { + return new ViewBulkOperation + { + Id = operation.Id, + OrganizationId = operation.OrganizationId, + ProjectId = operation.ProjectId, + Type = operation.Type, + Description = operation.Description, + Status = operation.Status, + Selection = new ViewBulkOperationSelection + { + Kind = operation.Selection.Kind, + IdCount = operation.Selection.Kind is BulkOperationSelectionKind.Ids ? operation.Selection.Ids?.Count : null, + Filter = operation.Selection.Filter, + Time = operation.Selection.Time, + UtcStart = operation.Selection.UtcStart, + UtcEnd = operation.Selection.UtcEnd + }, + Total = operation.Total, + Processed = operation.Processed, + Succeeded = operation.Succeeded, + Failed = operation.Failed, + CreatedByUserId = operation.CreatedByUserId, + CreatedUtc = operation.CreatedUtc, + StartedUtc = operation.StartedUtc, + CompletedUtc = operation.CompletedUtc, + UpdatedUtc = operation.UpdatedUtc, + Message = operation.Message + }; + } +} diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index 1b9a8f4029..633c62dde5 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -49,6 +49,7 @@ public class EventHandler( AppOptions appOptions, SystemSettingsService systemSettingsService, UsageService usageService, + EventDeletionService eventDeletionService, TimeProvider timeProvider, LinkGenerator linkGenerator, ILoggerFactory loggerFactory) @@ -665,23 +666,7 @@ public async Task> Handle(DeleteEvents message) _logger.LogInformation("User {User} deleted {RemovedCount} events in project ({ProjectId})", currentUser.Id, projectGroup.Count(), ev.ProjectId); } - await eventRepository.RemoveAsync(list); - - foreach (var projectGroup in projectGroups) - { - try - { - await usageService.IncrementDeletedAsync(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, projectGroup.Count()); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to increment deleted usage metrics for org {OrganizationId} project {ProjectId}: {Message}", projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, ex.Message); - } - } + await eventDeletionService.DeleteAsync(list.ToList()); if (results.Failure.Count == 0) return new WorkInProgressResult(); diff --git a/src/Exceptionless.Web/Api/Messages/BulkOperationMessages.cs b/src/Exceptionless.Web/Api/Messages/BulkOperationMessages.cs new file mode 100644 index 0000000000..54a2db3aa7 --- /dev/null +++ b/src/Exceptionless.Web/Api/Messages/BulkOperationMessages.cs @@ -0,0 +1,8 @@ +using Exceptionless.Web.Models; + +namespace Exceptionless.Web.Api.Messages; + +public record CreateBulkOperation(NewBulkOperation Operation); +public record GetBulkOperationById(string Id); +public record GetBulkOperationsByOrganization(string OrganizationId); +public record CancelBulkOperation(string Id); diff --git a/src/Exceptionless.Web/Api/Results/AcceptedResponse.cs b/src/Exceptionless.Web/Api/Results/AcceptedResponse.cs new file mode 100644 index 0000000000..aeb034f154 --- /dev/null +++ b/src/Exceptionless.Web/Api/Results/AcceptedResponse.cs @@ -0,0 +1,12 @@ +namespace Exceptionless.Web.Api.Results; + +/// Marks a successful handler result that is returned with the 202 Accepted status and a body. +public interface IAcceptedResponse +{ + object? Body { get; } +} + +public record AcceptedResponse(T Value) : IAcceptedResponse +{ + object? IAcceptedResponse.Body => Value; +} diff --git a/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs b/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs index 0a0ac78919..374aee1fa4 100644 --- a/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs +++ b/src/Exceptionless.Web/Api/Results/ApiResultMapper.cs @@ -152,6 +152,9 @@ private static IResult MapSuccess(Foundatio.Mediator.IResult result) if (value is WorkInProgressResult) return HttpResults.Json(value, statusCode: StatusCodes.Status202Accepted); + if (value is IAcceptedResponse accepted) + return HttpResults.Json(accepted.Body, statusCode: StatusCodes.Status202Accepted); + return HttpResults.Ok(value); } diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts new file mode 100644 index 0000000000..553e0e95c8 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts @@ -0,0 +1,84 @@ +import { expect, test } from '../fixtures/e2e-test'; +import { createRepresentativeEvent } from '../support/synthetic-event'; + +const EVENT_COUNT = 5; +const PAGE_SIZE = 2; + +test('deleting everything matching the filter runs in the background and shows progress', async ({ e2eApi, e2eScenario, page }) => { + // Arrange: more events than fit on one page, all in this test's own project. + const references = Array.from({ length: EVENT_COUNT }, (_, index) => `${e2eScenario.referenceId}-bulk-${index}`); + for (const referenceId of references) { + await e2eApi.submitEvent( + e2eScenario.projectId, + e2eScenario.projectToken, + createRepresentativeEvent({ + appUrl: e2eApi.environment.appUrl, + message: `${e2eScenario.message} ${referenceId}`, + referenceId, + runId: e2eApi.environment.runId + }) + ); + } + + for (const referenceId of references) { + await e2eApi.pollForEventByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId); + } + + await page.goto(`/event?project=${e2eScenario.projectId}&time=all&limit=${PAGE_SIZE}`); + const table = page.locator('table:has(thead [role="checkbox"])').first(); + await expect(table.locator('tbody tr')).toHaveCount(PAGE_SIZE, { timeout: 30_000 }); + + await test.step('selecting the page offers to select everything matching the filter', async () => { + // Act + await table.getByRole('checkbox', { name: 'Select all rows' }).click(); + + // Assert + await expect(page.getByRole('status').filter({ hasText: `All ${PAGE_SIZE} events on this page are selected.` })).toBeVisible(); + await expect(page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` })).toBeVisible(); + }); + + await test.step('selecting everything matching the filter can be undone', async () => { + // Act + await page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` }).click(); + await expect(page.getByRole('status').filter({ hasText: `All ${EVENT_COUNT} events are selected` })).toBeVisible(); + await page.getByRole('button', { name: 'Clear selection' }).click(); + + // Assert + await expect(page.getByRole('button', { name: 'Clear selection' })).toBeHidden(); + await expect(table.getByRole('checkbox', { name: 'Select all rows' })).not.toBeChecked(); + }); + + await test.step('deleting against the filter confirms the exact count', async () => { + // Arrange + await table.getByRole('checkbox', { name: 'Select all rows' }).click(); + await page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` }).click(); + + // Act + await page.getByRole('button', { name: 'Actions' }).click(); + await page.getByRole('menuitem', { name: 'Delete' }).click(); + + // Assert + const dialog = page.getByRole('alertdialog'); + await expect(dialog).toContainText(`Delete ${EVENT_COUNT} events matching this filter? This can't be undone.`); + + // Act + await dialog.getByRole('button', { name: `Delete ${EVENT_COUNT} Events` }).click(); + }); + + await test.step('the progress panel follows the operation to completion', async () => { + // Assert + const panel = page.getByRole('region', { name: 'Bulk operations' }); + await expect(panel).toContainText(/Delete \d+ events matching filter/, { timeout: 30_000 }); + await expect(panel).toContainText(`Completed · ${EVENT_COUNT} processed`, { timeout: 60_000 }); + await expect(table.getByRole('checkbox', { name: 'Select row' })).toHaveCount(0, { timeout: 30_000 }); + }); + + await test.step('every event is gone', async () => { + // Assert + for (const referenceId of references) { + await expect + .poll(async () => (await e2eApi.getEventsByReference(e2eScenario.userToken, e2eScenario.projectId, referenceId)).length, { timeout: 30_000 }) + .toBe(0); + } + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.svelte.ts new file mode 100644 index 0000000000..6cf26ede53 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.svelte.ts @@ -0,0 +1,114 @@ +import type { QueryClient } from '@tanstack/svelte-query'; + +import { type ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; +import { createMutation, createQuery, useQueryClient } from '@tanstack/svelte-query'; + +import type { BulkOperationUpdated } from '#features/websockets/models.ts'; + +import { accessToken } from '#features/auth/index.svelte.ts'; +import { queryKeys as eventQueryKeys } from '#features/events/api.svelte.ts'; +import { queryKeys as stackQueryKeys } from '#features/stacks/api.svelte.ts'; + +import { applyBulkOperationUpdate, isBulkOperationActive, type NewBulkOperation, type ViewBulkOperation } from './models'; + +/** While an operation is active the list is polled as well, so progress still moves when push messages are unavailable. */ +export const ACTIVE_OPERATION_POLL_INTERVAL_MS = 3000; + +export const queryKeys = { + cancel: (id: string | undefined) => [...queryKeys.type, id, 'cancel'] as const, + organization: (organizationId: string | undefined) => [...queryKeys.type, 'organization', organizationId] as const, + post: () => [...queryKeys.type, 'post'] as const, + type: ['BulkOperation'] as const +}; + +export interface GetBulkOperationsRequest { + route: { + organizationId: string | undefined; + }; +} + +export function cancelBulkOperation() { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current, + mutationFn: async (id: string) => { + const client = useFetchClient(); + const response = await client.postJSON(`bulk-operations/${id}/cancel`); + + return response.data!; + }, + mutationKey: queryKeys.cancel(undefined), + onSuccess: (operation) => { + upsertBulkOperation(queryClient, operation); + } + })); +} + +export function getBulkOperationsQuery(request: GetBulkOperationsRequest) { + return createQuery(() => ({ + enabled: () => !!accessToken.current && !!request.route.organizationId, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const client = useFetchClient(); + const response = await client.getJSON(`organizations/${request.route.organizationId}/bulk-operations`, { + signal + }); + + return response.data ?? []; + }, + queryKey: queryKeys.organization(request.route.organizationId), + refetchInterval: (query) => (query.state.data?.some((operation) => isBulkOperationActive(operation.status)) ? ACTIVE_OPERATION_POLL_INTERVAL_MS : false) + })); +} + +/** Refreshes everything a finished bulk operation may have changed. */ +export async function invalidateBulkOperationAffectedQueries(queryClient: QueryClient) { + await Promise.all([ + queryClient.invalidateQueries({ + queryKey: eventQueryKeys.type + }), + queryClient.invalidateQueries({ + queryKey: stackQueryKeys.type + }) + ]); +} + +export function postBulkOperation() { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current, + mutationFn: async (operation: NewBulkOperation) => { + const client = useFetchClient(); + const response = await client.postJSON('bulk-operations', operation); + + return response.data!; + }, + mutationKey: queryKeys.post(), + onSuccess: (operation) => { + upsertBulkOperation(queryClient, operation); + } + })); +} + +/** Applies a pushed update to the cached list. Unknown operations (started elsewhere) trigger a refetch instead. */ +export function updateBulkOperationFromMessage(queryClient: QueryClient, update: BulkOperationUpdated) { + const key = queryKeys.organization(update.organization_id); + const current = queryClient.getQueryData(key); + const next = current ? applyBulkOperationUpdate(current, update) : undefined; + if (!next) { + void queryClient.invalidateQueries({ + queryKey: key + }); + return; + } + + queryClient.setQueryData(key, next); +} + +function upsertBulkOperation(queryClient: QueryClient, operation: ViewBulkOperation) { + queryClient.setQueryData(queryKeys.organization(operation.organization_id), (current) => [ + operation, + ...(current ?? []).filter((existing) => existing.id !== operation.id) + ]); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.test.ts new file mode 100644 index 0000000000..8ecf1a2916 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.test.ts @@ -0,0 +1,93 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { BulkOperationSelectionKind, BulkOperationStatus, type ViewBulkOperation } from './models'; + +vi.mock('#features/auth/index.svelte.ts', () => ({ accessToken: { current: 'access-token' } })); +vi.mock('#features/events/api.svelte.ts', () => ({ queryKeys: { type: ['PersistentEvent'] } })); +vi.mock('#features/stacks/api.svelte.ts', () => ({ queryKeys: { type: ['Stack'] } })); +vi.mock('@foundatiofx/fetchclient', () => ({ useFetchClient: vi.fn() })); +vi.mock('@tanstack/svelte-query', () => ({ createMutation: vi.fn(), createQuery: vi.fn(), useQueryClient: vi.fn() })); + +import { invalidateBulkOperationAffectedQueries, queryKeys, updateBulkOperationFromMessage } from './api.svelte'; + +function createOperation(overrides: Partial = {}): ViewBulkOperation { + return { + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Delete events matching filter', + failed: 0, + id: 'operation-1', + organization_id: 'organization-id', + processed: 0, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BulkOperationStatus.Running, + succeeded: 0, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:00Z', + ...overrides + }; +} + +function createUpdate(id: string) { + return { + description: 'Delete events matching filter', + failed: 0, + id, + organization_id: 'organization-id', + processed: 75, + status: BulkOperationStatus.Running, + succeeded: 75, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:09Z' + }; +} + +describe('updateBulkOperationFromMessage', () => { + const queryClient = { getQueryData: vi.fn(), invalidateQueries: vi.fn(), setQueryData: vi.fn() }; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('writes the progress of a known operation into the cached list', () => { + // Arrange + queryClient.getQueryData.mockReturnValue([createOperation()]); + + // Act + updateBulkOperationFromMessage(queryClient as never, createUpdate('operation-1')); + + // Assert + expect(queryClient.setQueryData).toHaveBeenCalledWith(queryKeys.organization('organization-id'), [ + expect.objectContaining({ id: 'operation-1', processed: 75 }) + ]); + expect(queryClient.invalidateQueries).not.toHaveBeenCalled(); + }); + + it('reloads the list when the operation was started somewhere else', () => { + // Arrange + queryClient.getQueryData.mockReturnValue([createOperation()]); + + // Act + updateBulkOperationFromMessage(queryClient as never, createUpdate('started-elsewhere')); + + // Assert + expect(queryClient.setQueryData).not.toHaveBeenCalled(); + expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: queryKeys.organization('organization-id') }); + }); +}); + +describe('invalidateBulkOperationAffectedQueries', () => { + it('refreshes event and stack queries', async () => { + // Arrange + const queryClient = { invalidateQueries: vi.fn().mockResolvedValue(undefined) }; + + // Act + await invalidateBulkOperationAffectedQueries(queryClient as never); + + // Assert + expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: ['PersistentEvent'] }); + expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: ['Stack'] }); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte new file mode 100644 index 0000000000..330285106b --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte @@ -0,0 +1,69 @@ + + +
+
+ +
+

{operation.description}

+

+ {#if operation.status === BulkOperationStatus.Queued} + Waiting to start + {:else if operation.status === BulkOperationStatus.Running} + of + {:else if operation.status === BulkOperationStatus.Completed} + Completed · processed{#if operation.failed > 0}, failed{/if} + {:else if operation.status === BulkOperationStatus.Cancelled} + Cancelled after of + {:else} + Failed after of {#if operation.message}{` · ${operation.message}`}{/if} + {/if} +

+
+ {#if isActive} + + {:else} + + {/if} +
+ {#if isActive || operation.status === BulkOperationStatus.Completed} + + {/if} +
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts new file mode 100644 index 0000000000..64d3a3996b --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts @@ -0,0 +1,106 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { BulkOperationSelectionKind, BulkOperationStatus, type ViewBulkOperation } from '../models'; +import BulkOperationItem from './bulk-operation-item.svelte'; + +function createOperation(overrides: Partial = {}): ViewBulkOperation { + return { + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Delete 1,234 events matching filter', + failed: 0, + id: 'operation-1', + organization_id: 'organization-id', + processed: 500, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BulkOperationStatus.Running, + succeeded: 500, + total: 1234, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:00Z', + ...overrides + }; +} + +function statusText(): string { + return (screen.getByRole('status').textContent ?? '').replace(/\s+/g, ' ').trim(); +} + +describe('BulkOperationItem', () => { + afterEach(cleanup); + + it('shows the description, x of y progress and a Cancel button while running', async () => { + // Arrange + const onCancel = vi.fn(); + const operation = createOperation(); + render(BulkOperationItem, { props: { onCancel, onDismiss: vi.fn(), operation } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Cancel' })); + + // Assert + expect(screen.getByText('Delete 1,234 events matching filter')).toBeTruthy(); + expect(statusText()).toBe('500 of 1,234'); + expect(screen.getByRole('progressbar').getAttribute('aria-valuenow')).toBe('41'); + expect(onCancel).toHaveBeenCalledWith(operation); + expect(screen.queryByRole('button', { name: 'Dismiss' })).toBeNull(); + }); + + it('shows the queued state without a count', () => { + // Arrange & Act + render(BulkOperationItem, { + props: { onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation({ processed: 0, status: BulkOperationStatus.Queued }) } + }); + + // Assert + expect(statusText()).toBe('Waiting to start'); + expect(screen.getByRole('button', { name: 'Cancel' })).toBeTruthy(); + }); + + it('disables Cancel once cancellation was requested', () => { + // Arrange & Act + render(BulkOperationItem, { props: { isCancelling: true, onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation() } }); + + // Assert + expect((screen.getByRole('button', { name: 'Cancel' }) as HTMLButtonElement).disabled).toBe(true); + }); + + it('shows a completed operation with its failures and a Dismiss button', async () => { + // Arrange + const onDismiss = vi.fn(); + const operation = createOperation({ failed: 2, processed: 1234, status: BulkOperationStatus.Completed, succeeded: 1232 }); + render(BulkOperationItem, { props: { onCancel: vi.fn(), onDismiss, operation } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Dismiss' })); + + // Assert + expect(statusText()).toBe('Completed · 1,232 processed, 2 failed'); + expect(screen.queryByRole('button', { name: 'Cancel' })).toBeNull(); + expect(onDismiss).toHaveBeenCalledWith(operation); + }); + + it('shows how far a cancelled operation got', () => { + // Arrange & Act + render(BulkOperationItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation({ status: BulkOperationStatus.Cancelled }) } }); + + // Assert + expect(statusText()).toBe('Cancelled after 500 of 1,234'); + expect(screen.queryByRole('progressbar')).toBeNull(); + }); + + it('shows why an operation failed', () => { + // Arrange & Act + render(BulkOperationItem, { + props: { + onCancel: vi.fn(), + onDismiss: vi.fn(), + operation: createOperation({ message: 'The operation stopped because of an unexpected error.', status: BulkOperationStatus.Failed }) + } + }); + + // Assert + expect(statusText()).toBe('Failed after 500 of 1,234 · The operation stopped because of an unexpected error.'); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte new file mode 100644 index 0000000000..26e6378b54 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte @@ -0,0 +1,93 @@ + + +{#if visibleOperations.length > 0} +
+ {#each visibleOperations as operation (operation.id)} + + {/each} +
+{/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts new file mode 100644 index 0000000000..21a110e11d --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts @@ -0,0 +1,150 @@ +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { tick } from 'svelte'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { BulkOperationSelectionKind, BulkOperationStatus, type ViewBulkOperation } from '../models'; + +const mocks = vi.hoisted(() => ({ + cancelMutateAsync: vi.fn(), + invalidateAffected: vi.fn(), + queryClient: { id: 'query-client' }, + updateFromMessage: vi.fn() +})); + +vi.mock('@tanstack/svelte-query', () => ({ useQueryClient: () => mocks.queryClient })); +vi.mock('svelte-sonner', () => ({ toast: { error: vi.fn(), success: vi.fn() } })); +vi.mock('#features/organizations/context.svelte.ts', () => ({ organization: { current: 'organization-id' } })); +vi.mock('../api.svelte', async () => { + const { SvelteMap } = await import('svelte/reactivity'); + const state = new SvelteMap(); + + return { + __state: state, + cancelBulkOperation: () => ({ mutateAsync: mocks.cancelMutateAsync }), + getBulkOperationsQuery: () => ({ + get data() { + return state.get('operations'); + } + }), + invalidateBulkOperationAffectedQueries: mocks.invalidateAffected, + updateBulkOperationFromMessage: mocks.updateFromMessage + }; +}); + +import * as api from '../api.svelte'; +import BulkOperationPanel from './bulk-operation-panel.svelte'; + +const state = (api as unknown as { __state: Map }).__state; + +function createOperation(overrides: Partial = {}): ViewBulkOperation { + const now = new Date().toISOString(); + return { + created_by_user_id: 'user-id', + created_utc: now, + description: 'Delete 50 events matching filter', + failed: 0, + id: 'operation-1', + organization_id: 'organization-id', + processed: 20, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BulkOperationStatus.Running, + succeeded: 20, + total: 50, + type: 'events.delete', + updated_utc: now, + ...overrides + }; +} + +describe('BulkOperationPanel', () => { + beforeEach(() => { + state.clear(); + mocks.cancelMutateAsync.mockResolvedValue(undefined); + }); + + afterEach(cleanup); + + it('hydrates running operations from the list after a reload and ignores stale finished ones', async () => { + // Arrange + state.set('operations', [ + createOperation(), + createOperation({ id: 'long-finished', status: BulkOperationStatus.Completed, updated_utc: '2020-01-01T00:00:00Z' }) + ]); + + // Act + render(BulkOperationPanel); + await tick(); + + // Assert + expect(screen.getAllByText('Delete 50 events matching filter')).toHaveLength(1); + expect(screen.getByRole('status').textContent?.replace(/\s+/g, ' ').trim()).toBe('20 of 50'); + expect(mocks.invalidateAffected).not.toHaveBeenCalled(); + }); + + it('applies pushed updates for its organization and ignores other organizations', async () => { + // Arrange + state.set('operations', [createOperation()]); + render(BulkOperationPanel); + const update = { id: 'operation-1', organization_id: 'organization-id', processed: 30, status: BulkOperationStatus.Running }; + + // Act + document.dispatchEvent(new CustomEvent('BulkOperationUpdated', { detail: update })); + document.dispatchEvent(new CustomEvent('BulkOperationUpdated', { detail: { ...update, organization_id: 'another-organization' } })); + + // Assert + expect(mocks.updateFromMessage).toHaveBeenCalledTimes(1); + expect(mocks.updateFromMessage).toHaveBeenCalledWith(mocks.queryClient, update); + }); + + it('refreshes events and stacks once when a running operation finishes', async () => { + // Arrange + state.set('operations', [createOperation()]); + render(BulkOperationPanel); + await tick(); + + // Act + state.set('operations', [createOperation({ processed: 50, status: BulkOperationStatus.Completed, succeeded: 50 })]); + await tick(); + state.set('operations', [ + createOperation({ processed: 50, status: BulkOperationStatus.Completed, succeeded: 50, updated_utc: new Date().toISOString() }) + ]); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(1); + expect(mocks.invalidateAffected).toHaveBeenCalledWith(mocks.queryClient); + expect(screen.getByRole('status').textContent).toContain('Completed'); + }); + + it('requests cancellation of a running operation', async () => { + // Arrange + state.set('operations', [createOperation()]); + render(BulkOperationPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Cancel' })); + + // Assert + await waitFor(() => expect(mocks.cancelMutateAsync).toHaveBeenCalledWith('operation-1')); + }); + + it('removes a finished operation when it is dismissed', async () => { + // Arrange + state.set('operations', [createOperation({ status: BulkOperationStatus.Completed })]); + render(BulkOperationPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Dismiss' })); + + // Assert + expect(screen.queryByText('Delete 50 events matching filter')).toBeNull(); + }); + + it('renders nothing when there is nothing to show', () => { + // Arrange & Act + render(BulkOperationPanel); + + // Assert + expect(screen.queryByRole('region', { name: 'Bulk operations' })).toBeNull(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.test.ts new file mode 100644 index 0000000000..71bf1a2cb6 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from 'vitest'; + +import { + applyBulkOperationUpdate, + BulkOperationSelectionKind, + BulkOperationStatus, + FINISHED_OPERATION_VISIBLE_MS, + getProgressPercent, + selectVisibleOperations, + type ViewBulkOperation +} from './models'; + +function createOperation(overrides: Partial = {}): ViewBulkOperation { + return { + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Delete events matching filter', + failed: 0, + id: 'operation-1', + organization_id: 'organization-id', + processed: 0, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BulkOperationStatus.Running, + succeeded: 0, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:00Z', + ...overrides + }; +} + +function createUpdate(overrides: Record = {}) { + return { + description: 'Delete events matching filter', + failed: 0, + id: 'operation-1', + organization_id: 'organization-id', + processed: 0, + status: BulkOperationStatus.Running, + succeeded: 0, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:05Z', + ...overrides + }; +} + +describe('applyBulkOperationUpdate', () => { + it('patches the progress of a known operation', () => { + // Arrange + const operations = [createOperation(), createOperation({ id: 'operation-2' })]; + + // Act + const result = applyBulkOperationUpdate(operations, createUpdate({ failed: 1, processed: 40, succeeded: 39 })); + + // Assert + expect(result?.[0]).toMatchObject({ failed: 1, processed: 40, succeeded: 39 }); + expect(result?.[1]).toBe(operations[1]); + }); + + it('ignores a message that is older than what is already known', () => { + // Arrange + const operations = [createOperation({ processed: 60, updated_utc: '2026-01-01T10:00:10Z' })]; + + // Act + const result = applyBulkOperationUpdate(operations, createUpdate({ processed: 20 })); + + // Assert + expect(result).toBe(operations); + }); + + it('reports an unknown operation so the list can be reloaded', () => { + // Arrange + const operations = [createOperation()]; + + // Act + const result = applyBulkOperationUpdate(operations, createUpdate({ id: 'started-elsewhere', status: BulkOperationStatus.Queued, total: 5 })); + + // Assert + expect(result).toBeUndefined(); + }); +}); + +describe('getProgressPercent', () => { + it.each([ + [{ processed: 0, status: BulkOperationStatus.Queued, total: 0 }, 0], + [{ processed: 50, status: BulkOperationStatus.Running, total: 200 }, 25], + [{ processed: 5, status: BulkOperationStatus.Running, total: 3 }, 100], + [{ processed: 3, status: BulkOperationStatus.Completed, total: 0 }, 100] + ])('maps %j to %i percent', (operation, expected) => { + // Arrange & Act + const percent = getProgressPercent(operation); + + // Assert + expect(percent).toBe(expected); + }); +}); + +describe('selectVisibleOperations', () => { + const now = new Date('2026-01-01T10:30:00Z').getTime(); + + it('keeps active operations and recently finished ones, newest first', () => { + // Arrange + const active = createOperation({ created_utc: '2026-01-01T09:00:00Z', id: 'active', updated_utc: '2026-01-01T09:00:00Z' }); + const recent = createOperation({ + created_utc: '2026-01-01T10:20:00Z', + id: 'recent', + status: BulkOperationStatus.Completed, + updated_utc: new Date(now - FINISHED_OPERATION_VISIBLE_MS + 1000).toISOString() + }); + const old = createOperation({ + id: 'old', + status: BulkOperationStatus.Completed, + updated_utc: new Date(now - FINISHED_OPERATION_VISIBLE_MS - 1000).toISOString() + }); + + // Act + const result = selectVisibleOperations([active, old, recent], new Set(), now); + + // Assert + expect(result.map((operation) => operation.id)).toEqual(['recent', 'active']); + }); + + it('hides dismissed operations', () => { + // Arrange + const operation = createOperation({ id: 'dismissed' }); + + // Act + const result = selectVisibleOperations([operation], new Set(['dismissed']), now); + + // Assert + expect(result).toEqual([]); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.ts new file mode 100644 index 0000000000..b26afc3d72 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.ts @@ -0,0 +1,67 @@ +import type { BulkOperationUpdated } from '#features/websockets/models.ts'; + +import { BulkOperationSelectionKind, BulkOperationStatus, type NewBulkOperation, type ViewBulkOperation } from '#generated/api.ts'; + +export { BulkOperationSelectionKind, BulkOperationStatus, type NewBulkOperation, type ViewBulkOperation }; + +export const EVENTS_DELETE_OPERATION = 'events.delete'; + +/** Finished operations stay visible in the panel for this long, unless dismissed. */ +export const FINISHED_OPERATION_VISIBLE_MS = 2 * 60 * 1000; + +export function applyBulkOperationUpdate(operations: ViewBulkOperation[], update: BulkOperationUpdated): undefined | ViewBulkOperation[] { + const index = operations.findIndex((operation) => operation.id === update.id); + if (index < 0) { + return undefined; + } + + const current = operations[index]!; + if (new Date(update.updated_utc).getTime() < new Date(current.updated_utc).getTime()) { + // Messages can arrive out of order; never move an operation backwards. + return operations; + } + + const next = [...operations]; + next[index] = { + ...current, + description: update.description, + failed: update.failed, + message: update.message ?? null, + processed: update.processed, + status: update.status, + succeeded: update.succeeded, + total: update.total, + updated_utc: update.updated_utc + }; + + return next; +} + +export function getProgressPercent(operation: Pick): number { + if (operation.status === BulkOperationStatus.Completed) { + return 100; + } + + if (operation.total <= 0) { + return 0; + } + + return Math.min(100, Math.round((operation.processed / operation.total) * 100)); +} + +export function isBulkOperationActive(status: BulkOperationStatus): boolean { + return status === BulkOperationStatus.Queued || status === BulkOperationStatus.Running; +} + +/** Active operations plus recently finished ones the user has not dismissed, newest first. */ +export function selectVisibleOperations(operations: ViewBulkOperation[], dismissedIds: ReadonlySet, now: number): ViewBulkOperation[] { + return operations + .filter((operation) => { + if (dismissedIds.has(operation.id)) { + return false; + } + + return isBulkOperationActive(operation.status) || now - new Date(operation.updated_utc).getTime() < FINISHED_OPERATION_VISIBLE_MS; + }) + .sort((a, b) => new Date(b.created_utc).getTime() - new Date(a.created_utc).getTime()); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte index 115a40f3de..e8e2855fe1 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/dialogs/remove-event-dialog.svelte @@ -5,11 +5,13 @@ interface Props { count?: number; + /** The delete runs against everything matching the current filter instead of the selected rows. */ + matchingFilter?: boolean; open: boolean; remove: () => Promise; } - let { count = 1, open = $bindable(), remove }: Props = $props(); + let { count = 1, matchingFilter = false, open = $bindable(), remove }: Props = $props(); async function onSubmit() { await remove(); @@ -29,12 +31,17 @@ {/if} - Are you sure you want to delete - {#if count === 1} - this event + {#if matchingFilter} + Delete + {count === 1 ? 'event' : 'events'} matching this filter? This can't be undone. {:else} - events - {/if}? + Are you sure you want to delete + {#if count === 1} + this event + {:else} + events + {/if}? + {/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts new file mode 100644 index 0000000000..d63b4f5c50 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; + +import { createAllMatchingSelection, getEventsQueryIdentity } from './all-matching-selection.svelte'; + +describe('createAllMatchingSelection', () => { + it('is active only for the query it was made for', () => { + // Arrange + const identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id', time: '[now-7d TO now]' }); + const selection = createAllMatchingSelection(() => identity); + + // Act + selection.select(1234); + + // Assert + expect(selection.active).toEqual({ count: 1234, identity }); + }); + + it.each([ + ['filter', { filter: 'type:log', organizationId: 'organization-id', time: '[now-7d TO now]' }], + ['time range', { filter: 'type:error', organizationId: 'organization-id', time: '[now-30d TO now]' }], + ['project filter', { filter: 'type:error project:project-b', organizationId: 'organization-id', time: '[now-7d TO now]' }], + ['saved view', { filter: 'type:error', organizationId: 'organization-id', savedViewId: 'view-2', time: '[now-7d TO now]' }], + ['organization', { filter: 'type:error', organizationId: 'another-organization', time: '[now-7d TO now]' }] + ])('is dropped when the %s changes', (_, changed) => { + // Arrange + let identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id', time: '[now-7d TO now]' }); + const selection = createAllMatchingSelection(() => identity); + selection.select(1234); + + // Act + identity = getEventsQueryIdentity(changed); + + // Assert + expect(selection.active).toBeUndefined(); + }); + + it('does not come back when the original query is restored after being cleared', () => { + // Arrange + const original = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' }); + let identity = original; + const selection = createAllMatchingSelection(() => identity); + selection.select(10); + + // Act + identity = getEventsQueryIdentity({ filter: 'type:log', organizationId: 'organization-id' }); + selection.clear(); + identity = original; + + // Assert + expect(selection.active).toBeUndefined(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts new file mode 100644 index 0000000000..c6a31db878 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts @@ -0,0 +1,43 @@ +export interface AllMatchingSelection { + /** How many events matched when the user chose "select all matching". */ + count: number; + /** The query the selection was made for. */ + identity: string; +} + +/** Everything matching the current filter, as passed to an action menu. */ +export interface AllMatchingTarget { + count: number; + filter?: string; + offset?: string; + organizationId: string; + time?: string; +} + +/** + * Tracks the Gmail-style "everything matching this filter" selection. The selection belongs to the query it was made + * for: as soon as the query identity (organization, filter, time range, saved view) changes it is no longer active, so + * an action can never run against a different set of events than the user confirmed. + */ +export function createAllMatchingSelection(getIdentity: () => string) { + let selection = $state(); + + return { + get active(): AllMatchingSelection | undefined { + return selection && selection.identity === getIdentity() ? selection : undefined; + }, + clear() { + selection = undefined; + }, + select(count: number) { + selection = { + count, + identity: getIdentity() + }; + } + }; +} + +export function getEventsQueryIdentity(parts: { filter?: null | string; organizationId?: string; savedViewId?: string; time?: null | string }): string { + return JSON.stringify([parts.organizationId ?? null, parts.filter ?? null, parts.time ?? null, parts.savedViewId ?? null]); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte index fec3c8e401..131b8be99d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte @@ -3,24 +3,35 @@ @@ -56,10 +97,10 @@ {#snippet child({ props })} + +{:else if canSelectAllMatching} +
+ All {pageCount === 1 ? 'event' : 'events'} on this page {pageCount === 1 ? 'is' : 'are'} selected. + +
+{/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts new file mode 100644 index 0000000000..f9edce4079 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts @@ -0,0 +1,53 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import EventsSelectAllBanner from './events-select-all-banner.svelte'; + +function bannerText(): string { + return (screen.getByRole('status').textContent ?? '').replace(/\s+/g, ' ').trim(); +} + +describe('EventsSelectAllBanner', () => { + afterEach(cleanup); + + it('offers to select everything matching the filter once the page is selected', async () => { + // Arrange + const onSelectAllMatching = vi.fn(); + render(EventsSelectAllBanner, { props: { isPageSelected: true, onClear: vi.fn(), onSelectAllMatching, pageCount: 20, totalMatching: 1234 } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Select all 1,234 events matching this filter' })); + + // Assert + expect(bannerText()).toBe('All 20 events on this page are selected. Select all 1,234 events matching this filter'); + expect(onSelectAllMatching).toHaveBeenCalledOnce(); + }); + + it('confirms that everything is selected and offers to clear it', async () => { + // Arrange + const onClear = vi.fn(); + render(EventsSelectAllBanner, { + props: { allMatchingCount: 1234, isPageSelected: true, onClear, onSelectAllMatching: vi.fn(), pageCount: 20, totalMatching: 1234 } + }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Clear selection' })); + + // Assert + expect(bannerText()).toBe('All 1,234 events are selected · Clear selection'); + expect(screen.queryByRole('button', { name: /Select all/ })).toBeNull(); + expect(onClear).toHaveBeenCalledOnce(); + }); + + it.each([ + ['only some rows are selected', { isPageSelected: false, pageCount: 20, totalMatching: 1234 }], + ['the page already shows every match', { isPageSelected: true, pageCount: 20, totalMatching: 20 }], + ['the total is not known yet', { isPageSelected: true, pageCount: 20, totalMatching: undefined }] + ])('shows nothing when %s', (_, props) => { + // Arrange & Act + render(EventsSelectAllBanner, { props: { ...props, onClear: vi.fn(), onSelectAllMatching: vi.fn() } }); + + // Assert + expect(screen.queryByRole('status')).toBeNull(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts index 557b9f1f3e..e5eb392f9d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/options.svelte.ts @@ -62,6 +62,7 @@ export function getColumns renderComponent(Checkbox, { + 'aria-label': 'Select all rows', checked: table.getIsAllRowsSelected(), indeterminate: table.getIsSomeRowsSelected() && !table.getIsAllRowsSelected(), onCheckedChange: (checked: 'indeterminate' | boolean) => diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/index.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/index.ts new file mode 100644 index 0000000000..25eee615aa --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/index.ts @@ -0,0 +1,7 @@ +import Root from "./progress.svelte"; + +export { + Root, + // + Root as Progress, +}; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/progress.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/progress.svelte new file mode 100644 index 0000000000..12c78f5030 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/shared/components/ui/progress/progress.svelte @@ -0,0 +1,27 @@ + + + +
+
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts index 461805e466..e6e694de72 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts @@ -1,9 +1,26 @@ +import type { BulkOperationStatus } from '#generated/api.ts'; + export enum ChangeType { Added = 0, Saved = 1, Removed = 2 } +export interface BulkOperationUpdated { + description: string; + failed: number; + id: string; + message?: string; + organization_id: string; + processed: number; + project_id?: string; + status: BulkOperationStatus; + succeeded: number; + total: number; + type: string; + updated_utc: string; +} + export interface EntityChanged { change_type: ChangeType; data: Record; @@ -49,19 +66,22 @@ export interface WebSocketMessage { type: T; } -export type WebSocketMessageType = 'PlanChanged' | 'PlanOverage' | 'ReleaseNotification' | 'SystemNotification' | 'UserMembershipChanged' | `${string}Changed`; +export type WebSocketMessageType = + 'BulkOperationUpdated' | 'PlanChanged' | 'PlanOverage' | 'ReleaseNotification' | 'SystemNotification' | 'UserMembershipChanged' | `${string}Changed`; -export type WebSocketMessageValue = T extends 'PlanChanged' - ? PlanChanged - : T extends 'PlanOverage' - ? PlanOverage - : T extends 'UserMembershipChanged' - ? UserMembershipChanged - : T extends 'ReleaseNotification' - ? ReleaseNotification - : T extends 'SystemNotification' - ? SystemNotification - : EntityChanged; +export type WebSocketMessageValue = T extends 'BulkOperationUpdated' + ? BulkOperationUpdated + : T extends 'PlanChanged' + ? PlanChanged + : T extends 'PlanOverage' + ? PlanOverage + : T extends 'UserMembershipChanged' + ? UserMembershipChanged + : T extends 'ReleaseNotification' + ? ReleaseNotification + : T extends 'SystemNotification' + ? SystemNotification + : EntityChanged; export function isEntityChangedType(message: { message: unknown; type: WebSocketMessageType }): message is WebSocketMessage<`${string}Changed`> { return message.type !== 'PlanChanged' && message.type !== 'UserMembershipChanged' && message.type.endsWith('Changed'); @@ -73,7 +93,7 @@ export function isPlanOverageType(message: { message: unknown; type: WebSocketMe export function isWebSocketMessageType(type: string): type is WebSocketMessageType { return ( - (['PlanChanged', 'PlanOverage', 'UserMembershipChanged', 'ReleaseNotification', 'SystemNotification'] as const).includes( + (['BulkOperationUpdated', 'PlanChanged', 'PlanOverage', 'UserMembershipChanged', 'ReleaseNotification', 'SystemNotification'] as const).includes( type as Exclude ) || type.endsWith('Changed') ); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts index 4010d85b3e..3112cb20c9 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts @@ -7,6 +7,19 @@ export enum StackStatus { Discarded = "discarded", } +export enum BulkOperationStatus { + Queued = "Queued", + Running = "Running", + Completed = "Completed", + Failed = "Failed", + Cancelled = "Cancelled", +} + +export enum BulkOperationSelectionKind { + Ids = "Ids", + Filter = "Filter", +} + export enum BillingStatus { Trialing = 0, Active = 1, @@ -241,6 +254,36 @@ export interface Login { invite_token?: null | string; } +export interface NewBulkOperation { + /** The action to run, e.g. events.delete. */ + type: string; + /** @pattern ^[a-fA-F0-9]{24}$ */ + organization_id: string; + /** + * Optionally limits the operation to one project of the organization. + * @pattern ^[a-fA-F0-9]{24}$ + */ + project_id?: null | string; + selection: NewBulkOperationSelection; + /** Action specific parameters. Actions without parameters reject any value here. */ + parameters?: null | Record; +} + +export interface NewBulkOperationSelection { + kind: BulkOperationSelectionKind; + /** The ids to process when Kind is ids. */ + ids?: string[] | null; + /** The search filter when Kind is filter. An empty filter matches everything in the time window. */ + filter?: null | string; + /** + * The time range of the filter, e.g. [now-7d TO now]. It is resolved to absolute UTC dates when the + * operation is created. + */ + time?: null | string; + /** The UTC offset used to resolve Time, e.g. -5h. */ + offset?: null | string; +} + export interface NewOrganization { name: string; } @@ -755,6 +798,12 @@ export interface User { id: string; /** The organizations that the user has access to. */ organization_ids: string[]; + /** + * Session revocation generation, rotated when a password changes or resets, or an external login is removed or transferred away. + * Remains null until rotation; active users with a null generation accept only matching legacy null token generations. + * This value does not describe the password hashing format. + */ + authentication_version?: null | string; password?: null | string; salt?: null | string; password_reset_token?: null | string; @@ -806,6 +855,59 @@ export interface UserSavedViewOrderPreference { saved_view_ids: string[]; } +export interface ViewBulkOperation { + /** @pattern ^[a-fA-F0-9]{24}$ */ + id: string; + /** @pattern ^[a-fA-F0-9]{24}$ */ + organization_id: string; + /** @pattern ^[a-fA-F0-9]{24}$ */ + project_id?: null | string; + type: string; + description: string; + status: BulkOperationStatus; + selection: ViewBulkOperationSelection; + /** @format int64 */ + total: number; + /** @format int64 */ + processed: number; + /** @format int64 */ + succeeded: number; + /** @format int64 */ + failed: number; + /** @pattern ^[a-fA-F0-9]{24}$ */ + created_by_user_id: string; + /** @format date-time */ + created_utc: string; + /** @format date-time */ + started_utc?: null | string; + /** @format date-time */ + completed_utc?: null | string; + /** @format date-time */ + updated_utc: string; + message?: null | string; +} + +export interface ViewBulkOperationSelection { + kind: BulkOperationSelectionKind; + /** + * The number of ids when the selection is an id list. + * @format int32 + */ + id_count?: null | number; + filter?: null | string; + time?: null | string; + /** + * The absolute start of the resolved time window. + * @format date-time + */ + utc_start?: null | string; + /** + * The absolute end of the resolved time window, never later than when the operation was created. + * @format date-time + */ + utc_end?: null | string; +} + export interface ViewCurrentUser { hash?: null | string; has_local_account: boolean; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts index a0ad26433e..cf92fbbc21 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts @@ -27,6 +27,14 @@ export const StackStatusSchema = zodEnum([ "ignored", "discarded", ]); +export const BulkOperationStatusSchema = zodEnum([ + "Queued", + "Running", + "Completed", + "Failed", + "Cancelled", +]); +export const BulkOperationSelectionKindSchema = zodEnum(["Ids", "Filter"]); export const BillingStatusSchema = union([ literal(0), literal(1), @@ -294,6 +302,46 @@ export const LoginSchema = object({ }); export type LoginFormData = Infer; +export const NewBulkOperationSchema = object({ + type: string() + .min(1, "Type is required") + .max(100, "Type must be at most 100 characters"), + organization_id: string() + .length(24, "Organization id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Organization id has invalid format"), + project_id: string() + .length(24, "Project id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Project id has invalid format") + .nullable() + .optional(), + selection: lazy(() => NewBulkOperationSelectionSchema), + parameters: record(string(), string()).nullable().optional(), +}); +export type NewBulkOperationFormData = Infer; + +export const NewBulkOperationSelectionSchema = object({ + kind: BulkOperationSelectionKindSchema, + ids: array(string()).nullable().optional(), + filter: string() + .min(1, "Filter is required") + .max(2000, "Filter must be at most 2000 characters") + .nullable() + .optional(), + time: string() + .min(1, "Time is required") + .max(100, "Time must be at most 100 characters") + .nullable() + .optional(), + offset: string() + .min(1, "Offset is required") + .max(20, "Offset must be at most 20 characters") + .nullable() + .optional(), +}); +export type NewBulkOperationSelectionFormData = Infer< + typeof NewBulkOperationSelectionSchema +>; + export const NewOrganizationSchema = object({ name: string().min(1, "Name is required"), }); @@ -889,6 +937,7 @@ export const UserSchema = object({ .length(24, "Id must be exactly 24 characters") .regex(/^[a-fA-F0-9]{24}$/, "Id has invalid format"), organization_ids: array(string()), + authentication_version: string().nullable().optional(), password: string().min(1, "Password is required").nullable().optional(), salt: string().min(1, "Salt is required").nullable().optional(), password_reset_token: string() @@ -952,6 +1001,49 @@ export type UserSavedViewOrderPreferenceFormData = Infer< typeof UserSavedViewOrderPreferenceSchema >; +export const ViewBulkOperationSchema = object({ + id: string() + .length(24, "Id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Id has invalid format"), + organization_id: string() + .length(24, "Organization id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Organization id has invalid format"), + project_id: string() + .length(24, "Project id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Project id has invalid format") + .nullable() + .optional(), + type: string().min(1, "Type is required"), + description: string().min(1, "Description is required"), + status: BulkOperationStatusSchema, + selection: lazy(() => ViewBulkOperationSelectionSchema), + total: int(), + processed: int(), + succeeded: int(), + failed: int(), + created_by_user_id: string() + .length(24, "Created by user id must be exactly 24 characters") + .regex(/^[a-fA-F0-9]{24}$/, "Created by user id has invalid format"), + created_utc: iso.datetime(), + started_utc: iso.datetime().nullable().optional(), + completed_utc: iso.datetime().nullable().optional(), + updated_utc: iso.datetime(), + message: string().min(1, "Message is required").nullable().optional(), +}); +export type ViewBulkOperationFormData = Infer; + +export const ViewBulkOperationSelectionSchema = object({ + kind: BulkOperationSelectionKindSchema, + id_count: int32().nullable().optional(), + filter: string().min(1, "Filter is required").nullable().optional(), + time: string().min(1, "Time is required").nullable().optional(), + utc_start: iso.datetime().nullable().optional(), + utc_end: iso.datetime().nullable().optional(), +}); +export type ViewBulkOperationSelectionFormData = Infer< + typeof ViewBulkOperationSelectionSchema +>; + export const ViewCurrentUserSchema = object({ hash: string().min(1, "Hash is required").nullable().optional(), has_local_account: boolean(), diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index 445525a13d..567083e123 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -29,6 +29,7 @@ import { getIntercomTokenQuery } from '#features/auth/api.svelte.ts'; import { accessToken, gotoLogin } from '#features/auth/index.svelte.ts'; import { ChangePlanDialogHost, UpgradeRequiredDialog } from '#features/billing/index.ts'; + import BulkOperationPanel from '#features/bulk-operations/components/bulk-operation-panel.svelte'; import { createOrganizationEventNotificationRefresher, invalidatePersistentEventQueries, @@ -1008,6 +1009,7 @@ + {/if} + getEventsQueryIdentity({ + filter: eventsQueryParameters.filter, + organizationId: organization.current, + savedViewId: savedViewsState.activeSavedView?.id, + time: eventsQueryParameters.time + }) + ); + const pageRowCount = $derived(table.getRowModel().rows.length); + const isPageSelected = $derived(pageRowCount > 0 && table.getIsAllRowsSelected()); + + // The same request the list makes, asking only for the total, so the count always agrees with the list. + const matchingTotalQuery = getOrganizationEventsQuery({ + enabled: () => !isSavedViewRoutePending && isPageSelected, + get params() { + return { + filter: eventsQueryParameters.filter, + include: 'total' as const, + limit: 1, + mode: 'summary' as const, + offset: DEFAULT_OFFSET, + time: eventsQueryParameters.time + }; + }, + route: { + get organizationId() { + return organization.current; + } + } + }); + const totalMatching = $derived((matchingTotalQuery.data?.meta.total ?? eventsQuery.data?.meta.total) as number | undefined); + const allMatchingTarget = $derived.by(() => { + const selection = allMatching.active; + if (!selection || !organization.current) { + return undefined; + } + + return { + count: selection.count, + filter: eventsQueryParameters.filter ?? '', + offset: DEFAULT_OFFSET, + organizationId: organization.current, + time: eventsQueryParameters.time + }; + }); + + $effect(() => { + if (!allMatching.active || !isPageSelected) { + untrack(() => allMatching.clear()); + } + }); + async function handleRefresh() { table.resetRowSelection(); await eventsQuery.refetch(); @@ -822,6 +878,15 @@ table.setPageIndex(0); } + async function selectAllMatching() { + // Count again right now, so the number the user confirms is current. + const result = await matchingTotalQuery.refetch(); + const count = (result.data?.meta.total ?? totalMatching) as number | undefined; + if (count !== undefined) { + allMatching.select(count); + } + } + let reconcileTotalRequested = false; const debouncedRefetch = debounce(1500, async () => { const shouldReconcileTotal = reconcileTotalRequested; @@ -1038,6 +1103,18 @@ /> {/if} + { + allMatching.clear(); + table.resetRowSelection(); + }} + onSelectAllMatching={selectAllMatching} + pageCount={pageRowCount} + {totalMatching} + /> +
{#snippet footerChildren()}
- + allMatching.clear()} {table} />
diff --git a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs index fe059935db..4db79ecaa7 100644 --- a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs +++ b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs @@ -38,6 +38,7 @@ await Task.WhenAll( _subscriber.SubscribeAsync(OnEntityChangedAsync, shutdownToken), _subscriber.SubscribeAsync(OnPlanChangedAsync, shutdownToken), _subscriber.SubscribeAsync(OnPlanOverageAsync, shutdownToken), + _subscriber.SubscribeAsync(OnBulkOperationUpdatedAsync, shutdownToken), _subscriber.SubscribeAsync(OnUserMembershipChangedAsync, shutdownToken), _subscriber.SubscribeAsync(OnReleaseNotificationAsync, shutdownToken), _subscriber.SubscribeAsync(OnSystemNotificationAsync, shutdownToken) @@ -158,6 +159,17 @@ private Task OnPlanOverageAsync(PlanOverage planOverage, CancellationToken cance return Task.CompletedTask; } + internal Task OnBulkOperationUpdatedAsync(BulkOperationUpdated bulkOperationUpdated, CancellationToken cancellationToken = default) + { + if (bulkOperationUpdated is not null) + { + _logger.LogTrace("Sending bulk operation {OperationId} update to organization: {Organization}", bulkOperationUpdated.Id, bulkOperationUpdated.OrganizationId); + return GroupSendAsync(bulkOperationUpdated.OrganizationId, bulkOperationUpdated); + } + + return Task.CompletedTask; + } + private Task OnPlanChangedAsync(PlanChanged planChanged, CancellationToken cancellationToken = default) { if (planChanged is not null) diff --git a/src/Exceptionless.Web/Models/BulkOperation/NewBulkOperation.cs b/src/Exceptionless.Web/Models/BulkOperation/NewBulkOperation.cs new file mode 100644 index 0000000000..5dc51debbf --- /dev/null +++ b/src/Exceptionless.Web/Models/BulkOperation/NewBulkOperation.cs @@ -0,0 +1,51 @@ +using System.ComponentModel.DataAnnotations; +using Exceptionless.Core.Attributes; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BulkOperations; + +namespace Exceptionless.Web.Models; + +public record NewBulkOperation : IOwnedByOrganization +{ + /// The action to run, e.g. events.delete. + [Required] + [MaxLength(100)] + public string Type { get; set; } = null!; + + [ObjectId] + public string OrganizationId { get; set; } = null!; + + /// Optionally limits the operation to one project of the organization. + [ObjectId] + public string? ProjectId { get; set; } + + [Required] + public NewBulkOperationSelection Selection { get; set; } = null!; + + /// Action specific parameters. Actions without parameters reject any value here. + public Dictionary? Parameters { get; set; } +} + +public record NewBulkOperationSelection +{ + public BulkOperationSelectionKind Kind { get; set; } + + /// The ids to process when is ids. + [MaxLength(1000)] + public List? Ids { get; set; } + + /// The search filter when is filter. An empty filter matches everything in the time window. + [MaxLength(2000)] + public string? Filter { get; set; } + + /// + /// The time range of the filter, e.g. [now-7d TO now]. It is resolved to absolute UTC dates when the + /// operation is created. + /// + [MaxLength(100)] + public string? Time { get; set; } + + /// The UTC offset used to resolve , e.g. -5h. + [MaxLength(20)] + public string? Offset { get; set; } +} diff --git a/src/Exceptionless.Web/Models/BulkOperation/ViewBulkOperation.cs b/src/Exceptionless.Web/Models/BulkOperation/ViewBulkOperation.cs new file mode 100644 index 0000000000..72cab11289 --- /dev/null +++ b/src/Exceptionless.Web/Models/BulkOperation/ViewBulkOperation.cs @@ -0,0 +1,52 @@ +using Exceptionless.Core.Attributes; +using Exceptionless.Core.Models.BulkOperations; + +namespace Exceptionless.Web.Models; + +public record ViewBulkOperation +{ + [ObjectId] + public string Id { get; set; } = null!; + + [ObjectId] + public string OrganizationId { get; set; } = null!; + + [ObjectId] + public string? ProjectId { get; set; } + + public string Type { get; set; } = null!; + public string Description { get; set; } = null!; + public BulkOperationStatus Status { get; set; } + public ViewBulkOperationSelection Selection { get; set; } = null!; + + public long Total { get; set; } + public long Processed { get; set; } + public long Succeeded { get; set; } + public long Failed { get; set; } + + [ObjectId] + public string CreatedByUserId { get; set; } = null!; + + public DateTime CreatedUtc { get; set; } + public DateTime? StartedUtc { get; set; } + public DateTime? CompletedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } + public string? Message { get; set; } +} + +public record ViewBulkOperationSelection +{ + public BulkOperationSelectionKind Kind { get; set; } + + /// The number of ids when the selection is an id list. + public int? IdCount { get; set; } + + public string? Filter { get; set; } + public string? Time { get; set; } + + /// The absolute start of the resolved time window. + public DateTime? UtcStart { get; set; } + + /// The absolute end of the resolved time window, never later than when the operation was created. + public DateTime? UtcEnd { get; set; } +} diff --git a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json index c0474d670b..20858e3b00 100644 --- a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json +++ b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json @@ -845,6 +845,48 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "POST", + "route": "/api/v2/bulk-operations", + "displayName": "HTTP: POST api/v2/bulk-operations", + "tags": [ + "BulkOperation" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, + { + "method": "GET", + "route": "/api/v2/bulk-operations/{id:objectid}", + "displayName": "HTTP: GET api/v2/bulk-operations/{id:objectid}", + "tags": [ + "BulkOperation" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, + { + "method": "POST", + "route": "/api/v2/bulk-operations/{id:objectid}/cancel", + "displayName": "HTTP: POST api/v2/bulk-operations/{id:objectid}/cancel", + "tags": [ + "BulkOperation" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "POST", "route": "/api/v2/contact", @@ -1458,6 +1500,20 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "GET", + "route": "/api/v2/organizations/{organizationId:objectid}/bulk-operations", + "displayName": "HTTP: GET api/v2/organizations/{organizationId:objectid}/bulk-operations", + "tags": [ + "BulkOperation" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "GET", "route": "/api/v2/organizations/{organizationId:objectid}/events", diff --git a/tests/Exceptionless.Tests/Api/Data/openapi.json b/tests/Exceptionless.Tests/Api/Data/openapi.json index 88be9cf072..a2b561d3e4 100644 --- a/tests/Exceptionless.Tests/Api/Data/openapi.json +++ b/tests/Exceptionless.Tests/Api/Data/openapi.json @@ -3992,6 +3992,225 @@ } } }, + "/api/v2/bulk-operations": { + "post": { + "tags": [ + "BulkOperation" + ], + "summary": "Create", + "description": "Starts an action, such as \u0060events.delete\u0060, on everything that matches a selection. The selection is either a list of ids\nor a filter with a time range. The time range is resolved to absolute UTC dates when the operation is created, and events\nstored after that moment are never included. The operation runs in the background; poll it or listen for\n\u0060BulkOperationUpdated\u0060 messages to follow its progress.", + "requestBody": { + "description": "The bulk operation to start.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewBulkOperation" + } + }, + "application/*\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/NewBulkOperation" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "Accepted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ViewBulkOperation" + } + } + } + }, + "403": { + "description": "The action is not permitted for this organization or project.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "404": { + "description": "The organization or project could not be found.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "409": { + "description": "A destructive operation is already running for this project.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "422": { + "description": "The request is not valid.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "426": { + "description": "The filter uses a feature that requires a plan upgrade.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/api/v2/bulk-operations/{id}": { + "get": { + "tags": [ + "BulkOperation" + ], + "summary": "Get by id", + "operationId": "GetBulkOperationById", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "The identifier of the bulk operation.", + "required": true, + "schema": { + "pattern": "^[a-zA-Z\\d]{24,36}$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ViewBulkOperation" + } + } + } + }, + "404": { + "description": "The bulk operation could not be found.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/api/v2/organizations/{organizationId}/bulk-operations": { + "get": { + "tags": [ + "BulkOperation" + ], + "summary": "Get by organization", + "description": "Returns the active and recently finished bulk operations of the organization, newest first. Finished operations are kept for seven days.", + "parameters": [ + { + "name": "organizationId", + "in": "path", + "description": "The identifier of the organization.", + "required": true, + "schema": { + "pattern": "^[a-zA-Z\\d]{24,36}$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ViewBulkOperation" + } + } + } + } + }, + "404": { + "description": "The organization could not be found.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, + "/api/v2/bulk-operations/{id}/cancel": { + "post": { + "tags": [ + "BulkOperation" + ], + "summary": "Cancel", + "description": "Requests cancellation. A running operation stops after the batch it is processing; work already done is not undone. Cancelling a finished operation has no effect.", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "The identifier of the bulk operation.", + "required": true, + "schema": { + "pattern": "^[a-zA-Z\\d]{24,36}$", + "type": "string" + } + } + ], + "responses": { + "202": { + "description": "Accepted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ViewBulkOperation" + } + } + } + }, + "404": { + "description": "The bulk operation could not be found.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + } + } + } + }, "/api/v2/users/me": { "get": { "tags": [ @@ -12113,6 +12332,32 @@ "Unpaid" ] }, + "BulkOperationSelectionKind": { + "enum": [ + "Ids", + "Filter" + ], + "x-enumNames": [ + "Ids", + "Filter" + ] + }, + "BulkOperationStatus": { + "enum": [ + "Queued", + "Running", + "Completed", + "Failed", + "Cancelled" + ], + "x-enumNames": [ + "Queued", + "Running", + "Completed", + "Failed", + "Cancelled" + ] + }, "ChangePasswordModel": { "required": [ "current_password", @@ -12481,6 +12726,96 @@ } } }, + "NewBulkOperation": { + "required": [ + "type", + "selection", + "organization_id" + ], + "type": "object", + "properties": { + "type": { + "maxLength": 100, + "type": "string", + "description": "The action to run, e.g. events.delete." + }, + "organization_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": "string" + }, + "project_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": [ + "null", + "string" + ], + "description": "Optionally limits the operation to one project of the organization." + }, + "selection": { + "$ref": "#/components/schemas/NewBulkOperationSelection" + }, + "parameters": { + "type": [ + "null", + "object" + ], + "additionalProperties": { + "type": "string" + }, + "description": "Action specific parameters. Actions without parameters reject any value here." + } + } + }, + "NewBulkOperationSelection": { + "required": [ + "kind" + ], + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/BulkOperationSelectionKind" + }, + "ids": { + "maxItems": 1000, + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + }, + "description": "The ids to process when Kind is ids." + }, + "filter": { + "maxLength": 2000, + "type": [ + "null", + "string" + ], + "description": "The search filter when Kind is filter. An empty filter matches everything in the time window." + }, + "time": { + "maxLength": 100, + "type": [ + "null", + "string" + ], + "description": "The time range of the filter, e.g. [now-7d TO now]. It is resolved to absolute UTC dates when the\noperation is created." + }, + "offset": { + "maxLength": 20, + "type": [ + "null", + "string" + ], + "description": "The UTC offset used to resolve Time, e.g. -5h." + } + } + }, "NewOrganization": { "required": [ "name" @@ -14329,6 +14664,156 @@ } } }, + "ViewBulkOperation": { + "required": [ + "id", + "organization_id", + "type", + "description", + "status", + "selection", + "total", + "processed", + "succeeded", + "failed", + "created_by_user_id", + "created_utc", + "updated_utc" + ], + "type": "object", + "properties": { + "id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": "string" + }, + "organization_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": "string" + }, + "project_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": [ + "null", + "string" + ] + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "status": { + "$ref": "#/components/schemas/BulkOperationStatus" + }, + "selection": { + "$ref": "#/components/schemas/ViewBulkOperationSelection" + }, + "total": { + "type": "integer", + "format": "int64" + }, + "processed": { + "type": "integer", + "format": "int64" + }, + "succeeded": { + "type": "integer", + "format": "int64" + }, + "failed": { + "type": "integer", + "format": "int64" + }, + "created_by_user_id": { + "maxLength": 24, + "minLength": 24, + "pattern": "^[a-fA-F0-9]{24}$", + "type": "string" + }, + "created_utc": { + "type": "string", + "format": "date-time" + }, + "started_utc": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "completed_utc": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated_utc": { + "type": "string", + "format": "date-time" + }, + "message": { + "type": [ + "null", + "string" + ] + } + } + }, + "ViewBulkOperationSelection": { + "required": [ + "kind" + ], + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/BulkOperationSelectionKind" + }, + "id_count": { + "type": [ + "null", + "integer" + ], + "description": "The number of ids when the selection is an id list.", + "format": "int32" + }, + "filter": { + "type": [ + "null", + "string" + ] + }, + "time": { + "type": [ + "null", + "string" + ] + }, + "utc_start": { + "type": [ + "null", + "string" + ], + "description": "The absolute start of the resolved time window.", + "format": "date-time" + }, + "utc_end": { + "type": [ + "null", + "string" + ], + "description": "The absolute end of the resolved time window, never later than when the operation was created.", + "format": "date-time" + } + } + }, "ViewCurrentUser": { "required": [ "has_local_account", @@ -15211,6 +15696,9 @@ { "name": "SavedView" }, + { + "name": "BulkOperation" + }, { "name": "User" }, diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs new file mode 100644 index 0000000000..218394b0b2 --- /dev/null +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -0,0 +1,717 @@ +using System.Net; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Hubs; +using Exceptionless.Tests.Utility; +using Exceptionless.Web.Hubs; +using Exceptionless.Web.Models; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Repositories; +using Foundatio.Repositories.Utility; +using Foundatio.Utility; +using Microsoft.AspNetCore.Mvc; +using Xunit; + +namespace Exceptionless.Tests.Api.Endpoints; + +public class BulkOperationEndpointTests : IntegrationTestsBase +{ + private readonly IEventRepository _eventRepository; + private readonly BulkOperationService _bulkOperationService; + private readonly WorkItemJob _workItemJob; + + public BulkOperationEndpointTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _eventRepository = GetService(); + _bulkOperationService = GetService(); + _workItemJob = GetService(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + await GetService().CreateDataAsync(); + } + + [Fact] + public async Task Create_FilterSelection_DeletesEverythingMatchingThroughTheQueuedJob() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Type(Event.KnownTypes.Log).Date(now.AddHours(-1)); + d.Event().TestProject().Type(Event.KnownTypes.Log).Date(now.AddDays(-2)); + d.Event().TestProject().Project(SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID).Type(Event.KnownTypes.Log).Date(now.AddHours(-2)); + }); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "type:log")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var finished = await GetOperationAsync(created.Id); + + // Assert + Assert.Equal(BulkOperationStatus.Queued, created.Status); + Assert.Equal("events.delete", created.Type); + Assert.Equal(3, created.Total); + Assert.Equal("Delete 3 events matching filter", created.Description); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(3, finished.Processed); + Assert.Equal(3, finished.Succeeded); + Assert.Equal(0, finished.Failed); + Assert.Empty(await GetRemainingMessagesAsync(events)); + } + + [Fact] + public async Task Create_FilterSelection_OnlyDeletesMatchesInsideScopeFilterAndWindow() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Type(Event.KnownTypes.Log).Message("match").Date(now.AddHours(-1)); + d.Event().TestProject().Type(Event.KnownTypes.Error).Message("other type").Date(now.AddHours(-1)); + d.Event().TestProject().Type(Event.KnownTypes.Log).Message("outside window").Date(now.AddDays(-2)); + d.Event().TestProject().Project(SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID).Type(Event.KnownTypes.Log).Message("other project").Date(now.AddHours(-1)); + d.Event().FreeProject().Type(Event.KnownTypes.Log).Message("other organization").Date(now.AddHours(-1)); + }); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "type:log", "[now-1d TO now]")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + Assert.Equal(1, created.Total); + var remaining = await GetRemainingMessagesAsync(events); + Assert.DoesNotContain("match", remaining); + Assert.Contains("other type", remaining); + Assert.Contains("outside window", remaining); + Assert.Contains("other project", remaining); + Assert.Contains("other organization", remaining); + } + + [Fact] + public async Task Create_FilterNamingAnotherOrganization_CannotReachItsEvents() + { + // Arrange + var (_, events) = await CreateDataAsync(d => d.Event().FreeProject()); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, $"organization:{SampleDataService.FREE_ORG_ID}")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + Assert.Equal(0, created.Total); + Assert.NotNull(await _eventRepository.GetByIdAsync(events.Single().Id)); + } + + [Fact] + public async Task Create_FilterSelection_ResolvesTimeWindowToAbsoluteDatesCappedAtCreation() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("past").Date(now.AddHours(-1)); + d.Event().TestProject().Message("future dated").Date(now.AddMinutes(10)); + }); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "[now-7d TO now+7d]")); + TimeProvider.Advance(TimeSpan.FromHours(3)); + var reloaded = await GetOperationAsync(created.Id); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + Assert.NotNull(created.Selection.UtcEnd); + Assert.True(created.Selection.UtcEnd <= created.CreatedUtc, "The window end must not be later than the moment the operation was created."); + Assert.True(created.Selection.UtcStart < created.Selection.UtcEnd); + Assert.Equal(created.Selection.UtcStart, reloaded.Selection.UtcStart); + Assert.Equal(created.Selection.UtcEnd, reloaded.Selection.UtcEnd); + Assert.Equal(["future dated"], (await GetRemainingMessagesAsync(events)).ToArray()); + } + + [Fact] + public async Task Create_FilterSelection_DoesNotIncludeEventsStoredAfterCreation() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + await CreateDataAsync(d => d.Event().TestProject().Message("before").Date(now.AddHours(-1))); + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + + // Act + TimeProvider.Advance(TimeSpan.FromMinutes(2)); + var (_, lateEvents) = await CreateDataAsync(d => d.Event().TestProject().Message("late arrival").Date(now.AddHours(-1)).CreatedDate(TimeProvider.GetUtcNow().UtcDateTime)); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var finished = await GetOperationAsync(created.Id); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(1, finished.Succeeded); + Assert.NotNull(await _eventRepository.GetByIdAsync(lateEvents.Single().Id)); + } + + [Fact] + public async Task Create_IdSelection_DeletesTheIdsThroughTheSameSideEffectsAsTheSingleAction() + { + // Arrange + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject(); + d.Event().TestProject(); + d.Event().TestProject(); + }); + + // Act + var created = await CreateOperationAsync(new NewBulkOperation + { + Type = "events.delete", + OrganizationId = SampleDataService.TEST_ORG_ID, + Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = events.Take(2).Select(e => e.Id).ToList() } + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var finished = await GetOperationAsync(created.Id); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(2, finished.Succeeded); + Assert.Null(await _eventRepository.GetByIdAsync(events[0].Id)); + Assert.Null(await _eventRepository.GetByIdAsync(events[1].Id)); + Assert.NotNull(await _eventRepository.GetByIdAsync(events[2].Id)); + + var usage = await GetService().GetUsageAsync(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID); + Assert.Equal(2, usage.CurrentUsage.Deleted); + } + + [Fact] + public async Task Create_IdSelectionWithAnotherOrganizationsEvent_LeavesThatEventAndCountsItFailed() + { + // Arrange + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject(); + d.Event().FreeProject(); + }); + var own = events.Single(e => e.OrganizationId == SampleDataService.TEST_ORG_ID); + var foreign = events.Single(e => e.OrganizationId == SampleDataService.FREE_ORG_ID); + + // Act + var created = await CreateOperationAsync(new NewBulkOperation + { + Type = "events.delete", + OrganizationId = SampleDataService.TEST_ORG_ID, + Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [own.Id, foreign.Id] } + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var finished = await GetOperationAsync(created.Id); + Assert.Equal(1, finished.Succeeded); + Assert.Equal(1, finished.Failed); + Assert.Null(await _eventRepository.GetByIdAsync(own.Id)); + Assert.NotNull(await _eventRepository.GetByIdAsync(foreign.Id)); + } + + [Fact] + public async Task Create_ProjectScopedIdSelectionWithAnotherProjectsEvent_LeavesThatEvent() + { + // Arrange + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject(); + d.Event().TestProject().Project(SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID); + }); + var otherProject = events.Single(e => e.ProjectId == SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID); + + // Act + var created = await CreateOperationAsync(new NewBulkOperation + { + Type = "events.delete", + OrganizationId = SampleDataService.TEST_ORG_ID, + ProjectId = SampleDataService.TEST_PROJECT_ID, + Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = events.Select(e => e.Id).ToList() } + }); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + var finished = await GetOperationAsync(created.Id); + Assert.Equal(1, finished.Failed); + Assert.NotNull(await _eventRepository.GetByIdAsync(otherProject.Id)); + } + + [Fact] + public async Task Create_OtherOrganizationOrProject_IsNotFoundAndStartsNothing() + { + // Arrange + var freeProjectInTestOrganization = FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.FREE_PROJECT_ID, ""); + + // Act + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations") + .Content(FilterRequest(SampleDataService.FREE_ORG_ID, null, "")).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations") + .Content(freeProjectInTestOrganization).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPath("bulk-operations") + .Content(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")).StatusCodeShouldBeNotFound()); + + // Assert + Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); + } + + [Fact] + public async Task Create_WithoutAUserSession_IsRejected() + { + // Arrange + var request = FilterRequest(SampleDataService.TEST_ORG_ID, null, ""); + + // Act + await SendRequestAsync(r => r.Post().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeUnauthorized()); + await SendRequestAsync(r => r.Post().AsTestOrganizationClientUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeForbidden()); + + // Assert + Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + } + + [Fact] + public async Task Create_GlobalAdmin_CanStartOperationForAnyOrganization() + { + // Arrange + var request = FilterRequest(SampleDataService.FREE_ORG_ID, null, ""); + + // Act + var created = await SendRequestAsAsync(r => r.Post().AsGlobalAdminUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeAccepted()); + + // Assert + Assert.NotNull(created); + Assert.Equal(SampleDataService.FREE_ORG_ID, created.OrganizationId); + } + + [Fact] + public async Task Get_OperationOfAnotherOrganization_IsNotFoundForListGetAndCancel() + { + // Arrange + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + + // Act + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("bulk-operations", created.Id).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("bulk-operations", created.Id, "cancel").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "bulk-operations").StatusCodeShouldBeNotFound()); + var asAdmin = await SendRequestAsAsync(r => r.AsGlobalAdminUser().AppendPaths("bulk-operations", created.Id).StatusCodeShouldBeOk()); + + // Assert + Assert.Equal(created.Id, asAdmin?.Id); + Assert.Equal(BulkOperationStatus.Queued, (await GetOperationAsync(created.Id)).Status); + } + + [Fact] + public async Task GetByOrganization_ReturnsActiveAndRecentOperationsNewestFirst() + { + // Arrange + var first = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + TimeProvider.Advance(TimeSpan.FromSeconds(5)); + var second = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Act + var list = await SendRequestAsAsync>(r => r.AsTestOrganizationUser() + .AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "bulk-operations").StatusCodeShouldBeOk()); + + // Assert + Assert.NotNull(list); + Assert.Equal([second.Id, first.Id], list.Select(o => o.Id).ToArray()); + Assert.Equal(BulkOperationStatus.Queued, list.First().Status); + Assert.Equal(BulkOperationStatus.Completed, list.Last().Status); + } + + [Fact] + public async Task Create_WhileADestructiveOperationIsActiveForTheProject_ReturnsConflict() + { + // Arrange + var active = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Act + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations") + .Content(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "type:log")).StatusCodeShouldBeConflict()); + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations") + .Content(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")).StatusCodeShouldBeConflict()); + var otherProject = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID, "")); + var otherOrganization = await CreateOperationAsync(FilterRequest(SampleDataService.FREE_ORG_ID, SampleDataService.FREE_PROJECT_ID, ""), r => r.AsFreeOrganizationUser()); + + // Assert + Assert.NotEqual(active.Id, otherProject.Id); + Assert.Equal(SampleDataService.FREE_ORG_ID, otherOrganization.OrganizationId); + } + + [Fact] + public async Task Create_AfterTheActiveOperationFinishesOrIsCancelled_IsAllowedAgain() + { + // Arrange + var first = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var second = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Act + var cancelled = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("bulk-operations", second.Id, "cancel").StatusCodeShouldBeAccepted()); + var third = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Assert + Assert.Equal(BulkOperationStatus.Completed, (await GetOperationAsync(first.Id)).Status); + Assert.Equal(BulkOperationStatus.Cancelled, cancelled?.Status); + Assert.NotEqual(second.Id, third.Id); + } + + [Fact] + public async Task Create_WhileTheActiveOperationHasStoppedReportingProgress_IsAllowed() + { + // Arrange + await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Act + TimeProvider.Advance(BulkOperationService.StaleAfter + TimeSpan.FromMinutes(1)); + var next = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + + // Assert + Assert.Equal(BulkOperationStatus.Queued, next.Status); + } + + [Fact] + public async Task Create_InvalidRequests_AreRejectedWithoutStartingAnything() + { + // Arrange + var ids = Enumerable.Range(0, 1001).Select(_ => ObjectId.GenerateNewId().ToString()).ToList(); + var requests = new[] + { + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Type = "events.explode" }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Parameters = new Dictionary { ["x"] = "y" } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "type:(log"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "[now+1d TO now+2d]"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Ids = ["537650f3b77efe23a47914f3"] } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [] } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ["not-an-id"] } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Filter = "type:log", Ids = ["537650f3b77efe23a47914f3"] } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ids } } + }; + + // Act + foreach (var request in requests) + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeUnprocessableEntity()); + + // Assert + Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + } + + [Fact] + public async Task Create_PremiumFilterOnFreePlan_RequiresUpgrade() + { + // Arrange + var request = FilterRequest(SampleDataService.FREE_ORG_ID, null, "tag:premium-tag"); + + // Act + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeUpgradeRequired()); + + // Assert + Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); + } + + [Fact] + public async Task Cancel_QueuedOperation_CancelsImmediatelyAndTheJobDoesNothing() + { + // Arrange + var (_, events) = await CreateDataAsync(d => d.Event().TestProject()); + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + + // Act + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("bulk-operations", created.Id, "cancel").StatusCodeShouldBeAccepted()); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + Assert.Equal(BulkOperationStatus.Cancelled, response?.Status); + Assert.Equal(BulkOperationStatus.Cancelled, (await GetOperationAsync(created.Id)).Status); + Assert.NotNull(await _eventRepository.GetByIdAsync(events.Single().Id)); + } + + [Fact] + public async Task Execute_MoreMatchesThanOneBatch_ProcessesEveryBatchAndPublishesProgress() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var messages = new System.Collections.Concurrent.ConcurrentQueue(); + await GetService().SubscribeAsync(m => messages.Enqueue(m), TestCancellationToken); + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + + // Act + await RunOperationAsync(_bulkOperationService, created.Id, batchSize: 2, TestCancellationToken); + await WaitForAsync(() => messages.Any(m => m.Status == BulkOperationStatus.Completed)); + + // Assert + var finished = await GetOperationAsync(created.Id); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(5, finished.Processed); + Assert.Equal(5, finished.Succeeded); + Assert.Empty(await GetRemainingMessagesAsync(events)); + + var forOperation = messages.Where(m => m.Id == created.Id).ToList(); + Assert.Contains(forOperation, m => m.Status == BulkOperationStatus.Running && m.Processed == 2); + Assert.Contains(forOperation, m => m.Status == BulkOperationStatus.Running && m.Processed == 4); + Assert.Contains(forOperation, m => m.Status == BulkOperationStatus.Running && m.Processed == 5); + var last = forOperation.OrderBy(m => m.UpdatedUtc).ThenBy(m => m.Processed).Last(); + Assert.Equal(BulkOperationStatus.Completed, last.Status); + Assert.All(forOperation, m => Assert.Equal(SampleDataService.TEST_ORG_ID, m.OrganizationId)); + } + + [Fact] + public async Task Execute_JobRestartsMidOperation_ResumesAfterTheSavedCursorWithoutRepeatingBatches() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var recorder = new RecordingHandler(); + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + using var shutdown = new CancellationTokenSource(); + recorder.AfterBatch = () => shutdown.Cancel(); + + // Act + await RunOperationAsync(service, created.Id, batchSize: 2, shutdown.Token); + var interrupted = await service.GetAsync(created.Id); + recorder.AfterBatch = null; + await RunOperationAsync(service, created.Id, batchSize: 2, TestCancellationToken); + var finished = await service.GetAsync(created.Id); + + // Assert + Assert.NotNull(interrupted); + Assert.Equal(BulkOperationStatus.Running, interrupted.Status); + Assert.Equal(2, interrupted.Processed); + Assert.NotNull(interrupted.Cursor); + Assert.NotNull(finished); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(5, finished.Processed); + Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); + Assert.Equal([2, 2, 1], recorder.Batches.Select(b => b.Count).ToArray()); + } + + [Fact] + public async Task Execute_CancelRequestedDuringABatch_StopsBeforeTheNextBatch() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var recorder = new RecordingHandler(); + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + recorder.AfterBatch = () => service.CancelAsync(created.Id).GetAwaiter().GetResult(); + + // Act + await RunOperationAsync(service, created.Id, batchSize: 2, TestCancellationToken); + + // Assert + var finished = await service.GetAsync(created.Id); + Assert.NotNull(finished); + Assert.Equal(BulkOperationStatus.Cancelled, finished.Status); + Assert.Equal(2, finished.Processed); + Assert.NotNull(finished.CompletedUtc); + Assert.Single(recorder.Batches); + } + + [Fact] + public async Task Execute_NonDestructiveOperationsDoNotBlockEachOtherOrDestructiveOnes() + { + // Arrange + var recorder = new RecordingHandler(); + var service = CreateService(recorder, new RecordingHandler("test.destructive", true)); + + // Act + var first = await service.CreateAsync(NewRecordingOperation("test.record")); + var second = await service.CreateAsync(NewRecordingOperation("test.record")); + var destructive = await service.CreateAsync(NewRecordingOperation("test.destructive")); + var blocked = await service.CreateAsync(NewRecordingOperation("test.destructive")); + + // Assert + Assert.NotNull(first.Operation); + Assert.NotNull(second.Operation); + Assert.NotNull(destructive.Operation); + Assert.Null(blocked.Operation); + Assert.Equal(destructive.Operation.Id, blocked.Conflict?.Id); + } + + [Fact] + public async Task OnBulkOperationUpdatedAsync_SendsOnlyToTheOperationsOrganization() + { + // Arrange + var broker = GetService(); + var connectionMapping = GetService(); + var connectionManager = GetService(); + var ownSocket = new TestWebSocket(); + var otherSocket = new TestWebSocket(); + string ownConnection = connectionManager.AddWebSocket(ownSocket); + string otherConnection = connectionManager.AddWebSocket(otherSocket); + await connectionMapping.GroupAddAsync(SampleDataService.TEST_ORG_ID, ownConnection); + await connectionMapping.GroupAddAsync(SampleDataService.FREE_ORG_ID, otherConnection); + + try + { + // Act + await broker.OnBulkOperationUpdatedAsync(new BulkOperationUpdated + { + Id = "537650f3b77efe23a47914f9", + OrganizationId = SampleDataService.TEST_ORG_ID, + Type = "events.delete", + Description = "Delete events matching filter", + Status = BulkOperationStatus.Running, + Total = 10, + Processed = 4 + }, TestCancellationToken); + + await WaitForAsync(() => ownSocket.SentMessages.Count > 0); + + // Assert + string sent = Assert.Single(ownSocket.SentMessages); + Assert.Contains("\"type\":\"BulkOperationUpdated\"", sent); + Assert.Contains("\"processed\":4", sent); + Assert.Empty(otherSocket.SentMessages); + } + finally + { + await connectionMapping.GroupRemoveAsync(SampleDataService.TEST_ORG_ID, ownConnection); + await connectionMapping.GroupRemoveAsync(SampleDataService.FREE_ORG_ID, otherConnection); + await connectionManager.RemoveWebSocketAsync(ownConnection); + await connectionManager.RemoveWebSocketAsync(otherConnection); + } + } + + private static NewBulkOperation FilterRequest(string organizationId, string? projectId, string filter, string time = "[now-7d TO now]") + { + return new NewBulkOperation + { + Type = "events.delete", + OrganizationId = organizationId, + ProjectId = projectId, + Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Filter = filter, Time = time } + }; + } + + private async Task CreateOperationAsync(NewBulkOperation request, Action? authorize = null) + { + var created = await SendRequestAsAsync(r => + { + r.Post().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeAccepted(); + if (authorize is null) + r.AsTestOrganizationUser(); + else + authorize(r); + }); + + Assert.NotNull(created); + return created; + } + + private async Task GetOperationAsync(string id) + { + var operation = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("bulk-operations", id).StatusCodeShouldBeOk()); + Assert.NotNull(operation); + return operation; + } + + private async Task> GetRemainingMessagesAsync(IEnumerable events) + { + await RefreshDataAsync(); + var remaining = await _eventRepository.GetByIdsAsync(events.Select(e => e.Id).ToArray(), o => o.Cache(false)); + return remaining.Select(e => e.Message ?? String.Empty).OrderBy(m => m, StringComparer.Ordinal).ToList(); + } + + private static async Task WaitForAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(10); + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(25); + } + + private BulkOperationService CreateService(params IBulkOperationHandler[] handlers) + { + return new BulkOperationService(GetService(), GetService(), GetService(), GetService>(), + handlers, TimeProvider, GetService>()); + } + + private async Task CreateRecordingOperationAsync(BulkOperationService service) + { + var result = await service.CreateAsync(NewRecordingOperation("test.record")); + Assert.NotNull(result.Operation); + return result.Operation; + } + + private BulkOperation NewRecordingOperation(string type) + { + var now = TimeProvider.GetUtcNow().UtcDateTime; + return new BulkOperation + { + OrganizationId = SampleDataService.TEST_ORG_ID, + ProjectId = SampleDataService.TEST_PROJECT_ID, + Type = type, + Description = "Record events", + CreatedByUserId = "537650f3b77efe23a47914f8", + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Filter, + Filter = String.Empty, + UtcStart = now.AddDays(-7), + UtcEnd = now, + UtcCreatedBefore = now + } + }; + } + + private Task RunOperationAsync(BulkOperationService service, string operationId, int batchSize, CancellationToken shutdownToken) + { + var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(service, [GetService()], + new BulkOperationOptions { BatchSize = batchSize }, GetService(), TimeProvider, Log); + var context = new WorkItemContext(new BulkOperationWorkItem { OperationId = operationId }, "test-job", EmptyLock.Empty, + shutdownToken, static (_, _) => Task.CompletedTask); + + return handler.HandleItemAsync(context); + } + + private sealed class RecordingHandler(string type = "test.record", bool isDestructive = false) : IBulkOperationHandler + { + public List> Batches { get; } = []; + public Action? AfterBatch { get; set; } + + public string Type => type; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => isDestructive; + + public Task AuthorizeAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task ValidateAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public string Describe(BulkOperation operation) => "Record events"; + + public Task ExecuteBatchAsync(BulkOperation operation, IReadOnlyCollection ids, CancellationToken cancellationToken) + { + Batches.Add(ids.ToList()); + AfterBatch?.Invoke(); + return Task.FromResult(new BulkBatchResult(ids.Count, 0)); + } + } +} diff --git a/tests/Exceptionless.Tests/Extensions/RequestExtensions.cs b/tests/Exceptionless.Tests/Extensions/RequestExtensions.cs index b502952b03..9106f85074 100644 --- a/tests/Exceptionless.Tests/Extensions/RequestExtensions.cs +++ b/tests/Exceptionless.Tests/Extensions/RequestExtensions.cs @@ -55,6 +55,11 @@ public static AppSendBuilder StatusCodeShouldBeForbidden(this AppSendBuilder bui return builder.ExpectedStatus(HttpStatusCode.Forbidden); } + public static AppSendBuilder StatusCodeShouldBeConflict(this AppSendBuilder builder) + { + return builder.ExpectedStatus(HttpStatusCode.Conflict); + } + public static AppSendBuilder StatusCodeShouldBeUpgradeRequired(this AppSendBuilder builder) { return builder.ExpectedStatus(HttpStatusCode.UpgradeRequired); From fe83fc0d0732b47036dcf91484f35132bb43f559 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 18:33:37 -0600 Subject: [PATCH 02/42] Count selectable rows in the bulk delete e2e test The table body also contains a hidden row, so count the row checkboxes instead. Co-Authored-By: Claude Opus 5.5 --- .../ClientApp/e2e/tests/bulk-delete-matching.e2e.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts index 553e0e95c8..3bb5f7926b 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts @@ -26,7 +26,7 @@ test('deleting everything matching the filter runs in the background and shows p await page.goto(`/event?project=${e2eScenario.projectId}&time=all&limit=${PAGE_SIZE}`); const table = page.locator('table:has(thead [role="checkbox"])').first(); - await expect(table.locator('tbody tr')).toHaveCount(PAGE_SIZE, { timeout: 30_000 }); + await expect(table.getByRole('checkbox', { name: 'Select row' })).toHaveCount(PAGE_SIZE, { timeout: 30_000 }); await test.step('selecting the page offers to select everything matching the filter', async () => { // Act From 74cbb576eeadc4414bd173ec47fa32c78da6a806 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 19:00:10 -0600 Subject: [PATCH 03/42] Harden bulk operations: paging cost, active index, pinned stack filter, retries - Paging no longer counts total hits; the total is counted once at creation. - Active operations get their own index so the one-destructive-operation guard sees every one of them; the list shows the newest 50. - The stack level filter is resolved once at creation and pinned, so selection stays fixed and pages stay cheap. A filter that matches too many stacks is rejected with a 422 and the real message. - The work item carries organization, type and description so a lost record still produces a final Failed update. - Temporary errors are retried from the saved cursor; permanent errors and exhausted retries fail the operation. - Panel: Cancel names its operation and progress is no longer re-announced. - Guard the select-all count against a filter change during the refetch. - Split the e2e steps so assertions stay out of the Act sections. Co-Authored-By: Claude Opus 5.5 --- .../BulkOperationWorkItemHandler.cs | 72 +++- .../Models/BulkOperations/BulkOperation.cs | 3 + .../Models/WorkItems/BulkOperationWorkItem.cs | 9 + .../BulkOperations/BulkOperationService.cs | 192 +++++++--- .../EventBulkSelectionResolver.cs | 80 ++++- .../BulkOperations/IBulkOperationHandler.cs | 17 +- .../Api/Handlers/BulkOperationHandler.cs | 12 +- .../e2e/tests/bulk-delete-matching.e2e.ts | 23 +- .../components/bulk-operation-item.svelte | 19 +- .../bulk-operation-item.svelte.test.ts | 40 ++- .../bulk-operation-panel.svelte.test.ts | 8 +- .../all-matching-selection.svelte.test.ts | 15 + .../table/all-matching-selection.svelte.ts | 18 +- .../src/routes/(app)/event/+page.svelte | 6 +- .../BulkOperationEndpointTests.Reliability.cs | 328 ++++++++++++++++++ .../Endpoints/BulkOperationEndpointTests.cs | 26 +- 16 files changed, 759 insertions(+), 109 deletions(-) create mode 100644 tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs index 471466481c..86d48f84de 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -1,5 +1,7 @@ +using System.Text.Json; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories.Base; using Exceptionless.Core.Services.BulkOperations; using Foundatio.Jobs; using Foundatio.Lock; @@ -13,6 +15,12 @@ namespace Exceptionless.Core.Jobs.WorkItemHandlers; /// public class BulkOperationWorkItemHandler : WorkItemHandlerBase { + /// + /// How many consecutive batches may fail before the operation fails for good. Matches the queue, which delivers a + /// work item three times in total. + /// + public const int MaxTransientFailures = 3; + private readonly BulkOperationService _service; private readonly IReadOnlyDictionary _resolvers; private readonly BulkOperationOptions _options; @@ -43,7 +51,9 @@ public override async Task HandleItemAsync(WorkItemContext context) var operation = await _service.GetAsync(workItem.OperationId); if (operation is null) { + // The record expired or the cache was cleared. Say so, otherwise the progress panel waits forever. Log.LogWarning("Bulk operation {OperationId} no longer exists", workItem.OperationId); + await _service.PublishLostAsync(workItem); return; } @@ -67,6 +77,17 @@ public override async Task HandleItemAsync(WorkItemContext context) return; } + string? pinnedSelection = null; + if (operation.Selection.Kind is BulkOperationSelectionKind.Filter) + { + pinnedSelection = await _service.GetPinnedSelectionAsync(operation.Id); + if (pinnedSelection is null) + { + await FinishAsync(operation.Id, BulkOperationStatus.Failed, "The saved selection expired, so the operation could not continue."); + return; + } + } + operation = await _service.UpdateAsync(operation.Id, current => { if (!current.IsActive) @@ -100,10 +121,13 @@ public override async Task HandleItemAsync(WorkItemContext context) return; } - var page = await GetPageAsync(operation, resolver, context.CancellationToken); + var page = await GetPageAsync(operation, resolver, pinnedSelection, context.CancellationToken); if (page.Ids.Count == 0) break; + // The user who started the operation is authorized once, when it is created. Each batch re-checks the scope + // (organization and project) but not the user: someone removed from the organization mid-run still finishes + // the deletion they legitimately started, and any member of the organization can cancel it. var result = await handler.ExecuteBatchAsync(operation, page.Ids, context.CancellationToken); operation = await _service.UpdateAsync(operation.Id, current => @@ -115,6 +139,7 @@ public override async Task HandleItemAsync(WorkItemContext context) current.Succeeded += result.Succeeded; current.Failed += result.Failed; current.Cursor = page.NextCursor; + current.RetryCount = 0; current.Total = Math.Max(current.Total, current.Processed); current.Message = result.Message; return true; @@ -143,6 +168,32 @@ await _service.UpdateAsync(operation.Id, current => { context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {operation?.Id} for shutdown"); } + catch (Exception ex) when (!IsPermanent(ex)) + { + // Elasticsearch, cache and network hiccups: let the queue deliver the work item again. The saved counts and + // cursor mean the retry continues where this run stopped. After MaxTransientFailures in a row, give up. + var failed = await _service.UpdateAsync(workItem.OperationId, current => + { + if (!current.IsActive) + return false; + + current.RetryCount++; + current.Message = current.RetryCount >= MaxTransientFailures ? "The operation stopped because of repeated errors." : "Retrying after a temporary error."; + if (current.RetryCount >= MaxTransientFailures) + _service.Finish(current, BulkOperationStatus.Failed, current.Message); + + return true; + }); + + if (failed is { IsActive: false }) + { + Log.LogError(ex, "Bulk operation {OperationId} failed after repeated errors: {Message}", workItem.OperationId, ex.Message); + return; + } + + Log.LogWarning(ex, "Bulk operation {OperationId} hit a temporary error and will be retried: {Message}", workItem.OperationId, ex.Message); + throw; + } catch (Exception ex) { Log.LogError(ex, "Bulk operation {OperationId} failed: {Message}", workItem.OperationId, ex.Message); @@ -150,11 +201,26 @@ await _service.UpdateAsync(operation.Id, current => } } - private async Task GetPageAsync(BulkOperation operation, IBulkSelectionResolver resolver, CancellationToken cancellationToken) + /// + /// Errors that retrying cannot fix. Everything else (Elasticsearch, cache and network failures surface as many different + /// exception types) is treated as temporary and bounded by . + /// + internal static bool IsPermanent(Exception exception) + { + for (var current = exception; current is not null; current = current.InnerException) + { + if (current is DocumentLimitExceededException or ArgumentException or InvalidOperationException or NotSupportedException or FormatException or JsonException or InvalidCastException or NullReferenceException or KeyNotFoundException) + return true; + } + + return false; + } + + private async Task GetPageAsync(BulkOperation operation, IBulkSelectionResolver resolver, string? pinnedSelection, CancellationToken cancellationToken) { int size = Math.Max(1, _options.BatchSize); if (operation.Selection.Kind is not BulkOperationSelectionKind.Ids) - return await resolver.GetPageAsync(operation, operation.Cursor, size, cancellationToken); + return await resolver.GetPageAsync(operation, pinnedSelection, operation.Cursor, size, cancellationToken); // The cursor of an id selection is the index of the next id to process. var ids = operation.Selection.Ids ?? []; diff --git a/src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs b/src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs index 819f967040..e605fb7076 100644 --- a/src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs +++ b/src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs @@ -34,6 +34,9 @@ public class BulkOperation : IIdentity, IOwnedByOrganization public string? Message { get; set; } + /// Consecutive batches that failed with a temporary error. Reset by the next successful batch. + public int RetryCount { get; set; } + /// Opaque position of the last batch that finished; the job resumes after it. public string? Cursor { get; set; } diff --git a/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs index d4702f6c5c..9b13adc425 100644 --- a/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs +++ b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs @@ -3,4 +3,13 @@ namespace Exceptionless.Core.Models.WorkItems; public record BulkOperationWorkItem { public required string OperationId { get; init; } + + /// + /// The organization, type and description are carried so a final state can still be published if the operation + /// record is gone when the job runs. + /// + public required string OrganizationId { get; init; } + + public string? Type { get; init; } + public string? Description { get; init; } } diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index 2584cbcb32..9d8505af09 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -28,23 +28,29 @@ public class BulkOperationService /// An active operation that has not reported progress for this long is considered abandoned. public static readonly TimeSpan StaleAfter = TimeSpan.FromHours(2); - private const int MaxIndexedOperations = 500; + /// How many operations the list shows, newest first. + public const int ListLimit = 50; + + private const int IndexPageSize = 500; + private const int MaxIndexPages = 100; private readonly ICacheClient _cache; private readonly ILockProvider _lockProvider; private readonly IMessagePublisher _publisher; private readonly IQueue _workItemQueue; private readonly IReadOnlyDictionary _handlers; + private readonly IReadOnlyDictionary _resolvers; private readonly TimeProvider _timeProvider; private readonly ILogger _logger; public BulkOperationService(ICacheClient cache, ILockProvider lockProvider, IMessagePublisher publisher, IQueue workItemQueue, - IEnumerable handlers, TimeProvider timeProvider, ILogger logger) + IEnumerable handlers, IEnumerable resolvers, TimeProvider timeProvider, ILogger logger) { _cache = cache; _lockProvider = lockProvider; _publisher = publisher; _workItemQueue = workItemQueue; _handlers = handlers.ToDictionary(h => h.Type, StringComparer.Ordinal); + _resolvers = resolvers.ToDictionary(r => r.Entity); _timeProvider = timeProvider; _logger = logger; } @@ -60,24 +66,21 @@ public BulkOperationService(ICacheClient cache, ILockProvider lockProvider, IMes return cached.HasValue ? cached.Value : null; } - /// All operations of the organization that are still stored, newest first. - public async Task> GetByOrganizationAsync(string organizationId) + /// + /// The newest operations of the organization that are still stored, newest first. Every indexed id is read so a busy + /// organization cannot push recent operations out of the list; only the history that is shown is limited. + /// + public async Task> GetByOrganizationAsync(string organizationId, int limit = ListLimit) { - var ids = new List(); - for (int page = 1; page <= MaxIndexedOperations / 100; page++) - { - var batch = await _cache.GetListAsync(GetIndexKey(organizationId), page, 100); - if (!batch.HasValue || batch.Value.Count == 0) - break; - - ids.AddRange(batch.Value); - if (batch.Value.Count < 100) - break; - } + // Ids are ObjectIds, so descending order is newest first no matter how the cache orders its list. + var newestIds = (await GetIndexedIdsAsync(GetIndexKey(organizationId))) + .Distinct(StringComparer.Ordinal) + .OrderByDescending(id => id, StringComparer.Ordinal) + .Take(limit); var operations = new List(); var expired = new List(); - foreach (string id in ids.Distinct(StringComparer.Ordinal)) + foreach (string id in newestIds) { var operation = await GetAsync(id); if (operation is null) @@ -89,7 +92,13 @@ public async Task> GetByOrganizationAsync(string or if (expired.Count > 0) await _cache.ListRemoveAsync(GetIndexKey(organizationId), expired); - return operations.OrderByDescending(o => o.CreatedUtc).ToList(); + return operations.OrderByDescending(o => o.CreatedUtc).ThenByDescending(o => o.Id, StringComparer.Ordinal).ToList(); + } + + public async Task GetPinnedSelectionAsync(string id) + { + var cached = await _cache.GetAsync(GetPinnedKey(id)); + return cached.HasValue ? cached.Value : null; } /// @@ -100,6 +109,33 @@ public async Task CreateAsync(BulkOperation operation { var handler = GetHandler(operation.Type) ?? throw new ArgumentException($"Unknown bulk operation type '{operation.Type}'.", nameof(operation)); + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + operation.Id = ObjectId.GenerateNewId(utcNow).ToString(); + operation.Status = BulkOperationStatus.Queued; + operation.CreatedUtc = utcNow; + operation.UpdatedUtc = utcNow; + + // Count and pin the selection before taking the lock, so a slow search never holds up other requests. + string? pinnedSelection = null; + if (operation.Selection.Kind is BulkOperationSelectionKind.Filter) + { + if (!_resolvers.TryGetValue(handler.Entity, out var resolver)) + throw new InvalidOperationException($"No selection resolver is registered for {handler.Entity}."); + + var preparation = await resolver.PrepareAsync(operation, CancellationToken.None); + if (!preparation.Check.IsOk) + return BulkOperationCreateResult.Rejected(preparation.Check); + + operation.Total = preparation.Total; + pinnedSelection = preparation.PinnedState; + } + else + { + operation.Total = operation.Selection.Ids?.Count ?? 0; + } + + operation.Description = handler.Describe(operation); + await using var createLock = await _lockProvider.TryAcquireAsync($"bulk-operations:create:{operation.OrganizationId}", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); if (createLock is null) return BulkOperationCreateResult.Busy(); @@ -111,15 +147,19 @@ public async Task CreateAsync(BulkOperation operation return BulkOperationCreateResult.Conflicting(conflict); } - var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - operation.Id = ObjectId.GenerateNewId(utcNow).ToString(); - operation.Status = BulkOperationStatus.Queued; - operation.CreatedUtc = utcNow; - operation.UpdatedUtc = utcNow; - await SaveAsync(operation); + if (pinnedSelection is not null) + await _cache.SetAsync(GetPinnedKey(operation.Id), pinnedSelection, OperationLifetime); + await _cache.ListAddAsync(GetIndexKey(operation.OrganizationId), [operation.Id], OperationLifetime); - await _workItemQueue.EnqueueAsync(new BulkOperationWorkItem { OperationId = operation.Id }); + await _cache.ListAddAsync(GetActiveIndexKey(operation.OrganizationId), [operation.Id], OperationLifetime); + await _workItemQueue.EnqueueAsync(new BulkOperationWorkItem + { + Description = operation.Description, + OperationId = operation.Id, + OrganizationId = operation.OrganizationId, + Type = operation.Type + }); await PublishAsync(operation); return BulkOperationCreateResult.Created(operation); @@ -133,7 +173,7 @@ public async Task CreateAsync(BulkOperation operation { await using var updateLock = await _lockProvider.TryAcquireAsync($"bulk-operations:update:{id}", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); if (updateLock is null) - throw new InvalidOperationException($"Unable to lock bulk operation {id} for update."); + throw new TimeoutException($"Unable to lock bulk operation {id} for update."); var operation = await GetAsync(id); if (operation is null) @@ -144,6 +184,9 @@ public async Task CreateAsync(BulkOperation operation operation.UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime; await SaveAsync(operation); + if (!operation.IsActive) + await _cache.ListRemoveAsync(GetActiveIndexKey(operation.OrganizationId), [operation.Id]); + await PublishAsync(operation); return operation; } @@ -189,45 +232,96 @@ public void Finish(BulkOperation operation, BulkOperationStatus status, string? private async Task FindConflictingOperationAsync(BulkOperation candidate) { + // Only active operations can conflict, and every one of them is in the active index no matter how much history + // the organization has. var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - foreach (var existing in await GetByOrganizationAsync(candidate.OrganizationId)) + var finished = new List(); + BulkOperation? conflict = null; + foreach (string id in (await GetIndexedIdsAsync(GetActiveIndexKey(candidate.OrganizationId))).Distinct(StringComparer.Ordinal)) { - if (!existing.IsActive || utcNow - existing.UpdatedUtc >= StaleAfter) + var existing = await GetAsync(id); + if (existing is null || !existing.IsActive) + { + finished.Add(id); continue; + } - if (GetHandler(existing.Type) is not { IsDestructive: true }) + if (utcNow - existing.UpdatedUtc >= StaleAfter || GetHandler(existing.Type) is not { IsDestructive: true }) continue; // An organization wide operation can touch every project, so it conflicts with any project's operation. bool overlaps = existing.ProjectId is null || candidate.ProjectId is null || String.Equals(existing.ProjectId, candidate.ProjectId, StringComparison.Ordinal); if (overlaps) - return existing; + conflict ??= existing; } - return null; + if (finished.Count > 0) + await _cache.ListRemoveAsync(GetActiveIndexKey(candidate.OrganizationId), finished); + + return conflict; + } + + private async Task> GetIndexedIdsAsync(string key) + { + var ids = new List(); + for (int page = 1; page <= MaxIndexPages; page++) + { + var batch = await _cache.GetListAsync(key, page, IndexPageSize); + if (!batch.HasValue || batch.Value.Count == 0) + break; + + ids.AddRange(batch.Value); + if (batch.Value.Count < IndexPageSize) + break; + } + + return ids; } private Task SaveAsync(BulkOperation operation) => _cache.SetAsync(GetOperationKey(operation.Id), operation, OperationLifetime); - private async Task PublishAsync(BulkOperation operation) + /// + /// Tells clients that an operation can no longer run because its record is gone (for example the cache was cleared). + /// The work item carries enough to publish a final state so the progress panel does not wait forever. + /// + public Task PublishLostAsync(BulkOperationWorkItem workItem) + { + return PublishAsync(new BulkOperationUpdated + { + Description = workItem.Description ?? "Bulk operation", + Id = workItem.OperationId, + Message = "The operation's progress was lost and it could not continue.", + OrganizationId = workItem.OrganizationId, + Status = BulkOperationStatus.Failed, + Type = workItem.Type ?? String.Empty, + UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime + }); + } + + private Task PublishAsync(BulkOperation operation) + { + return PublishAsync(new BulkOperationUpdated + { + Id = operation.Id, + OrganizationId = operation.OrganizationId, + ProjectId = operation.ProjectId, + Type = operation.Type, + Description = operation.Description, + Status = operation.Status, + Total = operation.Total, + Processed = operation.Processed, + Succeeded = operation.Succeeded, + Failed = operation.Failed, + Message = operation.Message, + UpdatedUtc = operation.UpdatedUtc + }); + } + + private async Task PublishAsync(BulkOperationUpdated message) { try { - await _publisher.PublishAsync(new BulkOperationUpdated - { - Id = operation.Id, - OrganizationId = operation.OrganizationId, - ProjectId = operation.ProjectId, - Type = operation.Type, - Description = operation.Description, - Status = operation.Status, - Total = operation.Total, - Processed = operation.Processed, - Succeeded = operation.Succeeded, - Failed = operation.Failed, - Message = operation.Message, - UpdatedUtc = operation.UpdatedUtc - }); + await _publisher.PublishAsync(message); } catch (OperationCanceledException) { @@ -235,13 +329,15 @@ await _publisher.PublishAsync(new BulkOperationUpdated } catch (Exception ex) { - _logger.LogWarning(ex, "Failed to publish bulk operation update for {OperationId}: {Message}", operation.Id, ex.Message); + _logger.LogWarning(ex, "Failed to publish bulk operation update for {OperationId}: {Message}", message.Id, ex.Message); } } private static string GetOperationKey(string id) => $"bulk-operations:{id}"; private static string GetCancelKey(string id) => $"bulk-operations:{id}:cancel"; private static string GetIndexKey(string organizationId) => $"bulk-operations:organization:{organizationId}"; + private static string GetActiveIndexKey(string organizationId) => $"bulk-operations:organization:{organizationId}:active"; + private static string GetPinnedKey(string id) => $"bulk-operations:{id}:selection"; } public record BulkOperationCreateResult @@ -249,8 +345,10 @@ public record BulkOperationCreateResult public BulkOperation? Operation { get; init; } public BulkOperation? Conflict { get; init; } public bool IsBusy { get; init; } + public BulkOperationCheckResult? Rejection { get; init; } public static BulkOperationCreateResult Created(BulkOperation operation) => new() { Operation = operation }; public static BulkOperationCreateResult Conflicting(BulkOperation conflict) => new() { Conflict = conflict }; public static BulkOperationCreateResult Busy() => new() { IsBusy = true }; + public static BulkOperationCreateResult Rejected(BulkOperationCheckResult rejection) => new() { Rejection = rejection }; } diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs index 7d7392acd7..ab914a217a 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -3,10 +3,14 @@ using Exceptionless.Core.Queries.Validation; using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Repositories.Options; using Exceptionless.Core.Repositories.Queries; using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Extensions; +using Exceptionless.Core.Repositories.Base; using Foundatio.Repositories.Extensions; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; using Foundatio.Serializer; namespace Exceptionless.Core.Services.BulkOperations; @@ -48,36 +52,68 @@ public async Task ValidateSelectionAsync(Organization return BulkOperationCheckResult.Ok; } - public async Task CountAsync(BulkOperation operation, CancellationToken cancellationToken) + public async Task PrepareAsync(BulkOperation operation, CancellationToken cancellationToken) { - var scope = await GetScopeAsync(operation); - if (scope is null) - return 0; + var appFilter = await GetAppFilterAsync(operation); + if (appFilter is null) + return new BulkSelectionPreparation(BulkOperationCheckResult.Ok); + + // The events list resolves stack level terms (for example status:open) and removes deleted stacks by looking up + // matching stack ids. Do that once, here, and keep the result so every page uses the same stacks. + var query = ApplyScope(new RepositoryQuery(), appFilter, operation.Selection) + .FilterExpression(operation.Selection.Filter) + .EnforceEventStackFilter(); + + CountResult count; + try + { + count = await _eventRepository.CountAsync(query); + } + catch (DocumentLimitExceededException ex) + { + return BulkSelectionPreparation.Rejected(BulkOperationCheckResult.Invalid("selection.filter", ex.Message)); + } + + var pinned = new PinnedEventSelection + { + EventFilter = query.GetFilterExpression(), + ExcludedStackIds = query.GetExcludedStacks().ToArray(), + StackIds = query.GetStacks().ToArray() + }; - var result = await _eventRepository.CountAsync(q => BuildQuery(q, scope.Value.AppFilter, operation.Selection)); - return result.Total; + return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, count.Total, _serializer.SerializeToString(pinned)); } - public async Task GetPageAsync(BulkOperation operation, string? cursor, int size, CancellationToken cancellationToken) + public async Task GetPageAsync(BulkOperation operation, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) { - var scope = await GetScopeAsync(operation); - if (scope is null) + var appFilter = await GetAppFilterAsync(operation); + if (appFilter is null || pinnedState is null) return new BulkSelectionPage([], null); - var results = await _eventRepository.FindAsync( - q => BuildQuery(q, scope.Value.AppFilter, operation.Selection).Include(e => e.Id).SortExpression($"-{EventIndex.Alias.Date}"), - o => o.SearchAfterToken(cursor, _serializer).PageLimit(size)); + var pinned = _serializer.Deserialize(pinnedState) ?? new PinnedEventSelection(); + var query = ApplyScope(new RepositoryQuery(), appFilter, operation.Selection) + .FilterExpression(pinned.EventFilter) + .Include(e => e.Id) + .SortExpression($"-{EventIndex.Alias.Date}"); + + if (pinned.StackIds.Length > 0) + query.Stack(pinned.StackIds); + + if (pinned.ExcludedStackIds.Length > 0) + query.ExcludeStack(pinned.ExcludedStackIds); + + // Paging only needs the next ids; the total was counted once when the operation was created. + var options = new CommandOptions().SearchAfterToken(cursor, _serializer).PageLimit(size).TrackTotalHits(false); + var results = await _eventRepository.FindAsync(query, options); string[] ids = results.Hits.Select(h => h.Id).Where(id => !String.IsNullOrEmpty(id)).Select(id => id!).ToArray(); string? nextCursor = results.Hits.LastOrDefault()?.GetSortToken(_serializer); return new BulkSelectionPage(ids, ids.Length > 0 ? nextCursor : null); } - private static T BuildQuery(T query, AppFilter appFilter, BulkOperationSelection selection) where T : class, IRepositoryQuery + private static RepositoryQuery ApplyScope(RepositoryQuery query, AppFilter appFilter, BulkOperationSelection selection) { - query.AppFilter(appFilter) - .FilterExpression(selection.Filter) - .EnforceEventStackFilter(); + query.AppFilter(appFilter); if (selection.UtcStart.HasValue || selection.UtcEnd.HasValue) { @@ -92,7 +128,7 @@ private static T BuildQuery(T query, AppFilter appFilter, BulkOperationSelect return query; } - private async Task<(AppFilter AppFilter, Organization Organization)?> GetScopeAsync(BulkOperation operation) + private async Task GetAppFilterAsync(BulkOperation operation) { var organization = await _organizationRepository.GetByIdAsync(operation.OrganizationId, o => o.Cache()); if (organization is null) @@ -114,6 +150,14 @@ private static T BuildQuery(T query, AppFilter appFilter, BulkOperationSelect } appFilter.UsesPremiumFeatures = validation.UsesPremiumFeatures; - return (appFilter, organization); + return appFilter; + } + + /// What the stack level filter resolved to when the operation was created. + private sealed class PinnedEventSelection + { + public string? EventFilter { get; set; } + public string[] ExcludedStackIds { get; set; } = []; + public string[] StackIds { get; set; } = []; } } diff --git a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs index 7d1d31e459..fafdb340a4 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs @@ -42,14 +42,23 @@ public interface IBulkSelectionResolver /// Checks that the filter of a selection can be run for the organization (syntax, plan limits). Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection); - /// Counts everything that currently matches the selection. - Task CountAsync(BulkOperation operation, CancellationToken cancellationToken); + /// + /// Runs once when the operation is created. Counts what matches and pins every part of the selection that could + /// otherwise change while the operation runs (for events, the stack level filter), so the selection stays exactly + /// what the user confirmed and pages stay cheap. The pinned state is stored with the operation for its lifetime. + /// + Task PrepareAsync(BulkOperation operation, CancellationToken cancellationToken); /// /// Returns the next page of ids after . The cursor is a value based position, so - /// paging stays correct while earlier items are being deleted or changed. + /// paging stays correct while earlier items are being deleted or changed. Paging must not count totals. /// - Task GetPageAsync(BulkOperation operation, string? cursor, int size, CancellationToken cancellationToken); + Task GetPageAsync(BulkOperation operation, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken); +} + +public record BulkSelectionPreparation(BulkOperationCheckResult Check, long Total = 0, string? PinnedState = null) +{ + public static BulkSelectionPreparation Rejected(BulkOperationCheckResult check) => new(check); } public record BulkSelectionPage(IReadOnlyList Ids, string? NextCursor); diff --git a/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs b/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs index 5149567c9d..01a5294cad 100644 --- a/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs @@ -94,17 +94,13 @@ public async Task>> Handle(CreateBulk var selectionCheck = await resolver.ValidateSelectionAsync(organization, selection); if (!selectionCheck.IsOk) return ToResult>(selectionCheck); - - operation.Total = await resolver.CountAsync(operation, HttpContext.RequestAborted); - } - else - { - operation.Total = selection.Ids!.Count; } - operation.Description = handler.Describe(operation); - + // Counting and pinning the selection (and describing the operation) happens while it is created. var created = await bulkOperationService.CreateAsync(operation); + if (created.Rejection is not null) + return ToResult>(created.Rejection); + if (created.Conflict is not null) return Result.Conflict("Another destructive operation is already running for this project. Wait for it to finish or cancel it first."); diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts index 3bb5f7926b..ea074f91e0 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts @@ -37,10 +37,17 @@ test('deleting everything matching the filter runs in the background and shows p await expect(page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` })).toBeVisible(); }); - await test.step('selecting everything matching the filter can be undone', async () => { + await test.step('selecting everything matching the filter shows the total', async () => { // Act await page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` }).click(); + + // Assert await expect(page.getByRole('status').filter({ hasText: `All ${EVENT_COUNT} events are selected` })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Clear selection' })).toBeVisible(); + }); + + await test.step('clearing the selection deselects the page', async () => { + // Act await page.getByRole('button', { name: 'Clear selection' }).click(); // Assert @@ -48,7 +55,7 @@ test('deleting everything matching the filter runs in the background and shows p await expect(table.getByRole('checkbox', { name: 'Select all rows' })).not.toBeChecked(); }); - await test.step('deleting against the filter confirms the exact count', async () => { + await test.step('deleting against the filter asks to confirm the exact count', async () => { // Arrange await table.getByRole('checkbox', { name: 'Select all rows' }).click(); await page.getByRole('button', { name: `Select all ${EVENT_COUNT} events matching this filter` }).click(); @@ -58,14 +65,16 @@ test('deleting everything matching the filter runs in the background and shows p await page.getByRole('menuitem', { name: 'Delete' }).click(); // Assert - const dialog = page.getByRole('alertdialog'); - await expect(dialog).toContainText(`Delete ${EVENT_COUNT} events matching this filter? This can't be undone.`); + await expect(page.getByRole('alertdialog')).toContainText(`Delete ${EVENT_COUNT} events matching this filter? This can't be undone.`); + }); + await test.step('confirming runs in the background and the progress panel reaches completion', async () => { // Act - await dialog.getByRole('button', { name: `Delete ${EVENT_COUNT} Events` }).click(); - }); + await page + .getByRole('alertdialog') + .getByRole('button', { name: `Delete ${EVENT_COUNT} Events` }) + .click(); - await test.step('the progress panel follows the operation to completion', async () => { // Assert const panel = page.getByRole('region', { name: 'Bulk operations' }); await expect(panel).toContainText(/Delete \d+ events matching filter/, { timeout: 30_000 }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte index 330285106b..972c8be697 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte @@ -18,10 +18,20 @@ operation: ViewBulkOperation; } + const STATUS_LABELS: Record = { + [BulkOperationStatus.Cancelled]: 'cancelled', + [BulkOperationStatus.Completed]: 'completed', + [BulkOperationStatus.Failed]: 'failed', + [BulkOperationStatus.Queued]: 'waiting to start', + [BulkOperationStatus.Running]: 'in progress' + }; + let { isCancelling = false, onCancel, onDismiss, operation }: Props = $props(); const isActive = $derived(isBulkOperationActive(operation.status)); const percent = $derived(getProgressPercent(operation)); + // The visible counts change on every update. Screen readers only hear the operation's state, which changes rarely. + const announcement = $derived(`${operation.description}: ${STATUS_LABELS[operation.status]}`);
@@ -38,8 +48,9 @@ {/if}
+ {announcement}

{operation.description}

-

+

{#if operation.status === BulkOperationStatus.Queued} Waiting to start {:else if operation.status === BulkOperationStatus.Running} @@ -56,9 +67,11 @@

{#if isActive} - + {:else} - {/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts index 64d3a3996b..239d39277b 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts @@ -24,7 +24,7 @@ function createOperation(overrides: Partial = {}): ViewBulkOp } function statusText(): string { - return (screen.getByRole('status').textContent ?? '').replace(/\s+/g, ' ').trim(); + return (document.querySelector('[data-slot="bulk-operation-progress"]')?.textContent ?? '').replace(/\s+/g, ' ').trim(); } describe('BulkOperationItem', () => { @@ -37,14 +37,40 @@ describe('BulkOperationItem', () => { render(BulkOperationItem, { props: { onCancel, onDismiss: vi.fn(), operation } }); // Act - await fireEvent.click(screen.getByRole('button', { name: 'Cancel' })); + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); // Assert expect(screen.getByText('Delete 1,234 events matching filter')).toBeTruthy(); expect(statusText()).toBe('500 of 1,234'); expect(screen.getByRole('progressbar').getAttribute('aria-valuenow')).toBe('41'); expect(onCancel).toHaveBeenCalledWith(operation); - expect(screen.queryByRole('button', { name: 'Dismiss' })).toBeNull(); + expect(screen.queryByRole('button', { name: /^Dismiss / })).toBeNull(); + }); + + it('names the operation on its buttons and announces only state changes, not every progress update', async () => { + // Arrange + const { rerender } = render(BulkOperationItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation({ processed: 100 }) } }); + const announcement = () => screen.getByRole('status').textContent; + const initial = announcement(); + + // Act + await rerender({ operation: createOperation({ processed: 900 }) }); + const afterProgress = announcement(); + await rerender({ operation: createOperation({ processed: 1234, status: BulkOperationStatus.Completed }) }); + + // Assert + expect(initial).toBe('Delete 1,234 events matching filter: in progress'); + expect(afterProgress).toBe(initial); + expect(announcement()).toBe('Delete 1,234 events matching filter: completed'); + expect(screen.getByRole('button', { name: 'Dismiss Delete 1,234 events matching filter' })).toBeTruthy(); + }); + + it('labels Cancel with the operation it cancels', () => { + // Arrange & Act + render(BulkOperationItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation() } }); + + // Assert + expect(screen.getByRole('button', { name: 'Cancel Delete 1,234 events matching filter' })).toBeTruthy(); }); it('shows the queued state without a count', () => { @@ -55,7 +81,7 @@ describe('BulkOperationItem', () => { // Assert expect(statusText()).toBe('Waiting to start'); - expect(screen.getByRole('button', { name: 'Cancel' })).toBeTruthy(); + expect(screen.getByRole('button', { name: /^Cancel / })).toBeTruthy(); }); it('disables Cancel once cancellation was requested', () => { @@ -63,7 +89,7 @@ describe('BulkOperationItem', () => { render(BulkOperationItem, { props: { isCancelling: true, onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation() } }); // Assert - expect((screen.getByRole('button', { name: 'Cancel' }) as HTMLButtonElement).disabled).toBe(true); + expect((screen.getByRole('button', { name: /^Cancel / }) as HTMLButtonElement).disabled).toBe(true); }); it('shows a completed operation with its failures and a Dismiss button', async () => { @@ -73,11 +99,11 @@ describe('BulkOperationItem', () => { render(BulkOperationItem, { props: { onCancel: vi.fn(), onDismiss, operation } }); // Act - await fireEvent.click(screen.getByRole('button', { name: 'Dismiss' })); + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); // Assert expect(statusText()).toBe('Completed · 1,232 processed, 2 failed'); - expect(screen.queryByRole('button', { name: 'Cancel' })).toBeNull(); + expect(screen.queryByRole('button', { name: /^Cancel / })).toBeNull(); expect(onDismiss).toHaveBeenCalledWith(operation); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts index 21a110e11d..b70700dee7 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts @@ -77,7 +77,7 @@ describe('BulkOperationPanel', () => { // Assert expect(screen.getAllByText('Delete 50 events matching filter')).toHaveLength(1); - expect(screen.getByRole('status').textContent?.replace(/\s+/g, ' ').trim()).toBe('20 of 50'); + expect(document.querySelector('[data-slot="bulk-operation-progress"]')?.textContent?.replace(/\s+/g, ' ').trim()).toBe('20 of 50'); expect(mocks.invalidateAffected).not.toHaveBeenCalled(); }); @@ -113,7 +113,7 @@ describe('BulkOperationPanel', () => { // Assert expect(mocks.invalidateAffected).toHaveBeenCalledTimes(1); expect(mocks.invalidateAffected).toHaveBeenCalledWith(mocks.queryClient); - expect(screen.getByRole('status').textContent).toContain('Completed'); + expect(document.querySelector('[data-slot="bulk-operation-progress"]')?.textContent).toContain('Completed'); }); it('requests cancellation of a running operation', async () => { @@ -122,7 +122,7 @@ describe('BulkOperationPanel', () => { render(BulkOperationPanel); // Act - await fireEvent.click(screen.getByRole('button', { name: 'Cancel' })); + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); // Assert await waitFor(() => expect(mocks.cancelMutateAsync).toHaveBeenCalledWith('operation-1')); @@ -134,7 +134,7 @@ describe('BulkOperationPanel', () => { render(BulkOperationPanel); // Act - await fireEvent.click(screen.getByRole('button', { name: 'Dismiss' })); + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); // Assert expect(screen.queryByText('Delete 50 events matching filter')).toBeNull(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts index d63b4f5c50..cee0a284d6 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts @@ -34,6 +34,21 @@ describe('createAllMatchingSelection', () => { expect(selection.active).toBeUndefined(); }); + it('ignores a count that was fetched for a filter that is no longer showing', () => { + // Arrange + let identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' }); + const selection = createAllMatchingSelection(() => identity); + const identityWhenCountingStarted = selection.identity; + + // Act + identity = getEventsQueryIdentity({ filter: 'type:log', organizationId: 'organization-id' }); + const selected = selection.select(1234, identityWhenCountingStarted); + + // Assert + expect(selected).toBe(false); + expect(selection.active).toBeUndefined(); + }); + it('does not come back when the original query is restored after being cleared', () => { // Arrange const original = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts index c6a31db878..cdb891d0ff 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts @@ -29,11 +29,25 @@ export function createAllMatchingSelection(getIdentity: () => string) { clear() { selection = undefined; }, - select(count: number) { + /** The query the list is showing right now. Capture it before awaiting a count, then pass it to select. */ + get identity(): string { + return getIdentity(); + }, + /** + * Selects everything matching the current query. When an expected identity is given and the query changed in + * the meantime (the count was fetched for a different filter), nothing is selected and false is returned. + */ + select(count: number, expectedIdentity?: string): boolean { + const identity = getIdentity(); + if (expectedIdentity !== undefined && expectedIdentity !== identity) { + return false; + } + selection = { count, - identity: getIdentity() + identity }; + return true; } }; } diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte index 48b91a39d3..7f97ccf559 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte @@ -879,11 +879,13 @@ } async function selectAllMatching() { - // Count again right now, so the number the user confirms is current. + // Count again right now, so the number the user confirms is current. If the filter changes while the count is + // loading, the count belongs to the old filter and must not be bound to the new one. + const identity = allMatching.identity; const result = await matchingTotalQuery.refetch(); const count = (result.data?.meta.total ?? totalMatching) as number | undefined; if (count !== undefined) { - allMatching.select(count); + allMatching.select(count, identity); } } diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs new file mode 100644 index 0000000000..e8ecee2b7e --- /dev/null +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -0,0 +1,328 @@ +using System.Collections.Concurrent; +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Base; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Web.Models; +using Foundatio.Jobs; +using Foundatio.Messaging; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; +using Foundatio.Repositories.Utility; +using Foundatio.Serializer; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Api.Endpoints; + +public partial class BulkOperationEndpointTests +{ + [Fact] + public async Task GetPageAsync_Paging_DoesNotCountTotalHits() + { + // Arrange + ICommandOptions? capturedOptions = null; + var repository = new Mock(); + repository.Setup(r => r.FindAsync(It.IsAny(), It.IsAny())) + .Callback((_, options) => capturedOptions = options) + .ReturnsAsync(new FindResults()); + var resolver = CreateResolver(repository.Object); + + // Act + await resolver.GetPageAsync(NewRecordingOperation("test.record"), "{}", null, 10, TestCancellationToken); + + // Assert + Assert.NotNull(capturedOptions); + Assert.False(capturedOptions.ShouldTrackTotalHits()); + } + + [Fact] + public async Task Create_FilterMatchingTooManyStacks_IsRejectedWithTheRealMessage() + { + // Arrange + const string message = "Event filter resulted in too many matching stacks (30000 of 20000 max). Please limit your search criteria."; + var repository = new Mock(); + repository.Setup(r => r.CountAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new DocumentLimitExceededException(message)); + var service = CreateService([GetService()], [CreateResolver(repository.Object)]); + + // Act + var result = await service.CreateAsync(NewRecordingOperation("events.delete")); + + // Assert + Assert.Null(result.Operation); + Assert.NotNull(result.Rejection); + Assert.Equal(BulkOperationCheckStatus.Invalid, result.Rejection.Status); + Assert.Equal("selection.filter", result.Rejection.Field); + Assert.Equal(message, result.Rejection.Message); + Assert.Empty(await service.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + } + + [Fact] + public async Task Execute_StackStatusChangesAfterCreation_StillDeletesTheEventsThatWereConfirmed() + { + // Arrange + var (stacks, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("open 1").Status(StackStatus.Open); + d.Event().TestProject().Message("open 2").Status(StackStatus.Open); + d.Event().TestProject().Message("fixed").Status(StackStatus.Fixed); + }); + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "status:open")); + var openStacks = stacks.Where(s => s.Status == StackStatus.Open).ToList(); + + // Act + foreach (var stack in openStacks) + stack.Status = StackStatus.Fixed; + + await GetService().SaveAsync(openStacks, o => o.ImmediateConsistency()); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Assert + string? pinned = await _bulkOperationService.GetPinnedSelectionAsync(created.Id); + Assert.Equal(2, created.Total); + Assert.NotNull(pinned); + Assert.All(openStacks, stack => Assert.Contains(stack.Id, pinned)); + Assert.Equal(["fixed"], (await GetRemainingMessagesAsync(events)).ToArray()); + } + + [Fact] + public async Task Create_MoreThanFiveHundredOperations_StillSeesTheNewestActiveOneForTheConflictCheck() + { + // Arrange + var service = CreateService(new RecordingHandler(), new RecordingHandler("test.destructive", true)); + for (int index = 0; index < 500; index++) + Assert.NotNull((await service.CreateAsync(NewIdsOperation("test.record"))).Operation); + + // Act + var newest = await service.CreateAsync(NewIdsOperation("test.destructive")); + var blocked = await service.CreateAsync(NewIdsOperation("test.destructive")); + var list = await service.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID); + + // Assert + Assert.NotNull(newest.Operation); + Assert.Null(blocked.Operation); + Assert.Equal(newest.Operation.Id, blocked.Conflict?.Id); + Assert.Equal(BulkOperationService.ListLimit, list.Count); + Assert.Equal(newest.Operation.Id, list[0].Id); + } + + [Fact] + public async Task Execute_RecordMissingWhenTheJobRuns_PublishesAFinalFailedUpdate() + { + // Arrange + var messages = new ConcurrentQueue(); + await GetService().SubscribeAsync(m => messages.Enqueue(m), TestCancellationToken); + var workItem = new BulkOperationWorkItem + { + Description = "Delete 7 events matching filter", + OperationId = ObjectId.GenerateNewId().ToString(), + OrganizationId = SampleDataService.TEST_ORG_ID, + Type = "events.delete" + }; + + // Act + await RunWorkItemAsync(_bulkOperationService, workItem, 500, TestCancellationToken); + await WaitForAsync(() => !messages.IsEmpty); + + // Assert + var update = Assert.Single(messages); + Assert.Equal(workItem.OperationId, update.Id); + Assert.Equal(SampleDataService.TEST_ORG_ID, update.OrganizationId); + Assert.Equal("Delete 7 events matching filter", update.Description); + Assert.Equal(BulkOperationStatus.Failed, update.Status); + Assert.False(String.IsNullOrEmpty(update.Message)); + } + + [Fact] + public async Task Execute_TemporaryErrorMidOperation_IsRetriedFromTheSavedCursor() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var recorder = new RecordingHandler { Failure = call => call == 2 ? new TimeoutException("Elasticsearch timed out") : null }; + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + + // Act + var thrown = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + var interrupted = await service.GetAsync(created.Id); + await RunOperationAsync(service, created.Id, 2, TestCancellationToken); + var finished = await service.GetAsync(created.Id); + + // Assert + Assert.IsType(thrown); + Assert.NotNull(interrupted); + Assert.Equal(BulkOperationStatus.Running, interrupted.Status); + Assert.Equal(2, interrupted.Processed); + Assert.Equal(1, interrupted.RetryCount); + Assert.NotNull(finished); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(0, finished.RetryCount); + Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); + } + + [Fact] + public async Task Execute_TemporaryErrorsKeepHappening_FailsAfterTheRetriesAreUsedUp() + { + // Arrange + await CreateDataAsync(d => d.Event().TestProject()); + var recorder = new RecordingHandler { Failure = _ => new ApplicationException("Elasticsearch unavailable") }; + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + + // Act + var first = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + var second = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + var third = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + var finished = await service.GetAsync(created.Id); + + // Assert + Assert.IsType(first); + Assert.IsType(second); + Assert.Null(third); + Assert.NotNull(finished); + Assert.Equal(BulkOperationStatus.Failed, finished.Status); + Assert.Equal(3, finished.RetryCount); + } + + [Fact] + public async Task Execute_ErrorThatRetryingCannotFix_FailsImmediately() + { + // Arrange + await CreateDataAsync(d => d.Event().TestProject()); + var recorder = new RecordingHandler { Failure = _ => new InvalidOperationException("Bad request") }; + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + + // Act + var thrown = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + var finished = await service.GetAsync(created.Id); + + // Assert + Assert.Null(thrown); + Assert.NotNull(finished); + Assert.Equal(BulkOperationStatus.Failed, finished.Status); + Assert.Equal(0, finished.RetryCount); + } + + [Fact] + public async Task Execute_JobRestartsMidDelete_ResumesAndDeletesEachEventExactlyOnce() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Message($"event {i}").Date(now.AddMinutes(-i - 1)); + }); + var deleteHandler = new InterceptingHandler(GetService()); + var service = CreateService(deleteHandler); + var result = await service.CreateAsync(NewRecordingOperation("events.delete")); + Assert.NotNull(result.Operation); + using var shutdown = new CancellationTokenSource(); + deleteHandler.AfterBatch = () => shutdown.Cancel(); + + // Act + await RunOperationAsync(service, result.Operation.Id, 2, shutdown.Token); + var interrupted = await service.GetAsync(result.Operation.Id); + var remainingWhileInterrupted = await GetRemainingMessagesAsync(events); + deleteHandler.AfterBatch = null; + await RunOperationAsync(service, result.Operation.Id, 2, TestCancellationToken); + var finished = await service.GetAsync(result.Operation.Id); + + // Assert + Assert.NotNull(interrupted); + Assert.Equal(BulkOperationStatus.Running, interrupted.Status); + Assert.Equal(2, interrupted.Processed); + Assert.Equal(3, remainingWhileInterrupted.Count); + Assert.NotNull(finished); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(5, finished.Processed); + Assert.Equal(5, finished.Succeeded); + Assert.Equal(0, finished.Failed); + Assert.Empty(await GetRemainingMessagesAsync(events)); + } + + [Fact] + public async Task Cancel_RunningOperationThatStoppedReportingProgress_CancelsImmediately() + { + // Arrange + var service = CreateService(new RecordingHandler()); + var created = await CreateRecordingOperationAsync(service); + await service.UpdateAsync(created.Id, operation => + { + operation.Status = BulkOperationStatus.Running; + return true; + }); + TimeProvider.Advance(BulkOperationService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var cancelled = await service.CancelAsync(created.Id); + + // Assert + Assert.NotNull(cancelled); + Assert.Equal(BulkOperationStatus.Cancelled, cancelled.Status); + Assert.NotNull(cancelled.CompletedUtc); + } + + [Fact] + public async Task Cancel_FinishedOperation_ChangesNothing() + { + // Arrange + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + + // Act + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("bulk-operations", created.Id, "cancel").StatusCodeShouldBeAccepted()); + + // Assert + Assert.Equal(BulkOperationStatus.Completed, response?.Status); + Assert.Equal(BulkOperationStatus.Completed, (await GetOperationAsync(created.Id)).Status); + Assert.False(await _bulkOperationService.IsCancelRequestedAsync(created.Id)); + } + + private EventBulkSelectionResolver CreateResolver(IEventRepository eventRepository) + { + return new EventBulkSelectionResolver(eventRepository, GetService(), GetService(), + GetService(), GetService()); + } + + private BulkOperation NewIdsOperation(string type) + { + var operation = NewRecordingOperation(type); + operation.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [SampleDataService.TEST_ORG_ID] }; + return operation; + } + + /// Runs the real handler, with a hook after each batch to simulate the host stopping. + private sealed class InterceptingHandler(IBulkOperationHandler inner) : IBulkOperationHandler + { + public Action? AfterBatch { get; set; } + + public string Type => inner.Type; + public BulkOperationEntity Entity => inner.Entity; + public bool IsDestructive => inner.IsDestructive; + + public Task AuthorizeAsync(BulkOperationRequest request) => inner.AuthorizeAsync(request); + public Task ValidateAsync(BulkOperationRequest request) => inner.ValidateAsync(request); + public string Describe(BulkOperation operation) => inner.Describe(operation); + + public async Task ExecuteBatchAsync(BulkOperation operation, IReadOnlyCollection ids, CancellationToken cancellationToken) + { + var result = await inner.ExecuteBatchAsync(operation, ids, cancellationToken); + AfterBatch?.Invoke(); + return result; + } + } +} diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index 218394b0b2..4f4c018aa7 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -25,7 +25,7 @@ namespace Exceptionless.Tests.Api.Endpoints; -public class BulkOperationEndpointTests : IntegrationTestsBase +public partial class BulkOperationEndpointTests : IntegrationTestsBase { private readonly IEventRepository _eventRepository; private readonly BulkOperationService _bulkOperationService; @@ -651,9 +651,14 @@ private static async Task WaitForAsync(Func condition) } private BulkOperationService CreateService(params IBulkOperationHandler[] handlers) + { + return CreateService(handlers, [GetService()]); + } + + private BulkOperationService CreateService(IBulkOperationHandler[] handlers, IBulkSelectionResolver[] resolvers) { return new BulkOperationService(GetService(), GetService(), GetService(), GetService>(), - handlers, TimeProvider, GetService>()); + handlers, resolvers, TimeProvider, GetService>()); } private async Task CreateRecordingOperationAsync(BulkOperationService service) @@ -685,20 +690,29 @@ private BulkOperation NewRecordingOperation(string type) } private Task RunOperationAsync(BulkOperationService service, string operationId, int batchSize, CancellationToken shutdownToken) + { + return RunWorkItemAsync(service, new BulkOperationWorkItem { OperationId = operationId, OrganizationId = SampleDataService.TEST_ORG_ID }, batchSize, shutdownToken); + } + + private Task RunWorkItemAsync(BulkOperationService service, BulkOperationWorkItem workItem, int batchSize, CancellationToken shutdownToken) { var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(service, [GetService()], new BulkOperationOptions { BatchSize = batchSize }, GetService(), TimeProvider, Log); - var context = new WorkItemContext(new BulkOperationWorkItem { OperationId = operationId }, "test-job", EmptyLock.Empty, - shutdownToken, static (_, _) => Task.CompletedTask); + var context = new WorkItemContext(workItem, "test-job", EmptyLock.Empty, shutdownToken, static (_, _) => Task.CompletedTask); return handler.HandleItemAsync(context); } private sealed class RecordingHandler(string type = "test.record", bool isDestructive = false) : IBulkOperationHandler { + private int _calls; + public List> Batches { get; } = []; public Action? AfterBatch { get; set; } + /// Called with the 1 based batch number before the batch is recorded; return an exception to make it fail. + public Func? Failure { get; set; } + public string Type => type; public BulkOperationEntity Entity => BulkOperationEntity.Events; public bool IsDestructive => isDestructive; @@ -709,6 +723,10 @@ private sealed class RecordingHandler(string type = "test.record", bool isDestru public Task ExecuteBatchAsync(BulkOperation operation, IReadOnlyCollection ids, CancellationToken cancellationToken) { + var failure = Failure?.Invoke(++_calls); + if (failure is not null) + throw failure; + Batches.Add(ids.ToList()); AfterBatch?.Invoke(); return Task.FromResult(new BulkBatchResult(ids.Count, 0)); From d8cd35e19da1900ecdc7c7ca9c54f7a072de4d31 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 20:45:14 -0600 Subject: [PATCH 04/42] Continue bulk operations in a new work item after temporary errors The queue's retries cover a whole work item, so rethrowing let an operation that made progress between errors run out of deliveries while still marked running. Co-Authored-By: Claude Opus 5.5 --- .../BulkOperationWorkItemHandler.cs | 18 ++++--- .../BulkOperations/BulkOperationService.cs | 12 +++++ .../BulkOperationEndpointTests.Reliability.cs | 48 +++++++++++++++++-- .../Endpoints/BulkOperationEndpointTests.cs | 2 +- 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs index 86d48f84de..115c64f795 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -16,8 +16,7 @@ namespace Exceptionless.Core.Jobs.WorkItemHandlers; public class BulkOperationWorkItemHandler : WorkItemHandlerBase { /// - /// How many consecutive batches may fail before the operation fails for good. Matches the queue, which delivers a - /// work item three times in total. + /// How many temporary errors in a row (without a batch completing in between) end the operation. /// public const int MaxTransientFailures = 3; @@ -170,8 +169,9 @@ await _service.UpdateAsync(operation.Id, current => } catch (Exception ex) when (!IsPermanent(ex)) { - // Elasticsearch, cache and network hiccups: let the queue deliver the work item again. The saved counts and - // cursor mean the retry continues where this run stopped. After MaxTransientFailures in a row, give up. + // Elasticsearch, cache and network hiccups: continue in a new work item from the saved counts and cursor. Rethrowing + // would use the queue's retries, which cover the whole work item, so an operation that makes progress between errors + // would run out of deliveries while still marked running. After MaxTransientFailures in a row, give up. var failed = await _service.UpdateAsync(workItem.OperationId, current => { if (!current.IsActive) @@ -191,8 +191,14 @@ await _service.UpdateAsync(operation.Id, current => return; } - Log.LogWarning(ex, "Bulk operation {OperationId} hit a temporary error and will be retried: {Message}", workItem.OperationId, ex.Message); - throw; + Log.LogWarning(ex, "Bulk operation {OperationId} hit a temporary error and will continue in a new run: {Message}", workItem.OperationId, ex.Message); + if (failed is null) + return; + + if (_options.RetryDelay > TimeSpan.Zero) + await Task.Delay(_options.RetryDelay * failed.RetryCount, _timeProvider, context.CancellationToken); + + await _service.ContinueAsync(workItem); } catch (Exception ex) { diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index 9d8505af09..818c50e93a 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -15,6 +15,9 @@ public class BulkOperationOptions { /// How many ids are resolved and executed per batch. public int BatchSize { get; set; } = 500; + + /// How long to wait before continuing after a temporary error. Multiplied by the number of errors in a row. + public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(5); } /// @@ -57,6 +60,15 @@ public BulkOperationService(ICacheClient cache, ILockProvider lockProvider, IMes public IBulkOperationHandler? GetHandler(string type) => _handlers.GetValueOrDefault(type); + /// + /// Queues another run that continues from the saved cursor. The queue's retries cover a whole work item, so a long operation + /// continues in a new work item after a temporary error instead of using up the retries of the current one. + /// + public Task ContinueAsync(BulkOperationWorkItem workItem) + { + return _workItemQueue.EnqueueAsync(workItem); + } + public async Task GetAsync(string id) { if (String.IsNullOrEmpty(id)) diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs index e8ecee2b7e..40d29086bf 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -12,6 +12,7 @@ using Exceptionless.Web.Models; using Foundatio.Jobs; using Foundatio.Messaging; +using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; @@ -154,14 +155,18 @@ public async Task Execute_TemporaryErrorMidOperation_IsRetriedFromTheSavedCursor var service = CreateService(recorder); var created = await CreateRecordingOperationAsync(service); + long enqueuedBefore = await GetEnqueuedWorkItemCountAsync(); + // Act var thrown = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); + long continuations = await GetEnqueuedWorkItemCountAsync() - enqueuedBefore; var interrupted = await service.GetAsync(created.Id); await RunOperationAsync(service, created.Id, 2, TestCancellationToken); var finished = await service.GetAsync(created.Id); // Assert - Assert.IsType(thrown); + Assert.Null(thrown); + Assert.Equal(1, continuations); Assert.NotNull(interrupted); Assert.Equal(BulkOperationStatus.Running, interrupted.Status); Assert.Equal(2, interrupted.Processed); @@ -188,14 +193,51 @@ public async Task Execute_TemporaryErrorsKeepHappening_FailsAfterTheRetriesAreUs var finished = await service.GetAsync(created.Id); // Assert - Assert.IsType(first); - Assert.IsType(second); + Assert.Null(first); + Assert.Null(second); Assert.Null(third); Assert.NotNull(finished); Assert.Equal(BulkOperationStatus.Failed, finished.Status); Assert.Equal(3, finished.RetryCount); } + [Fact] + public async Task Execute_TemporaryErrorsBetweenProgress_ContinueInNewRunsUntilDone() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 6; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + // A temporary error after every completed batch: more errors in total than the queue's retries for one work item. + var recorder = new RecordingHandler { Failure = call => call % 2 == 0 ? new TimeoutException("Elasticsearch timed out") : null }; + var service = CreateService(recorder); + var created = await CreateRecordingOperationAsync(service); + long enqueuedBefore = await GetEnqueuedWorkItemCountAsync(); + var thrown = new List(); + + // Act + for (int run = 0; run < 10 && (await service.GetAsync(created.Id))!.IsActive; run++) + thrown.Add(await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 1, TestCancellationToken))); + + long continuations = await GetEnqueuedWorkItemCountAsync() - enqueuedBefore; + var finished = await service.GetAsync(created.Id); + + // Assert + Assert.All(thrown, Assert.Null); + Assert.Equal(thrown.Count - 1, continuations); + Assert.NotNull(finished); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); + } + + private async Task GetEnqueuedWorkItemCountAsync() + { + return (await GetService>().GetQueueStatsAsync()).Enqueued; + } + [Fact] public async Task Execute_ErrorThatRetryingCannotFix_FailsImmediately() { diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index 4f4c018aa7..bbb45c9e53 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -697,7 +697,7 @@ private Task RunOperationAsync(BulkOperationService service, string operationId, private Task RunWorkItemAsync(BulkOperationService service, BulkOperationWorkItem workItem, int batchSize, CancellationToken shutdownToken) { var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(service, [GetService()], - new BulkOperationOptions { BatchSize = batchSize }, GetService(), TimeProvider, Log); + new BulkOperationOptions { BatchSize = batchSize, RetryDelay = TimeSpan.Zero }, GetService(), TimeProvider, Log); var context = new WorkItemContext(workItem, "test-job", EmptyLock.Empty, shutdownToken, static (_, _) => Task.CompletedTask); return handler.HandleItemAsync(context); From 70801469e867364e9167a66626b3e9735ca2984a Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 20:49:50 -0600 Subject: [PATCH 05/42] Renew the bulk operation lease after every batch Without it, a run longer than the queue's work item timeout could be delivered to a second worker while the first was still deleting. Co-Authored-By: Claude Opus 5.5 --- .../BulkOperationWorkItemHandler.cs | 4 ++++ .../BulkOperationEndpointTests.Reliability.cs | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs index 115c64f795..834a282d75 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -34,6 +34,10 @@ public BulkOperationWorkItemHandler(BulkOperationService service, IEnumerable GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs index 40d29086bf..d0442448c2 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -233,6 +233,20 @@ public async Task Execute_TemporaryErrorsBetweenProgress_ContinueInNewRunsUntilD Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); } + [Fact] + public void Handler_RenewsTheQueueEntryAndLockOnEveryBatch() + { + // Arrange + var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(CreateService(new RecordingHandler()), [GetService()], + new BulkOperationOptions(), GetService(), TimeProvider, Log); + + // Act + bool renews = handler.AutoRenewLockOnProgress; + + // Assert + Assert.True(renews); + } + private async Task GetEnqueuedWorkItemCountAsync() { return (await GetService>().GetQueueStatsAsync()).Enqueued; From dff385c438f1423a73a7cf9c28f9326071542dd8 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 20:58:47 -0600 Subject: [PATCH 06/42] Reject unknown bulk selection kinds and never restart a finished id selection An undefined selection kind fell through to an empty filter that selected everything in scope. A restart after the last id batch but before completion started the ids over, because the saved cursor was empty. Co-Authored-By: Claude Opus 5.5 --- .../BulkOperationWorkItemHandler.cs | 11 +++++- .../Api/Handlers/BulkOperationHandler.cs | 5 +++ .../BulkOperationEndpointTests.Reliability.cs | 38 +++++++++++++++++++ .../Endpoints/BulkOperationEndpointTests.cs | 4 +- 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs index 834a282d75..777bdd2f28 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -20,6 +20,12 @@ public class BulkOperationWorkItemHandler : WorkItemHandlerBase /// public const int MaxTransientFailures = 3; + /// + /// Saved as the cursor once the last batch is done, so a run that restarts before the operation is marked complete finds + /// nothing left to do instead of starting the selection over. + /// + internal const string ExhaustedCursor = "exhausted"; + private readonly BulkOperationService _service; private readonly IReadOnlyDictionary _resolvers; private readonly BulkOperationOptions _options; @@ -141,7 +147,7 @@ public override async Task HandleItemAsync(WorkItemContext context) current.Processed += page.Ids.Count; current.Succeeded += result.Succeeded; current.Failed += result.Failed; - current.Cursor = page.NextCursor; + current.Cursor = page.NextCursor ?? ExhaustedCursor; current.RetryCount = 0; current.Total = Math.Max(current.Total, current.Processed); current.Message = result.Message; @@ -229,6 +235,9 @@ internal static bool IsPermanent(Exception exception) private async Task GetPageAsync(BulkOperation operation, IBulkSelectionResolver resolver, string? pinnedSelection, CancellationToken cancellationToken) { int size = Math.Max(1, _options.BatchSize); + if (operation.Cursor == ExhaustedCursor) + return new BulkSelectionPage([], null); + if (operation.Selection.Kind is not BulkOperationSelectionKind.Ids) return await resolver.GetPageAsync(operation, pinnedSelection, operation.Cursor, size, cancellationToken); diff --git a/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs b/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs index 01a5294cad..902817478d 100644 --- a/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs @@ -175,6 +175,11 @@ public async Task>> Handle(CancelBulk return (new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ids }, null); } + // Only an explicit filter selection may resolve a filter. An unknown kind must never fall through to an empty filter, + // which would select everything in scope. + if (model.Kind is not BulkOperationSelectionKind.Filter) + return (null, Result.Invalid(ValidationError.Create("selection.kind", "The selection kind must be ids or filter."))); + if (model.Ids is { Count: > 0 }) return (null, Result.Invalid(ValidationError.Create("selection", "A filter selection cannot include ids."))); diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs index d0442448c2..f04de155ae 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -233,6 +233,44 @@ public async Task Execute_TemporaryErrorsBetweenProgress_ContinueInNewRunsUntilD Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); } + [Fact] + public async Task Execute_RestartAfterTheLastIdBatchBeforeCompletion_DoesNotStartOver() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var recorder = new RecordingHandler(); + var service = CreateService(recorder); + var operation = NewRecordingOperation("test.record"); + operation.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = events.Select(e => e.Id).ToList() }; + var result = await service.CreateAsync(operation); + var created = result.Operation; + Assert.NotNull(created); + await RunOperationAsync(service, created.Id, 2, TestCancellationToken); + int batchesBefore = recorder.Batches.Count; + // The worker stopped after saving the last batch but before saving the completed status. + await service.UpdateAsync(created.Id, current => + { + current.Status = BulkOperationStatus.Running; + current.CompletedUtc = null; + return true; + }); + + // Act + await RunOperationAsync(service, created.Id, 2, TestCancellationToken); + var finished = await service.GetAsync(created.Id); + + // Assert + Assert.Equal(batchesBefore, recorder.Batches.Count); + Assert.NotNull(finished); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(5, finished.Processed); + } + [Fact] public void Handler_RenewsTheQueueEntryAndLockOnEveryBatch() { diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index bbb45c9e53..307ade9e7f 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -400,7 +400,9 @@ public async Task Create_InvalidRequests_AreRejectedWithoutStartingAnything() FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [] } }, FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ["not-an-id"] } }, FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Filter = "type:log", Ids = ["537650f3b77efe23a47914f3"] } }, - FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ids } } + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ids } }, + // An undefined kind must not fall through to an empty filter that selects everything. + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = (BulkOperationSelectionKind)2 } } }; // Act From 8928092148dbb2e64d0b003279749e8bc563fb1d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 21:05:29 -0600 Subject: [PATCH 07/42] Fail a bulk operation that could not be queued Otherwise it stayed queued with no work item to run it and blocked new operations on the project. Co-Authored-By: Claude Opus 5.5 --- .../BulkOperations/BulkOperationService.cs | 26 ++++++++++++++----- .../BulkOperationEndpointTests.Reliability.cs | 24 +++++++++++++++++ .../Endpoints/BulkOperationEndpointTests.cs | 4 +-- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index 818c50e93a..e2daa18732 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -165,13 +165,27 @@ public async Task CreateAsync(BulkOperation operation await _cache.ListAddAsync(GetIndexKey(operation.OrganizationId), [operation.Id], OperationLifetime); await _cache.ListAddAsync(GetActiveIndexKey(operation.OrganizationId), [operation.Id], OperationLifetime); - await _workItemQueue.EnqueueAsync(new BulkOperationWorkItem + try { - Description = operation.Description, - OperationId = operation.Id, - OrganizationId = operation.OrganizationId, - Type = operation.Type - }); + await _workItemQueue.EnqueueAsync(new BulkOperationWorkItem + { + Description = operation.Description, + OperationId = operation.Id, + OrganizationId = operation.OrganizationId, + Type = operation.Type + }); + } + catch (Exception ex) + { + // Nothing would ever run an operation that was not queued, and it would block new operations on the project. + _logger.LogError(ex, "Unable to queue bulk operation {OperationId}: {Message}", operation.Id, ex.Message); + Finish(operation, BulkOperationStatus.Failed, "The operation could not be started. Please try again."); + await SaveAsync(operation); + await _cache.ListRemoveAsync(GetActiveIndexKey(operation.OrganizationId), [operation.Id]); + await PublishAsync(operation); + throw; + } + await PublishAsync(operation); return BulkOperationCreateResult.Created(operation); diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs index f04de155ae..1c20dc8045 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -271,6 +271,30 @@ await service.UpdateAsync(created.Id, current => Assert.Equal(5, finished.Processed); } + [Fact] + public async Task Create_WhenTheQueueRejectsTheWorkItem_FailsTheOperationSoItDoesNotBlockTheProject() + { + // Arrange + await CreateDataAsync(d => d.Event().TestProject()); + var unavailableQueue = new Mock>(); + unavailableQueue.SetupGet(q => q.Serializer).Returns(GetService>().Serializer); + unavailableQueue.Setup(q => q.EnqueueAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new TimeoutException("Queue unavailable")); + var recorder = new RecordingHandler("test.destroy", isDestructive: true); + var failingService = CreateService([recorder], [GetService()], unavailableQueue.Object); + var service = CreateService([recorder], [GetService()]); + + // Act + var thrown = await Record.ExceptionAsync(async () => await failingService.CreateAsync(NewRecordingOperation("test.destroy"))); + var orphan = (await service.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)).Single(); + var retry = await service.CreateAsync(NewRecordingOperation("test.destroy")); + + // Assert + Assert.IsType(thrown); + Assert.Equal(BulkOperationStatus.Failed, orphan.Status); + Assert.NotNull(retry.Operation); + Assert.Equal(BulkOperationStatus.Queued, retry.Operation.Status); + } + [Fact] public void Handler_RenewsTheQueueEntryAndLockOnEveryBatch() { diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index 307ade9e7f..90c61987d6 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -657,9 +657,9 @@ private BulkOperationService CreateService(params IBulkOperationHandler[] handle return CreateService(handlers, [GetService()]); } - private BulkOperationService CreateService(IBulkOperationHandler[] handlers, IBulkSelectionResolver[] resolvers) + private BulkOperationService CreateService(IBulkOperationHandler[] handlers, IBulkSelectionResolver[] resolvers, IQueue? workItemQueue = null) { - return new BulkOperationService(GetService(), GetService(), GetService(), GetService>(), + return new BulkOperationService(GetService(), GetService(), GetService(), workItemQueue ?? GetService>(), handlers, resolvers, TimeProvider, GetService>()); } From 11181b51415bd82fbe94ea1d82413bbac291ed7f Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 21:14:09 -0600 Subject: [PATCH 08/42] Pin relative dates inside a bulk operation's filter A filter such as data.date:[now/d-4d TO now/d+1d} kept its relative dates, so a run that crossed midnight or resumed later could select different events than the user confirmed. Co-Authored-By: Claude Opus 5.5 --- .../EventBulkSelectionResolver.cs | 3 +- .../BulkOperations/RelativeDateFreezer.cs | 62 +++++++++++++++++++ .../Endpoints/BulkOperationEndpointTests.cs | 27 ++++++++ .../RelativeDateFreezerTests.cs | 38 ++++++++++++ 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs create mode 100644 tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs index ab914a217a..181811d47f 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -76,7 +76,8 @@ public async Task PrepareAsync(BulkOperation operation var pinned = new PinnedEventSelection { - EventFilter = query.GetFilterExpression(), + // Relative dates in the filter are pinned to the creation time, like the outer time window. + EventFilter = await RelativeDateFreezer.FreezeAsync(query.GetFilterExpression(), operation.Selection.UtcCreatedBefore ?? operation.CreatedUtc), ExcludedStackIds = query.GetExcludedStacks().ToArray(), StackIds = query.GetStacks().ToArray() }; diff --git a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs new file mode 100644 index 0000000000..c31c96cd57 --- /dev/null +++ b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs @@ -0,0 +1,62 @@ +using Foundatio.Parsers.LuceneQueries; +using Foundatio.Parsers.LuceneQueries.Nodes; +using Foundatio.Parsers.LuceneQueries.Visitors; + +namespace Exceptionless.Core.Services.BulkOperations; + +/// +/// Pins relative dates in a filter (for example data.date:[now/d-4d TO now/d+1d}) to a fixed time, so a bulk +/// operation keeps selecting the same events no matter when each batch runs. +/// +public static class RelativeDateFreezer +{ + public static async Task FreezeAsync(string? filter, DateTime utcNow) + { + if (String.IsNullOrWhiteSpace(filter) || filter.IndexOf("now", StringComparison.OrdinalIgnoreCase) < 0) + return filter; + + var result = await new LuceneQueryParser().ParseAsync(filter); + if (result is null) + return filter; + + await new FreezeRelativeDatesVisitor(utcNow).AcceptAsync(result, new QueryVisitorContext()); + return result.ToString(); + } + + private sealed class FreezeRelativeDatesVisitor(DateTime utcNow) : ChainableQueryVisitor + { + // Elasticsearch date math anchors an expression on a date with ||, for example 2026-11-03T04:00:00.000Z||/d-4d. + private readonly string _anchor = $"{utcNow.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.fffZ}||"; + + public override Task VisitAsync(TermRangeNode node, IQueryVisitorContext context) + { + if (TryFreeze(node.Min, out string? min)) + { + node.Min = min; + node.IsMinQuotedTerm = true; + } + + if (TryFreeze(node.Max, out string? max)) + { + node.Max = max; + node.IsMaxQuotedTerm = true; + } + + return Task.CompletedTask; + } + + private bool TryFreeze(string? value, out string? frozen) + { + frozen = null; + if (value is null || !value.StartsWith("now", StringComparison.OrdinalIgnoreCase)) + return false; + + string math = value[3..]; + if (math.Length > 0 && math[0] is not ('+' or '-' or '/')) + return false; + + frozen = _anchor + math; + return true; + } + } +} diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index 90c61987d6..4b25449f33 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -73,6 +73,33 @@ public async Task Create_FilterSelection_DeletesEverythingMatchingThroughTheQueu Assert.Empty(await GetRemainingMessagesAsync(events)); } + [Fact] + public async Task Create_FilterWithRelativeDates_PinsThemAndDeletesOnlyTheMatches() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("inside").Date(now.AddHours(-1)); + d.Event().TestProject().Message("outside").Date(now.AddHours(-3)); + }); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "date:[now-2h TO now]")); + string? pinned = await _bulkOperationService.GetPinnedSelectionAsync(created.Id); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var finished = await GetOperationAsync(created.Id); + + // Assert + Assert.NotNull(pinned); + Assert.DoesNotContain("now", pinned, StringComparison.OrdinalIgnoreCase); + Assert.Contains("||-2h", pinned); + Assert.Equal(1, created.Total); + Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(1, finished.Succeeded); + Assert.Equal(["outside"], (await GetRemainingMessagesAsync(events)).ToArray()); + } + [Fact] public async Task Create_FilterSelection_OnlyDeletesMatchesInsideScopeFilterAndWindow() { diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs new file mode 100644 index 0000000000..1f59b584ea --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs @@ -0,0 +1,38 @@ +using Exceptionless.Core.Services.BulkOperations; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class RelativeDateFreezerTests +{ + private static readonly DateTime UtcNow = new(2026, 11, 3, 4, 5, 6, 789, DateTimeKind.Utc); + + [Theory] + [InlineData("data.date:[now/d-4d TO now/d+1d}", "data.date:[\"2026-11-03T04:05:06.789Z||/d-4d\" TO \"2026-11-03T04:05:06.789Z||/d+1d\"}")] + [InlineData("date:>now-1h", "date:>\"2026-11-03T04:05:06.789Z||-1h\"")] + [InlineData("type:error AND date:[now-7d TO now]", "type:error AND date:[\"2026-11-03T04:05:06.789Z||-7d\" TO \"2026-11-03T04:05:06.789Z||\"]")] + public async Task FreezeAsync_RelativeDates_AreAnchoredToTheGivenTime(string filter, string expected) + { + // Act + string? frozen = await RelativeDateFreezer.FreezeAsync(filter, UtcNow); + + // Assert + Assert.Equal(expected, frozen); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("type:error")] + [InlineData("message:now")] + [InlineData("date:[2026-01-01 TO 2026-02-01]")] + [InlineData("tag:[nowhere TO zebra]")] + public async Task FreezeAsync_WithoutRelativeDates_LeavesTheFilterAlone(string? filter) + { + // Act + string? frozen = await RelativeDateFreezer.FreezeAsync(filter, UtcNow); + + // Assert + Assert.DoesNotContain("||", frozen ?? String.Empty); + } +} From 794c82ec7b7fcc0bad0aeebb8eb1d073fb6f6679 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 21:23:02 -0600 Subject: [PATCH 09/42] Pin relative dates only on date fields and retire stale bulk operations Relative bounds on string fields such as environment:[now TO z] are left alone, and custom data date fields keep their date type by naming the stored field. A stale destructive operation is now failed before a replacement is admitted, so a delayed delivery cannot run alongside it. Co-Authored-By: Claude Opus 5.5 --- .../Visitors/EventFieldsQueryVisitor.cs | 5 +- .../BulkOperations/BulkOperationService.cs | 26 ++++++++- .../EventBulkSelectionResolver.cs | 12 +++- .../BulkOperations/RelativeDateFreezer.cs | 56 +++++++++++-------- .../BulkOperationEndpointTests.Reliability.cs | 2 +- .../Endpoints/BulkOperationEndpointTests.cs | 7 ++- .../RelativeDateFreezerTests.cs | 21 +++++-- 7 files changed, 92 insertions(+), 37 deletions(-) diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs index 43065cc189..172d128832 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs @@ -67,7 +67,10 @@ public override Task VisitAsync(MissingNode node, IQueryVisitorContext context) return Task.CompletedTask; } - private string? GetCustomFieldName(string? field, string?[] terms) + /// + /// Maps a custom data or reference field to the stored idx field. The suffix is the stored type, picked from the terms. + /// + internal static string? GetCustomFieldName(string? field, string?[] terms) { if (String.IsNullOrEmpty(field)) return null; diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index e2daa18732..e94146cc82 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -272,13 +272,33 @@ public void Finish(BulkOperation operation, BulkOperationStatus status, string? continue; } - if (utcNow - existing.UpdatedUtc >= StaleAfter || GetHandler(existing.Type) is not { IsDestructive: true }) + if (GetHandler(existing.Type) is not { IsDestructive: true }) continue; // An organization wide operation can touch every project, so it conflicts with any project's operation. bool overlaps = existing.ProjectId is null || candidate.ProjectId is null || String.Equals(existing.ProjectId, candidate.ProjectId, StringComparison.Ordinal); - if (overlaps) - conflict ??= existing; + if (!overlaps) + continue; + + if (utcNow - existing.UpdatedUtc >= StaleAfter) + { + // Retire it before admitting a replacement, so a delayed delivery cannot run it alongside the new one. + var retired = await UpdateAsync(existing.Id, current => + { + if (!current.IsActive || utcNow - current.UpdatedUtc < StaleAfter) + return false; + + Finish(current, BulkOperationStatus.Failed, "Stopped because it made no progress for two hours."); + return true; + }); + + if (retired is null || !retired.IsActive) + continue; + + existing = retired; + } + + conflict ??= existing; } if (finished.Count > 0) diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs index 181811d47f..ad342edee3 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -1,3 +1,4 @@ +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Models; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Queries.Validation; @@ -27,15 +28,22 @@ public class EventBulkSelectionResolver : IBulkSelectionResolver private readonly IProjectRepository _projectRepository; private readonly PersistentEventQueryValidator _validator; private readonly ITextSerializer _serializer; + private readonly ExceptionlessElasticConfiguration _configuration; public EventBulkSelectionResolver(IEventRepository eventRepository, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, - PersistentEventQueryValidator validator, ITextSerializer serializer) + PersistentEventQueryValidator validator, ITextSerializer serializer, ExceptionlessElasticConfiguration configuration) { _eventRepository = eventRepository; _organizationRepository = organizationRepository; _projectRepository = projectRepository; _validator = validator; _serializer = serializer; + _configuration = configuration; + } + + private ValueTask IsDateFieldAsync(string field) + { + return ValueTask.FromResult(_configuration.Events.MappingResolver.GetMappingProperty(field, followAlias: true) is DateProperty); } public BulkOperationEntity Entity => BulkOperationEntity.Events; @@ -77,7 +85,7 @@ public async Task PrepareAsync(BulkOperation operation var pinned = new PinnedEventSelection { // Relative dates in the filter are pinned to the creation time, like the outer time window. - EventFilter = await RelativeDateFreezer.FreezeAsync(query.GetFilterExpression(), operation.Selection.UtcCreatedBefore ?? operation.CreatedUtc), + EventFilter = await RelativeDateFreezer.FreezeAsync(query.GetFilterExpression(), operation.Selection.UtcCreatedBefore ?? operation.CreatedUtc, IsDateFieldAsync), ExcludedStackIds = query.GetExcludedStacks().ToArray(), StackIds = query.GetStacks().ToArray() }; diff --git a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs index c31c96cd57..b2ffb64a27 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs @@ -1,3 +1,4 @@ +using Exceptionless.Core.Repositories.Queries; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; @@ -6,11 +7,15 @@ namespace Exceptionless.Core.Services.BulkOperations; /// /// Pins relative dates in a filter (for example data.date:[now/d-4d TO now/d+1d}) to a fixed time, so a bulk -/// operation keeps selecting the same events no matter when each batch runs. +/// operation keeps selecting the same events no matter when each batch runs. Only date fields are changed: a string range +/// such as environment:[now TO z] is left alone. /// public static class RelativeDateFreezer { - public static async Task FreezeAsync(string? filter, DateTime utcNow) + /// The filter to pin. + /// The time that now stands for. + /// Whether a built-in (non custom data) field is mapped as a date. + public static async Task FreezeAsync(string? filter, DateTime utcNow, Func> isDateFieldAsync) { if (String.IsNullOrWhiteSpace(filter) || filter.IndexOf("now", StringComparison.OrdinalIgnoreCase) < 0) return filter; @@ -19,44 +24,51 @@ public static class RelativeDateFreezer if (result is null) return filter; - await new FreezeRelativeDatesVisitor(utcNow).AcceptAsync(result, new QueryVisitorContext()); + await new FreezeRelativeDatesVisitor(utcNow, isDateFieldAsync).AcceptAsync(result, new QueryVisitorContext()); return result.ToString(); } - private sealed class FreezeRelativeDatesVisitor(DateTime utcNow) : ChainableQueryVisitor + private sealed class FreezeRelativeDatesVisitor(DateTime utcNow, Func> isDateFieldAsync) : ChainableQueryVisitor { // Elasticsearch date math anchors an expression on a date with ||, for example 2026-11-03T04:00:00.000Z||/d-4d. private readonly string _anchor = $"{utcNow.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.fffZ}||"; - public override Task VisitAsync(TermRangeNode node, IQueryVisitorContext context) + public override async Task VisitAsync(TermRangeNode node, IQueryVisitorContext context) { - if (TryFreeze(node.Min, out string? min)) + if (String.IsNullOrEmpty(node.Field) || (!IsRelative(node.Min) && !IsRelative(node.Max))) + return; + + // Custom data fields are stored by type, and the events index picks a date field only when the bounds look like + // dates. Freezing the bounds would hide that, so name the stored date field directly. + string? storedField = EventFieldsQueryVisitor.GetCustomFieldName(node.Field, [node.Min, node.Max]); + if (!String.Equals(storedField, node.Field, StringComparison.Ordinal)) + { + if (storedField is null || !storedField.EndsWith("-d", StringComparison.Ordinal)) + return; + + node.Field = storedField; + } + else if (!await isDateFieldAsync(node.Field)) + { + return; + } + + if (IsRelative(node.Min)) { - node.Min = min; + node.Min = _anchor + node.Min![3..]; node.IsMinQuotedTerm = true; } - if (TryFreeze(node.Max, out string? max)) + if (IsRelative(node.Max)) { - node.Max = max; + node.Max = _anchor + node.Max![3..]; node.IsMaxQuotedTerm = true; } - - return Task.CompletedTask; } - private bool TryFreeze(string? value, out string? frozen) + private static bool IsRelative(string? value) { - frozen = null; - if (value is null || !value.StartsWith("now", StringComparison.OrdinalIgnoreCase)) - return false; - - string math = value[3..]; - if (math.Length > 0 && math[0] is not ('+' or '-' or '/')) - return false; - - frozen = _anchor + math; - return true; + return value is not null && value.StartsWith("now", StringComparison.OrdinalIgnoreCase) && (value.Length == 3 || value[3] is '+' or '-' or '/'); } } } diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs index 1c20dc8045..edc2307ffd 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -413,7 +413,7 @@ public async Task Cancel_FinishedOperation_ChangesNothing() private EventBulkSelectionResolver CreateResolver(IEventRepository eventRepository) { return new EventBulkSelectionResolver(eventRepository, GetService(), GetService(), - GetService(), GetService()); + GetService(), GetService(), GetService()); } private BulkOperation NewIdsOperation(string type) diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index 4b25449f33..131740da00 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -399,17 +399,20 @@ public async Task Create_AfterTheActiveOperationFinishesOrIsCancelled_IsAllowedA } [Fact] - public async Task Create_WhileTheActiveOperationHasStoppedReportingProgress_IsAllowed() + public async Task Create_WhileTheActiveOperationHasStoppedReportingProgress_RetiresItAndIsAllowed() { // Arrange - await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + var stale = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); // Act TimeProvider.Advance(BulkOperationService.StaleAfter + TimeSpan.FromMinutes(1)); var next = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); + var retired = await GetOperationAsync(stale.Id); // Assert Assert.Equal(BulkOperationStatus.Queued, next.Status); + // A delayed delivery of the stale operation must not run alongside its replacement. + Assert.Equal(BulkOperationStatus.Failed, retired.Status); } [Fact] diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs index 1f59b584ea..155521230e 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs @@ -1,20 +1,26 @@ using Exceptionless.Core.Services.BulkOperations; +using Foundatio.Xunit; using Xunit; namespace Exceptionless.Tests.Services.BulkOperations; -public class RelativeDateFreezerTests +public class RelativeDateFreezerTests : TestWithLoggingBase { private static readonly DateTime UtcNow = new(2026, 11, 3, 4, 5, 6, 789, DateTimeKind.Utc); + public RelativeDateFreezerTests(ITestOutputHelper output) : base(output) { } + + // Built-in fields are looked up in the index mapping; here only date and created_utc are dates. + private static ValueTask IsDateFieldAsync(string field) => ValueTask.FromResult(field is "date" or "created_utc"); + [Theory] - [InlineData("data.date:[now/d-4d TO now/d+1d}", "data.date:[\"2026-11-03T04:05:06.789Z||/d-4d\" TO \"2026-11-03T04:05:06.789Z||/d+1d\"}")] [InlineData("date:>now-1h", "date:>\"2026-11-03T04:05:06.789Z||-1h\"")] [InlineData("type:error AND date:[now-7d TO now]", "type:error AND date:[\"2026-11-03T04:05:06.789Z||-7d\" TO \"2026-11-03T04:05:06.789Z||\"]")] - public async Task FreezeAsync_RelativeDates_AreAnchoredToTheGivenTime(string filter, string expected) + [InlineData("data.date:[now/d-4d TO now/d+1d}", "idx.date-d:[\"2026-11-03T04:05:06.789Z||/d-4d\" TO \"2026-11-03T04:05:06.789Z||/d+1d\"}")] + public async Task FreezeAsync_RelativeDatesOnDateFields_AreAnchoredToTheGivenTime(string filter, string expected) { // Act - string? frozen = await RelativeDateFreezer.FreezeAsync(filter, UtcNow); + string? frozen = await RelativeDateFreezer.FreezeAsync(filter, UtcNow, IsDateFieldAsync); // Assert Assert.Equal(expected, frozen); @@ -27,10 +33,13 @@ public async Task FreezeAsync_RelativeDates_AreAnchoredToTheGivenTime(string fil [InlineData("message:now")] [InlineData("date:[2026-01-01 TO 2026-02-01]")] [InlineData("tag:[nowhere TO zebra]")] - public async Task FreezeAsync_WithoutRelativeDates_LeavesTheFilterAlone(string? filter) + [InlineData("environment:[now TO z]")] + [InlineData("data.name:[now TO z]")] + [InlineData("ref.parent:[now TO z]")] + public async Task FreezeAsync_WithoutRelativeDatesOnDateFields_LeavesTheFilterAlone(string? filter) { // Act - string? frozen = await RelativeDateFreezer.FreezeAsync(filter, UtcNow); + string? frozen = await RelativeDateFreezer.FreezeAsync(filter, UtcNow, IsDateFieldAsync); // Assert Assert.DoesNotContain("||", frozen ?? String.Empty); From 629d02bdf681d913961664c91ac9f23e7913a937 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Mon, 2 Nov 2026 21:30:12 -0600 Subject: [PATCH 10/42] Pin relative dates in ranges that take their field from a group Grouped filters such as date:(>now-2h AND <=now) put the field on the group, and custom data groups are typed by all of their terms, as the events index does. Co-Authored-By: Claude Opus 5.5 --- .../Visitors/EventFieldsQueryVisitor.cs | 14 +++- .../BulkOperations/RelativeDateFreezer.cs | 67 ++++++++++++++++--- .../RelativeDateFreezerTests.cs | 5 ++ 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs index 172d128832..a41f8c7dc2 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs @@ -8,6 +8,16 @@ namespace Exceptionless.Core.Repositories.Queries; public class EventFieldsQueryVisitor : ChainableQueryVisitor { public override async Task VisitAsync(GroupNode node, IQueryVisitorContext context) + { + node.Field = GetCustomFieldName(node.Field, GetGroupChildTerms(node)) ?? node.Field; + foreach (var child in node.Children) + await child.AcceptAsync(this, context); + } + + /// + /// The terms of a group's unfielded children, which pick the stored type of a custom field set on the group. + /// + internal static string[] GetGroupChildTerms(GroupNode node) { var childTerms = new List(); if (node.Left is TermNode { Field: null, Term: not null } leftTermNode) @@ -32,9 +42,7 @@ public override async Task VisitAsync(GroupNode node, IQueryVisitorContext conte childTerms.Add(rightTermRangeNode.Max); } - node.Field = GetCustomFieldName(node.Field, childTerms.ToArray()) ?? node.Field; - foreach (var child in node.Children) - await child.AcceptAsync(this, context); + return childTerms.ToArray(); } public override Task VisitAsync(TermNode node, IQueryVisitorContext context) diff --git a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs index b2ffb64a27..18a2918a71 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs @@ -33,24 +33,50 @@ private sealed class FreezeRelativeDatesVisitor(DateTime utcNow, Funcnow-2h AND <=now) takes its field from the group. + string? field = node.Field ?? GetGroupField(node); + if (String.IsNullOrEmpty(field)) return; - // Custom data fields are stored by type, and the events index picks a date field only when the bounds look like - // dates. Freezing the bounds would hide that, so name the stored date field directly. - string? storedField = EventFieldsQueryVisitor.GetCustomFieldName(node.Field, [node.Min, node.Max]); - if (!String.Equals(storedField, node.Field, StringComparison.Ordinal)) + if (node.Field is null) { - if (storedField is null || !storedField.EndsWith("-d", StringComparison.Ordinal)) + // The group already decided the stored field: a custom field left unchanged there is not a date. + if (!IsStoredDateField(field) && (IsCustomField(field) || !await isDateFieldAsync(field))) return; - - node.Field = storedField; } - else if (!await isDateFieldAsync(node.Field)) + else { - return; + // Custom data fields are stored by type, and the events index picks a date field only when the bounds look like + // dates. Freezing the bounds would hide that, so name the stored date field directly. + string? storedField = EventFieldsQueryVisitor.GetCustomFieldName(field, [node.Min, node.Max]); + if (IsCustomField(field)) + { + if (!IsStoredDateField(storedField)) + return; + + node.Field = storedField; + } + else if (!IsStoredDateField(field) && !await isDateFieldAsync(field)) + { + return; + } } if (IsRelative(node.Min)) @@ -66,6 +92,27 @@ public override async Task VisitAsync(TermRangeNode node, IQueryVisitorContext c } } + private static bool IsCustomField(string field) + { + return !String.Equals(EventFieldsQueryVisitor.GetCustomFieldName(field, []), field, StringComparison.Ordinal); + } + + private static bool IsStoredDateField(string? field) + { + return field is not null && field.StartsWith("idx.", StringComparison.OrdinalIgnoreCase) && field.EndsWith("-d", StringComparison.Ordinal); + } + + private static string? GetGroupField(IQueryNode node) + { + for (var parent = node.Parent; parent is not null; parent = parent.Parent) + { + if (parent is GroupNode { Field: { Length: > 0 } groupField }) + return groupField; + } + + return null; + } + private static bool IsRelative(string? value) { return value is not null && value.StartsWith("now", StringComparison.OrdinalIgnoreCase) && (value.Length == 3 || value[3] is '+' or '-' or '/'); diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs index 155521230e..bce2a78e2d 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs @@ -16,6 +16,8 @@ public RelativeDateFreezerTests(ITestOutputHelper output) : base(output) { } [Theory] [InlineData("date:>now-1h", "date:>\"2026-11-03T04:05:06.789Z||-1h\"")] [InlineData("type:error AND date:[now-7d TO now]", "type:error AND date:[\"2026-11-03T04:05:06.789Z||-7d\" TO \"2026-11-03T04:05:06.789Z||\"]")] + [InlineData("date:(>now-2h AND <=now)", "date:(>\"2026-11-03T04:05:06.789Z||-2h\" AND <=\"2026-11-03T04:05:06.789Z||\")")] + [InlineData("data.date:(>=now/d AND =\"2026-11-03T04:05:06.789Z||/d\" AND <\"2026-11-03T04:05:06.789Z||/d+1d\")")] [InlineData("data.date:[now/d-4d TO now/d+1d}", "idx.date-d:[\"2026-11-03T04:05:06.789Z||/d-4d\" TO \"2026-11-03T04:05:06.789Z||/d+1d\"}")] public async Task FreezeAsync_RelativeDatesOnDateFields_AreAnchoredToTheGivenTime(string filter, string expected) { @@ -34,6 +36,9 @@ public async Task FreezeAsync_RelativeDatesOnDateFields_AreAnchoredToTheGivenTim [InlineData("date:[2026-01-01 TO 2026-02-01]")] [InlineData("tag:[nowhere TO zebra]")] [InlineData("environment:[now TO z]")] + [InlineData("environment:(>now AND now-2h AND <=now)")] [InlineData("data.name:[now TO z]")] [InlineData("ref.parent:[now TO z]")] public async Task FreezeAsync_WithoutRelativeDatesOnDateFields_LeavesTheFilterAlone(string? filter) From fd19ca0fe43fc5d7be74765e6f151bb1febbdf0b Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 3 Nov 2026 23:13:58 -0600 Subject: [PATCH 11/42] Track bulk operations and project work as background tasks Replace the bulk operation record with a generic BackgroundTask stored by BackgroundTaskService. Bulk specific logic stays in BulkOperationService. Reset data and sample data return a task id and report progress through a decorating work item handler. Add GET/list/cancel task endpoints. --- src/Exceptionless.Core/Bootstrapper.cs | 12 +- .../BackgroundTaskTrackingWorkItemHandler.cs | 172 ++++++++++ .../BulkOperationWorkItemHandler.cs | 132 +++---- .../Models/BackgroundTasks/BackgroundTask.cs | 76 +++++ .../BackgroundTasks/BackgroundTaskTypes.cs | 8 + ...Operation.cs => BulkOperationSelection.cs} | 43 +-- ...ionUpdated.cs => BackgroundTaskUpdated.cs} | 10 +- .../Models/WorkItems/BulkOperationWorkItem.cs | 6 +- .../WorkItems/GenerateSampleEventsWorkItem.cs | 3 +- .../WorkItems/IBackgroundTaskWorkItem.cs | 10 + .../WorkItems/ResetProjectDataWorkItem.cs | 3 +- .../BackgroundTasks/BackgroundTaskService.cs | 322 ++++++++++++++++++ .../BulkOperations/BulkOperationService.cs | 322 ++++-------------- .../DeleteEventsBulkOperationHandler.cs | 17 +- .../EventBulkSelectionResolver.cs | 33 +- .../BulkOperations/IBulkOperationHandler.cs | 21 +- .../Utility/SampleDataService.cs | 30 +- src/Exceptionless.Web/Api/ApiEndpoints.cs | 2 +- ...ndpoints.cs => BackgroundTaskEndpoints.cs} | 51 +-- ...ionHandler.cs => BackgroundTaskHandler.cs} | 134 ++++---- .../Api/Handlers/ProjectHandler.cs | 22 +- .../Api/Messages/BackgroundTaskMessages.cs | 8 + .../Api/Messages/BulkOperationMessages.cs | 8 - .../ClientApp/src/lib/generated/api.ts | 39 ++- .../ClientApp/src/lib/generated/schemas.ts | 24 +- .../Hubs/MessageBusBroker.cs | 10 +- .../ViewBackgroundTask.cs} | 20 +- .../Api/Data/endpoint-manifest.json | 86 ++--- .../Exceptionless.Tests/Api/Data/openapi.json | 122 ++++--- .../BulkOperationEndpointTests.Reliability.cs | 130 +++---- .../Endpoints/BulkOperationEndpointTests.cs | 166 ++++----- 31 files changed, 1265 insertions(+), 777 deletions(-) create mode 100644 src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs create mode 100644 src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs create mode 100644 src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs rename src/Exceptionless.Core/Models/BulkOperations/{BulkOperation.cs => BulkOperationSelection.cs} (53%) rename src/Exceptionless.Core/Models/Messaging/{BulkOperationUpdated.cs => BackgroundTaskUpdated.cs} (65%) create mode 100644 src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs create mode 100644 src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs rename src/Exceptionless.Web/Api/Endpoints/{BulkOperationEndpoints.cs => BackgroundTaskEndpoints.cs} (58%) rename src/Exceptionless.Web/Api/Handlers/{BulkOperationHandler.cs => BackgroundTaskHandler.cs} (66%) create mode 100644 src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs delete mode 100644 src/Exceptionless.Web/Api/Messages/BulkOperationMessages.cs rename src/Exceptionless.Web/Models/{BulkOperation/ViewBulkOperation.cs => BackgroundTask/ViewBackgroundTask.cs} (64%) diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index d6f73449a1..3f004628fc 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -26,6 +26,7 @@ using Exceptionless.Core.Seed; using Exceptionless.Core.Serialization; using Exceptionless.Core.Services; +using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Core.Services.BulkOperations; using Exceptionless.Core.Services.SourceMaps; using Exceptionless.Core.Utility; @@ -104,12 +105,12 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); - handlers.Register(s.GetRequiredService); + handlers.Register(() => TrackWithBackgroundTask(s, s.GetRequiredService())); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); - handlers.Register(s.GetRequiredService); + handlers.Register(() => TrackWithBackgroundTask(s, s.GetRequiredService())); return handlers; }); @@ -237,6 +238,8 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(typeof(IBulkOperationHandler), typeof(Bootstrapper).Assembly); @@ -535,6 +538,11 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log logger.LogWarning("Jobs running in process"); } + private static IWorkItemHandler TrackWithBackgroundTask(IServiceProvider container, IWorkItemHandler handler) + { + return new BackgroundTaskTrackingWorkItemHandler(handler, container.GetRequiredService(), container.GetRequiredService()); + } + private static IQueue CreateQueue(IServiceProvider container, TimeSpan? workItemTimeout = null) where T : class { var loggerFactory = container.GetRequiredService(); diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs new file mode 100644 index 0000000000..bb5437979f --- /dev/null +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs @@ -0,0 +1,172 @@ +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Queues; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs.WorkItemHandlers; + +/// +/// Wraps the handler of a work item that is tracked by a background task. Progress, success and failure are written to the +/// task in the job process that runs the work (not from a bus subscriber, which every web instance would run), then the +/// original behavior happens: the wrapped handler runs and the job's own progress callback still renews the locks. Work items +/// without a task id pass straight through. +/// +internal sealed class BackgroundTaskTrackingWorkItemHandler : IWorkItemHandler +{ + private readonly IWorkItemHandler _inner; + private readonly BackgroundTaskService _taskService; + private readonly TimeProvider _timeProvider; + + public BackgroundTaskTrackingWorkItemHandler(IWorkItemHandler inner, BackgroundTaskService taskService, TimeProvider timeProvider) + { + _inner = inner; + _taskService = taskService; + _timeProvider = timeProvider; + } + + public bool AutoRenewLockOnProgress + { + get => _inner.AutoRenewLockOnProgress; + set => _inner.AutoRenewLockOnProgress = value; + } + + public ILogger Log + { + get => _inner.Log; + set => _inner.Log = value; + } + + public Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) + { + return _inner.GetWorkItemLockAsync(workItem, cancellationToken); + } + + public void LogProcessingQueueEntry(IQueueEntry queueEntry, Type workItemDataType, object workItem) + { + _inner.LogProcessingQueueEntry(queueEntry, workItemDataType, workItem); + } + + public void LogAutoCompletedQueueEntry(IQueueEntry queueEntry, Type workItemDataType, object workItem) + { + _inner.LogAutoCompletedQueueEntry(queueEntry, workItemDataType, workItem); + } + + public async Task HandleItemAsync(WorkItemContext context) + { + if (context.Data is not IBackgroundTaskWorkItem { TaskId: { Length: > 0 } taskId }) + { + await _inner.HandleItemAsync(context); + return; + } + + string? lastMessage = null; + var tracked = new WorkItemContext(context.Data, context.JobId, context.WorkItemLock, context.CancellationToken, async (progress, message) => + { + lastMessage = message ?? lastMessage; + await TryUpdateAsync(taskId, task => + { + if (!task.IsActive) + return false; + + task.Start(UtcNow); + task.Progress = Math.Clamp(progress, 0, 100); + task.Message = message; + return true; + }); + + await context.ReportProgressAsync(progress, message); + }); + + await StartAsync(taskId); + + try + { + await _inner.HandleItemAsync(tracked); + } + catch (Exception ex) + { + await TryUpdateAsync(taskId, task => + { + if (!task.IsActive) + return false; + + task.Finish(BackgroundTaskStatus.Failed, $"Failed: {ex.Message}", UtcNow); + return true; + }); + + throw; + } + + context.Result = tracked.Result; + + // A result that is not a success (for example a shutdown) means the queue runs the work item again, so the task keeps running. + if (!tracked.Result.IsSuccess) + return; + + await TryUpdateAsync(taskId, task => + { + if (!task.IsActive) + return false; + + task.Finish(BackgroundTaskStatus.Completed, lastMessage ?? "Completed.", UtcNow); + return true; + }); + } + + private DateTime UtcNow => _timeProvider.GetUtcNow().UtcDateTime; + + /// + /// Marks the task running. A queue retry runs the work item again after it failed, which brings the task back to running + /// and back into the active index. A cancelled or completed task stays as it is. + /// + private async Task StartAsync(string taskId) + { + bool wasFinished = false; + var task = await TryUpdateAsync(taskId, current => + { + if (current.Status is BackgroundTaskStatus.Cancelled or BackgroundTaskStatus.Completed) + return false; + + wasFinished = current.Status is BackgroundTaskStatus.Failed; + current.Start(UtcNow); + if (wasFinished) + { + current.CompletedUtc = null; + current.Message = null; + current.Progress = 0; + } + + return true; + }); + + if (wasFinished && task is not null) + await TryAsync(taskId, () => _taskService.AddToActiveIndexAsync(task.OrganizationId, task.Id)); + } + + private async Task TryUpdateAsync(string taskId, Func update) + { + BackgroundTask? result = null; + await TryAsync(taskId, async () => result = await _taskService.UpdateAsync(taskId, update)); + return result; + } + + // Tracking is best effort: a cache or lock problem must not fail the work the user asked for. + private async Task TryAsync(string taskId, Func action) + { + try + { + await action(); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + Log.LogWarning(ex, "Unable to update background task {TaskId}: {Message}", taskId, ex.Message); + } + } +} diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs index 777bdd2f28..a1c12bf08f 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -1,7 +1,9 @@ using System.Text.Json; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories.Base; +using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Core.Services.BulkOperations; using Foundatio.Jobs; using Foundatio.Lock; @@ -10,108 +12,109 @@ namespace Exceptionless.Core.Jobs.WorkItemHandlers; /// -/// Runs a stored bulk operation in batches. After every batch the counts and the resume cursor are saved and a progress +/// Runs a stored bulk task in batches. After every batch the counts and the resume cursor are saved and a progress /// message is published; between batches it checks for cancellation. A restarted job continues from the saved cursor. /// public class BulkOperationWorkItemHandler : WorkItemHandlerBase { /// - /// How many temporary errors in a row (without a batch completing in between) end the operation. + /// How many temporary errors in a row (without a batch completing in between) end the task. /// public const int MaxTransientFailures = 3; /// - /// Saved as the cursor once the last batch is done, so a run that restarts before the operation is marked complete finds + /// Saved as the cursor once the last batch is done, so a run that restarts before the task is marked complete finds /// nothing left to do instead of starting the selection over. /// internal const string ExhaustedCursor = "exhausted"; private readonly BulkOperationService _service; + private readonly BackgroundTaskService _taskService; private readonly IReadOnlyDictionary _resolvers; private readonly BulkOperationOptions _options; private readonly ILockProvider _lockProvider; private readonly TimeProvider _timeProvider; - public BulkOperationWorkItemHandler(BulkOperationService service, IEnumerable resolvers, BulkOperationOptions options, + public BulkOperationWorkItemHandler(BulkOperationService service, BackgroundTaskService taskService, IEnumerable resolvers, BulkOperationOptions options, ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory) { _service = service; + _taskService = taskService; _resolvers = resolvers.ToDictionary(r => r.Entity); _options = options; _lockProvider = lockProvider; _timeProvider = timeProvider; - // Progress is reported after every batch. Renewing the queue entry and the operation lock then keeps a long run from + // Progress is reported after every batch. Renewing the queue entry and the task lock then keeps a long run from // timing out and being delivered to a second worker while this one is still deleting. AutoRenewLockOnProgress = true; } public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) { - string cacheKey = $"{nameof(BulkOperationWorkItem)}:{((BulkOperationWorkItem)workItem).OperationId}"; + string cacheKey = $"{nameof(BulkOperationWorkItem)}:{((BulkOperationWorkItem)workItem).TaskId}"; return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(15), cancellationToken); } public override async Task HandleItemAsync(WorkItemContext context) { var workItem = context.GetData()!; - ArgumentException.ThrowIfNullOrWhiteSpace(workItem.OperationId); + ArgumentException.ThrowIfNullOrWhiteSpace(workItem.TaskId); - var operation = await _service.GetAsync(workItem.OperationId); - if (operation is null) + var task = await _taskService.GetAsync(workItem.TaskId); + if (task is null) { // The record expired or the cache was cleared. Say so, otherwise the progress panel waits forever. - Log.LogWarning("Bulk operation {OperationId} no longer exists", workItem.OperationId); + Log.LogWarning("Bulk operation {TaskId} no longer exists", workItem.TaskId); await _service.PublishLostAsync(workItem); return; } - using var _ = Log.BeginScope(new ExceptionlessState().Organization(operation.OrganizationId).Tag("BulkOperation")); - if (!operation.IsActive) + using var _ = Log.BeginScope(new ExceptionlessState().Organization(task.OrganizationId).Tag("BulkOperation")); + if (!task.IsActive) { - Log.LogInformation("Skipping bulk operation {OperationId} because it is already {Status}", operation.Id, operation.Status); + Log.LogInformation("Skipping bulk operation {TaskId} because it is already {Status}", task.Id, task.Status); return; } - var handler = _service.GetHandler(operation.Type); + var handler = _service.GetHandler(task.Type); if (handler is null || !_resolvers.TryGetValue(handler.Entity, out var resolver)) { - await FinishAsync(operation.Id, BulkOperationStatus.Failed, $"Unknown bulk operation type '{operation.Type}'."); + await FinishAsync(task.Id, BackgroundTaskStatus.Failed, $"Unknown bulk operation type '{task.Type}'."); return; } - if (await _service.IsCancelRequestedAsync(operation.Id)) + if (await _taskService.IsCancelRequestedAsync(task.Id)) { - await FinishAsync(operation.Id, BulkOperationStatus.Cancelled, "Cancelled by user."); + await FinishAsync(task.Id, BackgroundTaskStatus.Cancelled, "Cancelled by user."); return; } string? pinnedSelection = null; - if (operation.Selection.Kind is BulkOperationSelectionKind.Filter) + if (task.Bulk!.Selection.Kind is BulkOperationSelectionKind.Filter) { - pinnedSelection = await _service.GetPinnedSelectionAsync(operation.Id); + pinnedSelection = await _service.GetPinnedSelectionAsync(task.Id); if (pinnedSelection is null) { - await FinishAsync(operation.Id, BulkOperationStatus.Failed, "The saved selection expired, so the operation could not continue."); + await FinishAsync(task.Id, BackgroundTaskStatus.Failed, "The saved selection expired, so the operation could not continue."); return; } } - operation = await _service.UpdateAsync(operation.Id, current => + task = await _taskService.UpdateAsync(task.Id, current => { if (!current.IsActive) return false; - current.Status = BulkOperationStatus.Running; - current.StartedUtc ??= _timeProvider.GetUtcNow().UtcDateTime; + current.Start(UtcNow); return true; }); - if (operation is null || operation.Status is not BulkOperationStatus.Running) + if (task is null || task.Status is not BackgroundTaskStatus.Running) return; - Log.LogInformation("Running bulk operation {OperationId} {Type} from cursor {Cursor}", operation.Id, operation.Type, operation.Cursor); - await context.ReportProgressAsync(GetPercent(operation), operation.Description); + Log.LogInformation("Running bulk operation {TaskId} {Type} from cursor {Cursor}", task.Id, task.Type, task.Bulk!.Cursor); + await context.ReportProgressAsync(GetPercent(task), task.Description); try { @@ -119,27 +122,27 @@ public override async Task HandleItemAsync(WorkItemContext context) { if (context.CancellationToken.IsCancellationRequested) { - // The host is shutting down. Leave the operation running so the queue retry resumes it from the cursor. - context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {operation.Id} for shutdown"); + // The host is shutting down. Leave the task running so the queue retry resumes it from the cursor. + context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {task.Id} for shutdown"); return; } - if (await _service.IsCancelRequestedAsync(operation.Id)) + if (await _taskService.IsCancelRequestedAsync(task.Id)) { - await FinishAsync(operation.Id, BulkOperationStatus.Cancelled, $"Cancelled by user after {operation.Processed:N0} of {operation.Total:N0}."); + await FinishAsync(task.Id, BackgroundTaskStatus.Cancelled, $"Cancelled by user after {task.Processed:N0} of {task.Total:N0}."); return; } - var page = await GetPageAsync(operation, resolver, pinnedSelection, context.CancellationToken); + var page = await GetPageAsync(task, resolver, pinnedSelection, context.CancellationToken); if (page.Ids.Count == 0) break; - // The user who started the operation is authorized once, when it is created. Each batch re-checks the scope + // The user who started the task is authorized once, when it is created. Each batch re-checks the scope // (organization and project) but not the user: someone removed from the organization mid-run still finishes // the deletion they legitimately started, and any member of the organization can cancel it. - var result = await handler.ExecuteBatchAsync(operation, page.Ids, context.CancellationToken); + var result = await handler.ExecuteBatchAsync(task, page.Ids, context.CancellationToken); - operation = await _service.UpdateAsync(operation.Id, current => + task = await _taskService.UpdateAsync(task.Id, current => { if (!current.IsActive) return false; @@ -147,73 +150,74 @@ public override async Task HandleItemAsync(WorkItemContext context) current.Processed += page.Ids.Count; current.Succeeded += result.Succeeded; current.Failed += result.Failed; - current.Cursor = page.NextCursor ?? ExhaustedCursor; - current.RetryCount = 0; + current.Bulk!.Cursor = page.NextCursor ?? ExhaustedCursor; + current.Bulk!.RetryCount = 0; current.Total = Math.Max(current.Total, current.Processed); current.Message = result.Message; + current.Progress = GetPercent(current); return true; }); - if (operation is null || !operation.IsActive) + if (task is null || !task.IsActive) return; - await context.ReportProgressAsync(GetPercent(operation), $"{operation.Processed:N0} of {operation.Total:N0}"); + await context.ReportProgressAsync(GetPercent(task), $"{task.Processed:N0} of {task.Total:N0}"); if (page.NextCursor is null) break; } - await _service.UpdateAsync(operation.Id, current => + await _taskService.UpdateAsync(task.Id, current => { if (!current.IsActive) return false; current.Total = current.Processed; - _service.Finish(current, BulkOperationStatus.Completed, current.Failed > 0 ? $"Completed with {current.Failed:N0} failures." : "Completed."); + current.Finish(BackgroundTaskStatus.Completed, current.Failed > 0 ? $"Completed with {current.Failed:N0} failures." : "Completed.", UtcNow); return true; }); } catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested) { - context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {operation?.Id} for shutdown"); + context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {task?.Id} for shutdown"); } catch (Exception ex) when (!IsPermanent(ex)) { // Elasticsearch, cache and network hiccups: continue in a new work item from the saved counts and cursor. Rethrowing - // would use the queue's retries, which cover the whole work item, so an operation that makes progress between errors + // would use the queue's retries, which cover the whole work item, so an task that makes progress between errors // would run out of deliveries while still marked running. After MaxTransientFailures in a row, give up. - var failed = await _service.UpdateAsync(workItem.OperationId, current => + var failed = await _taskService.UpdateAsync(workItem.TaskId, current => { if (!current.IsActive) return false; - current.RetryCount++; - current.Message = current.RetryCount >= MaxTransientFailures ? "The operation stopped because of repeated errors." : "Retrying after a temporary error."; - if (current.RetryCount >= MaxTransientFailures) - _service.Finish(current, BulkOperationStatus.Failed, current.Message); + current.Bulk!.RetryCount++; + current.Message = current.Bulk!.RetryCount >= MaxTransientFailures ? "The operation stopped because of repeated errors." : "Retrying after a temporary error."; + if (current.Bulk!.RetryCount >= MaxTransientFailures) + current.Finish(BackgroundTaskStatus.Failed, current.Message, UtcNow); return true; }); if (failed is { IsActive: false }) { - Log.LogError(ex, "Bulk operation {OperationId} failed after repeated errors: {Message}", workItem.OperationId, ex.Message); + Log.LogError(ex, "Bulk operation {TaskId} failed after repeated errors: {Message}", workItem.TaskId, ex.Message); return; } - Log.LogWarning(ex, "Bulk operation {OperationId} hit a temporary error and will continue in a new run: {Message}", workItem.OperationId, ex.Message); + Log.LogWarning(ex, "Bulk operation {TaskId} hit a temporary error and will continue in a new run: {Message}", workItem.TaskId, ex.Message); if (failed is null) return; if (_options.RetryDelay > TimeSpan.Zero) - await Task.Delay(_options.RetryDelay * failed.RetryCount, _timeProvider, context.CancellationToken); + await Task.Delay(_options.RetryDelay * failed.Bulk!.RetryCount, _timeProvider, context.CancellationToken); await _service.ContinueAsync(workItem); } catch (Exception ex) { - Log.LogError(ex, "Bulk operation {OperationId} failed: {Message}", workItem.OperationId, ex.Message); - await FinishAsync(workItem.OperationId, BulkOperationStatus.Failed, "The operation stopped because of an unexpected error."); + Log.LogError(ex, "Bulk operation {TaskId} failed: {Message}", workItem.TaskId, ex.Message); + await FinishAsync(workItem.TaskId, BackgroundTaskStatus.Failed, "The operation stopped because of an unexpected error."); } } @@ -232,37 +236,39 @@ internal static bool IsPermanent(Exception exception) return false; } - private async Task GetPageAsync(BulkOperation operation, IBulkSelectionResolver resolver, string? pinnedSelection, CancellationToken cancellationToken) + private async Task GetPageAsync(BackgroundTask task, IBulkSelectionResolver resolver, string? pinnedSelection, CancellationToken cancellationToken) { int size = Math.Max(1, _options.BatchSize); - if (operation.Cursor == ExhaustedCursor) + if (task.Bulk!.Cursor == ExhaustedCursor) return new BulkSelectionPage([], null); - if (operation.Selection.Kind is not BulkOperationSelectionKind.Ids) - return await resolver.GetPageAsync(operation, pinnedSelection, operation.Cursor, size, cancellationToken); + if (task.Bulk!.Selection.Kind is not BulkOperationSelectionKind.Ids) + return await resolver.GetPageAsync(task, pinnedSelection, task.Bulk!.Cursor, size, cancellationToken); // The cursor of an id selection is the index of the next id to process. - var ids = operation.Selection.Ids ?? []; - int start = Int32.TryParse(operation.Cursor, out int parsed) ? parsed : 0; + var ids = task.Bulk!.Selection.Ids ?? []; + int start = Int32.TryParse(task.Bulk!.Cursor, out int parsed) ? parsed : 0; var batch = ids.Skip(start).Take(size).ToList(); int next = start + batch.Count; return new BulkSelectionPage(batch, next < ids.Count ? next.ToString() : null); } - private Task FinishAsync(string operationId, BulkOperationStatus status, string message) + private DateTime UtcNow => _timeProvider.GetUtcNow().UtcDateTime; + + private Task FinishAsync(string taskId, BackgroundTaskStatus status, string message) { - return _service.UpdateAsync(operationId, current => + return _taskService.UpdateAsync(taskId, current => { if (!current.IsActive) return false; - _service.Finish(current, status, message); + current.Finish(status, message, UtcNow); return true; }); } - private static int GetPercent(BulkOperation operation) + private static int GetPercent(BackgroundTask task) { - return operation.Total <= 0 ? 0 : (int)Math.Min(99, operation.Processed * 100 / operation.Total); + return task.Total <= 0 ? 0 : (int)Math.Min(99, task.Processed * 100 / task.Total); } } diff --git a/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs new file mode 100644 index 0000000000..504febcffb --- /dev/null +++ b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTask.cs @@ -0,0 +1,76 @@ +using System.Text.Json.Serialization; +using Exceptionless.Core.Models.BulkOperations; +using Foundatio.Repositories.Models; + +namespace Exceptionless.Core.Models.BackgroundTasks; + +/// +/// Long-running work that a user can follow in the progress panel: a bulk action, a project data reset, sample data +/// generation. Stored in the cache with a limited lifetime and indexed per organization; it is intentionally not +/// persisted in Elasticsearch. Everything a bulk batch changes (counts and cursor) lives in this one record so a +/// batch is saved atomically. +/// +public class BackgroundTask : IIdentity, IOwnedByOrganization, IHaveDates +{ + public string Id { get; set; } = null!; + public string OrganizationId { get; set; } = null!; + public string? ProjectId { get; set; } + + /// What the task does, e.g. events.delete or projects.reset-data. + public string Type { get; set; } = null!; + + public string Description { get; set; } = null!; + public BackgroundTaskStatus Status { get; set; } = BackgroundTaskStatus.Queued; + + /// Percent complete, 0 to 100. + public int Progress { get; set; } + + /// Item counts. All zero for tasks that only report a percent. + public long Total { get; set; } + public long Processed { get; set; } + public long Succeeded { get; set; } + public long Failed { get; set; } + + public string? Message { get; set; } + + /// Whether the user can stop the task. Tasks that cannot be stopped midway are not cancellable. + public bool CanCancel { get; set; } + + public string CreatedByUserId { get; set; } = null!; + public DateTime CreatedUtc { get; set; } + public DateTime? StartedUtc { get; set; } + public DateTime? CompletedUtc { get; set; } + public DateTime UpdatedUtc { get; set; } + + /// Only set for bulk actions: what they apply to and where they resume. + public BulkOperationDetails? Bulk { get; set; } + + [JsonIgnore] + public bool IsActive => Status is BackgroundTaskStatus.Queued or BackgroundTaskStatus.Running; + + /// Marks the task as running. The start time is only recorded the first time. + public void Start(DateTime utcNow) + { + Status = BackgroundTaskStatus.Running; + StartedUtc ??= utcNow; + } + + public void Finish(BackgroundTaskStatus status, string? message, DateTime utcNow) + { + Status = status; + Message = message; + CompletedUtc = utcNow; + if (status is BackgroundTaskStatus.Completed) + Progress = 100; + } +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum BackgroundTaskStatus +{ + Queued, + Running, + Completed, + Failed, + Cancelled +} diff --git a/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs new file mode 100644 index 0000000000..ff51c9a86b --- /dev/null +++ b/src/Exceptionless.Core/Models/BackgroundTasks/BackgroundTaskTypes.cs @@ -0,0 +1,8 @@ +namespace Exceptionless.Core.Models.BackgroundTasks; + +/// The types of tasks that are not bulk actions. Bulk actions use the key of their handler, e.g. events.delete. +public static class BackgroundTaskTypes +{ + public const string ResetProjectData = "projects.reset-data"; + public const string GenerateProjectSampleData = "projects.sample-data"; +} diff --git a/src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs b/src/Exceptionless.Core/Models/BulkOperations/BulkOperationSelection.cs similarity index 53% rename from src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs rename to src/Exceptionless.Core/Models/BulkOperations/BulkOperationSelection.cs index e605fb7076..1bfec8a567 100644 --- a/src/Exceptionless.Core/Models/BulkOperations/BulkOperation.cs +++ b/src/Exceptionless.Core/Models/BulkOperations/BulkOperationSelection.cs @@ -1,57 +1,18 @@ using System.Text.Json.Serialization; -using Foundatio.Repositories.Models; namespace Exceptionless.Core.Models.BulkOperations; -/// -/// A long-running action applied to every item that matches a selection. Stored in the cache with a limited -/// lifetime and indexed per organization; it is intentionally not persisted in Elasticsearch. -/// -public class BulkOperation : IIdentity, IOwnedByOrganization +/// The part of a that only bulk actions have. +public class BulkOperationDetails { - public string Id { get; set; } = null!; - public string OrganizationId { get; set; } = null!; - public string? ProjectId { get; set; } - - /// The handler key, e.g. events.delete. - public string Type { get; set; } = null!; - - public string Description { get; set; } = null!; public BulkOperationSelection Selection { get; set; } = null!; public Dictionary Parameters { get; set; } = new(); - public BulkOperationStatus Status { get; set; } = BulkOperationStatus.Queued; - - public long Total { get; set; } - public long Processed { get; set; } - public long Succeeded { get; set; } - public long Failed { get; set; } - - public string CreatedByUserId { get; set; } = null!; - public DateTime CreatedUtc { get; set; } - public DateTime? StartedUtc { get; set; } - public DateTime? CompletedUtc { get; set; } - public DateTime UpdatedUtc { get; set; } - - public string? Message { get; set; } /// Consecutive batches that failed with a temporary error. Reset by the next successful batch. public int RetryCount { get; set; } /// Opaque position of the last batch that finished; the job resumes after it. public string? Cursor { get; set; } - - [JsonIgnore] - public bool IsActive => Status is BulkOperationStatus.Queued or BulkOperationStatus.Running; -} - -[JsonConverter(typeof(JsonStringEnumConverter))] -public enum BulkOperationStatus -{ - Queued, - Running, - Completed, - Failed, - Cancelled } /// The kind of item an operation targets. Each entity has one selection resolver. diff --git a/src/Exceptionless.Core/Models/Messaging/BulkOperationUpdated.cs b/src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs similarity index 65% rename from src/Exceptionless.Core/Models/Messaging/BulkOperationUpdated.cs rename to src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs index 27d7c68f3f..d16f69c23e 100644 --- a/src/Exceptionless.Core/Models/Messaging/BulkOperationUpdated.cs +++ b/src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs @@ -1,23 +1,25 @@ -using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.BackgroundTasks; namespace Exceptionless.Core.Messaging.Models; /// -/// Published after every batch and every status change of a bulk operation. Intentionally independent of the +/// Published after every progress report and every status change of a background task. Intentionally independent of the /// transport that delivers it to clients, and free of the selection and parameters. /// -public record BulkOperationUpdated +public record BackgroundTaskUpdated { public required string Id { get; set; } public required string OrganizationId { get; set; } public string? ProjectId { get; set; } public required string Type { get; set; } public required string Description { get; set; } - public BulkOperationStatus Status { get; set; } + public BackgroundTaskStatus Status { get; set; } + public int Progress { get; set; } public long Total { get; set; } public long Processed { get; set; } public long Succeeded { get; set; } public long Failed { get; set; } public string? Message { get; set; } + public bool CanCancel { get; set; } public DateTime UpdatedUtc { get; set; } } diff --git a/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs index 9b13adc425..523bd67151 100644 --- a/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs +++ b/src/Exceptionless.Core/Models/WorkItems/BulkOperationWorkItem.cs @@ -1,11 +1,11 @@ namespace Exceptionless.Core.Models.WorkItems; -public record BulkOperationWorkItem +public record BulkOperationWorkItem : IBackgroundTaskWorkItem { - public required string OperationId { get; init; } + public required string TaskId { get; init; } /// - /// The organization, type and description are carried so a final state can still be published if the operation + /// The organization, type and description are carried so a final state can still be published if the task /// record is gone when the job runs. /// public required string OrganizationId { get; init; } diff --git a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs index 4bf9a8db67..ecc627b735 100644 --- a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs +++ b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs @@ -1,9 +1,10 @@ namespace Exceptionless.Core.Models.WorkItems; -public record GenerateSampleEventsWorkItem +public record GenerateSampleEventsWorkItem : IBackgroundTaskWorkItem { public string? OrganizationId { get; init; } public string? ProjectId { get; init; } public int EventCount { get; init; } = 250; public int DaysBack { get; init; } = 7; + public string? TaskId { get; init; } } diff --git a/src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs new file mode 100644 index 0000000000..3c3f7d499d --- /dev/null +++ b/src/Exceptionless.Core/Models/WorkItems/IBackgroundTaskWorkItem.cs @@ -0,0 +1,10 @@ +namespace Exceptionless.Core.Models.WorkItems; + +/// +/// A work item whose progress is tracked by a background task, so the user can follow it in the progress panel. +/// +public interface IBackgroundTaskWorkItem +{ + /// The tracking task, or null when the work item was queued without one. + string? TaskId { get; } +} diff --git a/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs index 4d4410c6c0..929a05b55b 100644 --- a/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs +++ b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs @@ -1,7 +1,8 @@ namespace Exceptionless.Core.Models.WorkItems; -public record ResetProjectDataWorkItem +public record ResetProjectDataWorkItem : IBackgroundTaskWorkItem { public required string OrganizationId { get; init; } public required string ProjectId { get; init; } + public string? TaskId { get; init; } } diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs new file mode 100644 index 0000000000..5af12c2c16 --- /dev/null +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -0,0 +1,322 @@ +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Repositories.Utility; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Services.BackgroundTasks; + +public class BackgroundTaskOptions +{ + /// How long one attempt to lock a task waits. A busy lock is retried by the resilience policy. + public TimeSpan LockAcquireTimeout { get; set; } = TimeSpan.FromSeconds(5); +} + +/// +/// Stores background tasks in the cache (with a limited lifetime), keeps a per-organization index and an index of the +/// active tasks, and publishes every change so the progress panel can follow along. +/// +public class BackgroundTaskService +{ + public static readonly TimeSpan TaskLifetime = TimeSpan.FromDays(7); + + /// An active task that has not reported progress for this long is considered abandoned. + public static readonly TimeSpan StaleAfter = TimeSpan.FromHours(2); + + /// How many tasks the list shows, newest first. + public const int ListLimit = 50; + + /// Every cache key and lock resource of background tasks starts with this prefix. + public const string KeyPrefix = "background-tasks"; + + private const int IndexPageSize = 500; + private const int MaxIndexPages = 100; + private readonly ICacheClient _cache; + private readonly ILockProvider _lockProvider; + private readonly IMessagePublisher _publisher; + private readonly IQueue _workItemQueue; + private readonly IResiliencePolicy _lockPolicy; + private readonly BackgroundTaskOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public BackgroundTaskService(ICacheClient cache, ILockProvider lockProvider, IMessagePublisher publisher, IQueue workItemQueue, + IResiliencePolicyProvider resiliencePolicyProvider, BackgroundTaskOptions options, TimeProvider timeProvider, ILogger logger) + { + _cache = cache; + _lockProvider = lockProvider; + _publisher = publisher; + _workItemQueue = workItemQueue; + _options = options; + _timeProvider = timeProvider; + _logger = logger; + + // Several workers and web instances update the same record, so a busy lock is normal: wait a little, then try again. + _lockPolicy = resiliencePolicyProvider.GetPolicy(b => b.WithMaxAttempts(5).WithLinearDelay(TimeSpan.FromMilliseconds(250)), logger, timeProvider); + } + + public static string GetTaskCacheKey(string id) => $"{KeyPrefix}:{id}"; + public static string GetCancelCacheKey(string id) => $"{KeyPrefix}:{id}:cancel"; + public static string GetOrganizationIndexCacheKey(string organizationId) => $"{KeyPrefix}:organization:{organizationId}"; + public static string GetActiveIndexCacheKey(string organizationId) => $"{KeyPrefix}:organization:{organizationId}:active"; + public static string GetUpdateLockResource(string id) => $"{KeyPrefix}:update:{id}"; + public static string GetCreateLockResource(string organizationId) => $"{KeyPrefix}:create:{organizationId}"; + + public async Task GetAsync(string id) + { + if (String.IsNullOrEmpty(id)) + return null; + + var cached = await _cache.GetAsync(GetTaskCacheKey(id)); + return cached.HasValue ? cached.Value : null; + } + + /// + /// The newest tasks of the organization that are still stored, newest first. Every indexed id is read so a busy + /// organization cannot push recent tasks out of the list; only the history that is shown is limited. + /// + public async Task> GetByOrganizationAsync(string organizationId, int limit = ListLimit) + { + // Ids are ObjectIds, so descending order is newest first no matter how the cache orders its list. + var newestIds = (await GetIndexedIdsAsync(GetOrganizationIndexCacheKey(organizationId))) + .Distinct(StringComparer.Ordinal) + .OrderByDescending(id => id, StringComparer.Ordinal) + .Take(limit); + + var tasks = new List(); + var expired = new List(); + foreach (string id in newestIds) + { + var task = await GetAsync(id); + if (task is null) + expired.Add(id); + else if (String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal)) + tasks.Add(task); + } + + if (expired.Count > 0) + await _cache.ListRemoveAsync(GetOrganizationIndexCacheKey(organizationId), expired); + + return tasks.OrderByDescending(t => t.CreatedUtc).ThenByDescending(t => t.Id, StringComparer.Ordinal).ToList(); + } + + /// The ids of the organization's tasks that have not finished. + public async Task> GetActiveIdsAsync(string organizationId) + { + return (await GetIndexedIdsAsync(GetActiveIndexCacheKey(organizationId))).Distinct(StringComparer.Ordinal).ToList(); + } + + /// Puts a task that is running again (for example after a queue retry) back into the active index. + public Task AddToActiveIndexAsync(string organizationId, string id) + { + return _cache.ListAddAsync(GetActiveIndexCacheKey(organizationId), [id], TaskLifetime); + } + + /// Removes finished (or expired) ids from the active index of the organization. + public Task RemoveFromActiveIndexAsync(string organizationId, IReadOnlyCollection ids) + { + return _cache.ListRemoveAsync(GetActiveIndexCacheKey(organizationId), ids); + } + + /// + /// Gives the task an id, stores it as queued and adds it to the indexes. Nothing is published or queued yet. + /// + public async Task AddAsync(BackgroundTask task) + { + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + task.Id = ObjectId.GenerateNewId(utcNow).ToString(); + task.Status = BackgroundTaskStatus.Queued; + task.CreatedUtc = utcNow; + task.UpdatedUtc = utcNow; + + await SaveAsync(task); + await _cache.ListAddAsync(GetOrganizationIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime); + await _cache.ListAddAsync(GetActiveIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime); + } + + /// + /// Adds a task and queues the work item that runs it. The work item is created after the task has its id. + /// + public async Task CreateAndEnqueueAsync(BackgroundTask task, Func createWorkItem) where T : class + { + await AddAsync(task); + await EnqueueAsync(task, createWorkItem(task)); + return task; + } + + /// + /// Queues the work item of a task that was added and publishes it. When queueing fails the task is failed, because + /// nothing would ever run it, and the error is rethrown. + /// + public async Task EnqueueAsync(BackgroundTask task, T workItem) where T : class + { + try + { + await _workItemQueue.EnqueueAsync(workItem); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to queue background task {TaskId}: {Message}", task.Id, ex.Message); + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + task.Finish(BackgroundTaskStatus.Failed, "The task could not be started. Please try again.", utcNow); + task.UpdatedUtc = utcNow; + await SaveAsync(task); + await RemoveFromActiveIndexAsync(task.OrganizationId, [task.Id]); + await PublishAsync(task); + throw; + } + + await PublishAsync(task); + } + + /// + /// Queues another run of a work item, e.g. to continue a long task after a temporary error. The queue's retries cover a + /// whole work item, so continuing in a new work item does not use up the retries of the current one. + /// + public Task ContinueAsync(T workItem) where T : class + { + return _workItemQueue.EnqueueAsync(workItem); + } + + /// + /// Loads the task, lets change it, then saves and publishes it. The change is skipped when + /// returns false. Returns the stored task, or null when it no longer exists. + /// + /// + /// Tasks live in the cache rather than in Elasticsearch, so the repository's PatchAsync does not apply. The lock makes the + /// load, change and save one step; a busy lock is retried with the resilience policy before giving up. + /// + public async Task UpdateAsync(string id, Func update) + { + await using var updateLock = await _lockPolicy.ExecuteAsync(async ct => + await _lockProvider.AcquireAsync(GetUpdateLockResource(id), TimeSpan.FromSeconds(30), _options.LockAcquireTimeout), CancellationToken.None); + + var task = await GetAsync(id); + if (task is null) + return null; + + if (!update(task)) + return task; + + task.UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime; + await SaveAsync(task); + if (!task.IsActive) + await RemoveFromActiveIndexAsync(task.OrganizationId, [task.Id]); + + await PublishAsync(task); + return task; + } + + /// + /// Requests cancellation. A queued (or abandoned) task is cancelled right away; a running bulk task stops at the next + /// batch boundary, after the batch that is in flight finishes. Tasks that cannot be stopped are rejected while they are + /// alive, but an abandoned one can always be cleared. + /// + public async Task CancelAsync(string id) + { + var task = await GetAsync(id); + if (task is null) + return BackgroundTaskCancelResult.NotFound; + + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + if (!task.IsActive) + return BackgroundTaskCancelResult.Cancelled(task); + + bool isAbandoned = utcNow - task.UpdatedUtc >= StaleAfter; + if (!task.CanCancel && !isAbandoned) + return BackgroundTaskCancelResult.NotCancellable(task); + + await _cache.SetAsync(GetCancelCacheKey(id), true, TaskLifetime); + + var updated = await UpdateAsync(id, current => + { + if (!current.IsActive) + return false; + + bool isRunningNow = current.Status is BackgroundTaskStatus.Running && utcNow - current.UpdatedUtc < StaleAfter; + if (isRunningNow) + return false; + + current.Finish(BackgroundTaskStatus.Cancelled, "Cancelled before it started.", utcNow); + return true; + }); + + return updated is null ? BackgroundTaskCancelResult.NotFound : BackgroundTaskCancelResult.Cancelled(updated); + } + + public async Task IsCancelRequestedAsync(string id) + { + return (await _cache.GetAsync(GetCancelCacheKey(id))).HasValue; + } + + public Task SaveAsync(BackgroundTask task) => _cache.SetAsync(GetTaskCacheKey(task.Id), task, TaskLifetime); + + public Task PublishAsync(BackgroundTask task) + { + return PublishAsync(new BackgroundTaskUpdated + { + Id = task.Id, + OrganizationId = task.OrganizationId, + ProjectId = task.ProjectId, + Type = task.Type, + Description = task.Description, + Status = task.Status, + Progress = task.Progress, + Total = task.Total, + Processed = task.Processed, + Succeeded = task.Succeeded, + Failed = task.Failed, + Message = task.Message, + CanCancel = task.CanCancel, + UpdatedUtc = task.UpdatedUtc + }); + } + + public async Task PublishAsync(BackgroundTaskUpdated message) + { + try + { + await _publisher.PublishAsync(message); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to publish background task update for {TaskId}: {Message}", message.Id, ex.Message); + } + } + + private async Task> GetIndexedIdsAsync(string key) + { + var ids = new List(); + for (int page = 1; page <= MaxIndexPages; page++) + { + var batch = await _cache.GetListAsync(key, page, IndexPageSize); + if (!batch.HasValue || batch.Value.Count == 0) + break; + + ids.AddRange(batch.Value); + if (batch.Value.Count < IndexPageSize) + break; + } + + return ids; + } +} + +public record BackgroundTaskCancelResult +{ + public BackgroundTask? Task { get; init; } + public bool IsNotCancellable { get; init; } + + public static readonly BackgroundTaskCancelResult NotFound = new(); + public static BackgroundTaskCancelResult Cancelled(BackgroundTask task) => new() { Task = task }; + public static BackgroundTaskCancelResult NotCancellable(BackgroundTask task) => new() { Task = task, IsNotCancellable = true }; +} diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index e94146cc82..58745b696c 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -1,12 +1,10 @@ using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; using Foundatio.Caching; -using Foundatio.Jobs; using Foundatio.Lock; -using Foundatio.Messaging; -using Foundatio.Queues; -using Foundatio.Repositories.Utility; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Services.BulkOperations; @@ -21,251 +19,133 @@ public class BulkOperationOptions } /// -/// Stores bulk operations in the cache (with a limited lifetime), keeps a per-organization index of operation ids, -/// enforces the one-active-destructive-operation-per-project rule, and publishes every change. +/// The bulk specific part of background tasks: the registry of handlers and selection resolvers, counting and pinning a +/// selection, and the one-active-destructive-task-per-project rule. Storage, indexes and notifications are +/// . /// public class BulkOperationService { - public static readonly TimeSpan OperationLifetime = TimeSpan.FromDays(7); - - /// An active operation that has not reported progress for this long is considered abandoned. - public static readonly TimeSpan StaleAfter = TimeSpan.FromHours(2); - - /// How many operations the list shows, newest first. - public const int ListLimit = 50; - - private const int IndexPageSize = 500; - private const int MaxIndexPages = 100; + private readonly BackgroundTaskService _taskService; private readonly ICacheClient _cache; private readonly ILockProvider _lockProvider; - private readonly IMessagePublisher _publisher; - private readonly IQueue _workItemQueue; private readonly IReadOnlyDictionary _handlers; private readonly IReadOnlyDictionary _resolvers; private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - public BulkOperationService(ICacheClient cache, ILockProvider lockProvider, IMessagePublisher publisher, IQueue workItemQueue, - IEnumerable handlers, IEnumerable resolvers, TimeProvider timeProvider, ILogger logger) + public BulkOperationService(BackgroundTaskService taskService, ICacheClient cache, ILockProvider lockProvider, + IEnumerable handlers, IEnumerable resolvers, TimeProvider timeProvider) { + _taskService = taskService; _cache = cache; _lockProvider = lockProvider; - _publisher = publisher; - _workItemQueue = workItemQueue; _handlers = handlers.ToDictionary(h => h.Type, StringComparer.Ordinal); _resolvers = resolvers.ToDictionary(r => r.Entity); _timeProvider = timeProvider; - _logger = logger; } + public static string GetPinnedSelectionCacheKey(string id) => $"{BackgroundTaskService.KeyPrefix}:{id}:selection"; + public IBulkOperationHandler? GetHandler(string type) => _handlers.GetValueOrDefault(type); /// - /// Queues another run that continues from the saved cursor. The queue's retries cover a whole work item, so a long operation + /// Queues another run that continues from the saved cursor. The queue's retries cover a whole work item, so a long task /// continues in a new work item after a temporary error instead of using up the retries of the current one. /// - public Task ContinueAsync(BulkOperationWorkItem workItem) - { - return _workItemQueue.EnqueueAsync(workItem); - } - - public async Task GetAsync(string id) - { - if (String.IsNullOrEmpty(id)) - return null; - - var cached = await _cache.GetAsync(GetOperationKey(id)); - return cached.HasValue ? cached.Value : null; - } - - /// - /// The newest operations of the organization that are still stored, newest first. Every indexed id is read so a busy - /// organization cannot push recent operations out of the list; only the history that is shown is limited. - /// - public async Task> GetByOrganizationAsync(string organizationId, int limit = ListLimit) - { - // Ids are ObjectIds, so descending order is newest first no matter how the cache orders its list. - var newestIds = (await GetIndexedIdsAsync(GetIndexKey(organizationId))) - .Distinct(StringComparer.Ordinal) - .OrderByDescending(id => id, StringComparer.Ordinal) - .Take(limit); - - var operations = new List(); - var expired = new List(); - foreach (string id in newestIds) - { - var operation = await GetAsync(id); - if (operation is null) - expired.Add(id); - else if (String.Equals(operation.OrganizationId, organizationId, StringComparison.Ordinal)) - operations.Add(operation); - } - - if (expired.Count > 0) - await _cache.ListRemoveAsync(GetIndexKey(organizationId), expired); - - return operations.OrderByDescending(o => o.CreatedUtc).ThenByDescending(o => o.Id, StringComparer.Ordinal).ToList(); - } + public Task ContinueAsync(BulkOperationWorkItem workItem) => _taskService.ContinueAsync(workItem); public async Task GetPinnedSelectionAsync(string id) { - var cached = await _cache.GetAsync(GetPinnedKey(id)); + var cached = await _cache.GetAsync(GetPinnedSelectionCacheKey(id)); return cached.HasValue ? cached.Value : null; } /// - /// Stores and enqueues the operation unless it conflicts with an active destructive operation on the same project. + /// Stores and enqueues the task unless it conflicts with an active destructive task on the same project. /// The check and the write happen under a per-organization lock so two requests cannot both pass it. /// - public async Task CreateAsync(BulkOperation operation) + public async Task CreateAsync(BackgroundTask task) { - var handler = GetHandler(operation.Type) ?? throw new ArgumentException($"Unknown bulk operation type '{operation.Type}'.", nameof(operation)); + var handler = GetHandler(task.Type) ?? throw new ArgumentException($"Unknown bulk operation type '{task.Type}'.", nameof(task)); + var bulk = task.Bulk ?? throw new ArgumentException("A bulk task needs its bulk details.", nameof(task)); - var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - operation.Id = ObjectId.GenerateNewId(utcNow).ToString(); - operation.Status = BulkOperationStatus.Queued; - operation.CreatedUtc = utcNow; - operation.UpdatedUtc = utcNow; + task.CanCancel = true; // Count and pin the selection before taking the lock, so a slow search never holds up other requests. string? pinnedSelection = null; - if (operation.Selection.Kind is BulkOperationSelectionKind.Filter) + if (bulk.Selection.Kind is BulkOperationSelectionKind.Filter) { if (!_resolvers.TryGetValue(handler.Entity, out var resolver)) throw new InvalidOperationException($"No selection resolver is registered for {handler.Entity}."); - var preparation = await resolver.PrepareAsync(operation, CancellationToken.None); + task.CreatedUtc = _timeProvider.GetUtcNow().UtcDateTime; + var preparation = await resolver.PrepareAsync(task, CancellationToken.None); if (!preparation.Check.IsOk) return BulkOperationCreateResult.Rejected(preparation.Check); - operation.Total = preparation.Total; + task.Total = preparation.Total; pinnedSelection = preparation.PinnedState; } else { - operation.Total = operation.Selection.Ids?.Count ?? 0; + task.Total = bulk.Selection.Ids?.Count ?? 0; } - operation.Description = handler.Describe(operation); + task.Description = handler.Describe(task); - await using var createLock = await _lockProvider.TryAcquireAsync($"bulk-operations:create:{operation.OrganizationId}", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); + await using var createLock = await _lockProvider.TryAcquireAsync(BackgroundTaskService.GetCreateLockResource(task.OrganizationId), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); if (createLock is null) return BulkOperationCreateResult.Busy(); if (handler.IsDestructive) { - var conflict = await FindConflictingOperationAsync(operation); + var conflict = await FindConflictingTaskAsync(task); if (conflict is not null) return BulkOperationCreateResult.Conflicting(conflict); } - await SaveAsync(operation); + await _taskService.AddAsync(task); if (pinnedSelection is not null) - await _cache.SetAsync(GetPinnedKey(operation.Id), pinnedSelection, OperationLifetime); + await _cache.SetAsync(GetPinnedSelectionCacheKey(task.Id), pinnedSelection, BackgroundTaskService.TaskLifetime); - await _cache.ListAddAsync(GetIndexKey(operation.OrganizationId), [operation.Id], OperationLifetime); - await _cache.ListAddAsync(GetActiveIndexKey(operation.OrganizationId), [operation.Id], OperationLifetime); - try - { - await _workItemQueue.EnqueueAsync(new BulkOperationWorkItem - { - Description = operation.Description, - OperationId = operation.Id, - OrganizationId = operation.OrganizationId, - Type = operation.Type - }); - } - catch (Exception ex) + await _taskService.EnqueueAsync(task, new BulkOperationWorkItem { - // Nothing would ever run an operation that was not queued, and it would block new operations on the project. - _logger.LogError(ex, "Unable to queue bulk operation {OperationId}: {Message}", operation.Id, ex.Message); - Finish(operation, BulkOperationStatus.Failed, "The operation could not be started. Please try again."); - await SaveAsync(operation); - await _cache.ListRemoveAsync(GetActiveIndexKey(operation.OrganizationId), [operation.Id]); - await PublishAsync(operation); - throw; - } - - await PublishAsync(operation); - - return BulkOperationCreateResult.Created(operation); - } - - /// - /// Loads the operation, lets change it, then saves and publishes it. The change is skipped - /// when returns false. Returns the stored operation, or null when it no longer exists. - /// - public async Task UpdateAsync(string id, Func update) - { - await using var updateLock = await _lockProvider.TryAcquireAsync($"bulk-operations:update:{id}", TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); - if (updateLock is null) - throw new TimeoutException($"Unable to lock bulk operation {id} for update."); - - var operation = await GetAsync(id); - if (operation is null) - return null; - - if (!update(operation)) - return operation; - - operation.UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime; - await SaveAsync(operation); - if (!operation.IsActive) - await _cache.ListRemoveAsync(GetActiveIndexKey(operation.OrganizationId), [operation.Id]); + Description = task.Description, + TaskId = task.Id, + OrganizationId = task.OrganizationId, + Type = task.Type + }); - await PublishAsync(operation); - return operation; + return BulkOperationCreateResult.Created(task); } /// - /// Requests cancellation. A queued (or abandoned) operation is cancelled right away; a running one stops at the - /// next batch boundary, after the batch that is in flight finishes. + /// Tells clients that a task can no longer run because its record is gone (for example the cache was cleared). + /// The work item carries enough to publish a final state so the progress panel does not wait forever. /// - public async Task CancelAsync(string id) + public Task PublishLostAsync(BulkOperationWorkItem workItem) { - var operation = await GetAsync(id); - if (operation is null || !operation.IsActive) - return operation; - - await _cache.SetAsync(GetCancelKey(id), true, OperationLifetime); - - var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - return await UpdateAsync(id, current => + return _taskService.PublishAsync(new BackgroundTaskUpdated { - if (!current.IsActive) - return false; - - bool isRunningNow = current.Status is BulkOperationStatus.Running && utcNow - current.UpdatedUtc < StaleAfter; - if (isRunningNow) - return false; - - Finish(current, BulkOperationStatus.Cancelled, "Cancelled before it started.", utcNow); - return true; + Description = workItem.Description ?? "Bulk operation", + Id = workItem.TaskId, + Message = "The operation's progress was lost and it could not continue.", + OrganizationId = workItem.OrganizationId, + Status = BackgroundTaskStatus.Failed, + Type = workItem.Type ?? String.Empty, + UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime }); } - public async Task IsCancelRequestedAsync(string id) - { - return (await _cache.GetAsync(GetCancelKey(id))).HasValue; - } - - public void Finish(BulkOperation operation, BulkOperationStatus status, string? message, DateTime? utcNow = null) - { - operation.Status = status; - operation.Message = message; - operation.CompletedUtc = utcNow ?? _timeProvider.GetUtcNow().UtcDateTime; - } - - private async Task FindConflictingOperationAsync(BulkOperation candidate) + private async Task FindConflictingTaskAsync(BackgroundTask candidate) { - // Only active operations can conflict, and every one of them is in the active index no matter how much history + // Only active tasks can conflict, and every one of them is in the active index no matter how much history // the organization has. var utcNow = _timeProvider.GetUtcNow().UtcDateTime; var finished = new List(); - BulkOperation? conflict = null; - foreach (string id in (await GetIndexedIdsAsync(GetActiveIndexKey(candidate.OrganizationId))).Distinct(StringComparer.Ordinal)) + BackgroundTask? conflict = null; + foreach (string id in await _taskService.GetActiveIdsAsync(candidate.OrganizationId)) { - var existing = await GetAsync(id); + var existing = await _taskService.GetAsync(id); if (existing is null || !existing.IsActive) { finished.Add(id); @@ -275,20 +155,20 @@ public void Finish(BulkOperation operation, BulkOperationStatus status, string? if (GetHandler(existing.Type) is not { IsDestructive: true }) continue; - // An organization wide operation can touch every project, so it conflicts with any project's operation. + // An organization wide task can touch every project, so it conflicts with any project's task. bool overlaps = existing.ProjectId is null || candidate.ProjectId is null || String.Equals(existing.ProjectId, candidate.ProjectId, StringComparison.Ordinal); if (!overlaps) continue; - if (utcNow - existing.UpdatedUtc >= StaleAfter) + if (utcNow - existing.UpdatedUtc >= BackgroundTaskService.StaleAfter) { // Retire it before admitting a replacement, so a delayed delivery cannot run it alongside the new one. - var retired = await UpdateAsync(existing.Id, current => + var retired = await _taskService.UpdateAsync(existing.Id, current => { - if (!current.IsActive || utcNow - current.UpdatedUtc < StaleAfter) + if (!current.IsActive || utcNow - current.UpdatedUtc < BackgroundTaskService.StaleAfter) return false; - Finish(current, BulkOperationStatus.Failed, "Stopped because it made no progress for two hours."); + current.Finish(BackgroundTaskStatus.Failed, "Stopped because it made no progress for two hours.", utcNow); return true; }); @@ -302,99 +182,21 @@ public void Finish(BulkOperation operation, BulkOperationStatus status, string? } if (finished.Count > 0) - await _cache.ListRemoveAsync(GetActiveIndexKey(candidate.OrganizationId), finished); + await _taskService.RemoveFromActiveIndexAsync(candidate.OrganizationId, finished); return conflict; } - - private async Task> GetIndexedIdsAsync(string key) - { - var ids = new List(); - for (int page = 1; page <= MaxIndexPages; page++) - { - var batch = await _cache.GetListAsync(key, page, IndexPageSize); - if (!batch.HasValue || batch.Value.Count == 0) - break; - - ids.AddRange(batch.Value); - if (batch.Value.Count < IndexPageSize) - break; - } - - return ids; - } - - private Task SaveAsync(BulkOperation operation) => _cache.SetAsync(GetOperationKey(operation.Id), operation, OperationLifetime); - - /// - /// Tells clients that an operation can no longer run because its record is gone (for example the cache was cleared). - /// The work item carries enough to publish a final state so the progress panel does not wait forever. - /// - public Task PublishLostAsync(BulkOperationWorkItem workItem) - { - return PublishAsync(new BulkOperationUpdated - { - Description = workItem.Description ?? "Bulk operation", - Id = workItem.OperationId, - Message = "The operation's progress was lost and it could not continue.", - OrganizationId = workItem.OrganizationId, - Status = BulkOperationStatus.Failed, - Type = workItem.Type ?? String.Empty, - UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime - }); - } - - private Task PublishAsync(BulkOperation operation) - { - return PublishAsync(new BulkOperationUpdated - { - Id = operation.Id, - OrganizationId = operation.OrganizationId, - ProjectId = operation.ProjectId, - Type = operation.Type, - Description = operation.Description, - Status = operation.Status, - Total = operation.Total, - Processed = operation.Processed, - Succeeded = operation.Succeeded, - Failed = operation.Failed, - Message = operation.Message, - UpdatedUtc = operation.UpdatedUtc - }); - } - - private async Task PublishAsync(BulkOperationUpdated message) - { - try - { - await _publisher.PublishAsync(message); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to publish bulk operation update for {OperationId}: {Message}", message.Id, ex.Message); - } - } - - private static string GetOperationKey(string id) => $"bulk-operations:{id}"; - private static string GetCancelKey(string id) => $"bulk-operations:{id}:cancel"; - private static string GetIndexKey(string organizationId) => $"bulk-operations:organization:{organizationId}"; - private static string GetActiveIndexKey(string organizationId) => $"bulk-operations:organization:{organizationId}:active"; - private static string GetPinnedKey(string id) => $"bulk-operations:{id}:selection"; } public record BulkOperationCreateResult { - public BulkOperation? Operation { get; init; } - public BulkOperation? Conflict { get; init; } + public BackgroundTask? Task { get; init; } + public BackgroundTask? Conflict { get; init; } public bool IsBusy { get; init; } public BulkOperationCheckResult? Rejection { get; init; } - public static BulkOperationCreateResult Created(BulkOperation operation) => new() { Operation = operation }; - public static BulkOperationCreateResult Conflicting(BulkOperation conflict) => new() { Conflict = conflict }; + public static BulkOperationCreateResult Created(BackgroundTask task) => new() { Task = task }; + public static BulkOperationCreateResult Conflicting(BackgroundTask conflict) => new() { Conflict = conflict }; public static BulkOperationCreateResult Busy() => new() { IsBusy = true }; public static BulkOperationCreateResult Rejected(BulkOperationCheckResult rejection) => new() { Rejection = rejection }; } diff --git a/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs index 323379cd26..b8c9a3fc89 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs @@ -1,3 +1,4 @@ +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Repositories; using Foundatio.Repositories; @@ -45,21 +46,21 @@ public Task ValidateAsync(BulkOperationRequest request return Task.FromResult(BulkOperationCheckResult.Ok); } - public string Describe(BulkOperation operation) + public string Describe(BackgroundTask task) { - return operation.Selection.Kind is BulkOperationSelectionKind.Ids - ? $"Delete {operation.Total:N0} selected events" - : $"Delete {operation.Total:N0} events matching filter"; + return task.Bulk!.Selection.Kind is BulkOperationSelectionKind.Ids + ? $"Delete {task.Total:N0} selected events" + : $"Delete {task.Total:N0} events matching filter"; } - public async Task ExecuteBatchAsync(BulkOperation operation, IReadOnlyCollection ids, CancellationToken cancellationToken) + public async Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) { var events = await _eventRepository.GetByIdsAsync(ids.ToArray(), o => o.Cache(false)); - // The operation may only touch its own organization (and project), no matter what ids it was given. + // The task may only touch its own organization (and project), no matter what ids it was given. var inScope = events - .Where(e => String.Equals(e.OrganizationId, operation.OrganizationId, StringComparison.Ordinal) - && (operation.ProjectId is null || String.Equals(e.ProjectId, operation.ProjectId, StringComparison.Ordinal))) + .Where(e => String.Equals(e.OrganizationId, task.OrganizationId, StringComparison.Ordinal) + && (task.ProjectId is null || String.Equals(e.ProjectId, task.ProjectId, StringComparison.Ordinal))) .ToList(); await _eventDeletionService.DeleteAsync(inScope); diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs index ad342edee3..e293d003db 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -1,5 +1,6 @@ using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Queries.Validation; using Exceptionless.Core.Repositories; @@ -60,16 +61,16 @@ public async Task ValidateSelectionAsync(Organization return BulkOperationCheckResult.Ok; } - public async Task PrepareAsync(BulkOperation operation, CancellationToken cancellationToken) + public async Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken) { - var appFilter = await GetAppFilterAsync(operation); + var appFilter = await GetAppFilterAsync(task); if (appFilter is null) return new BulkSelectionPreparation(BulkOperationCheckResult.Ok); // The events list resolves stack level terms (for example status:open) and removes deleted stacks by looking up // matching stack ids. Do that once, here, and keep the result so every page uses the same stacks. - var query = ApplyScope(new RepositoryQuery(), appFilter, operation.Selection) - .FilterExpression(operation.Selection.Filter) + var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection) + .FilterExpression(task.Bulk!.Selection.Filter) .EnforceEventStackFilter(); CountResult count; @@ -85,7 +86,7 @@ public async Task PrepareAsync(BulkOperation operation var pinned = new PinnedEventSelection { // Relative dates in the filter are pinned to the creation time, like the outer time window. - EventFilter = await RelativeDateFreezer.FreezeAsync(query.GetFilterExpression(), operation.Selection.UtcCreatedBefore ?? operation.CreatedUtc, IsDateFieldAsync), + EventFilter = await RelativeDateFreezer.FreezeAsync(query.GetFilterExpression(), task.Bulk!.Selection.UtcCreatedBefore ?? task.CreatedUtc, IsDateFieldAsync), ExcludedStackIds = query.GetExcludedStacks().ToArray(), StackIds = query.GetStacks().ToArray() }; @@ -93,14 +94,14 @@ public async Task PrepareAsync(BulkOperation operation return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, count.Total, _serializer.SerializeToString(pinned)); } - public async Task GetPageAsync(BulkOperation operation, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) + public async Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) { - var appFilter = await GetAppFilterAsync(operation); + var appFilter = await GetAppFilterAsync(task); if (appFilter is null || pinnedState is null) return new BulkSelectionPage([], null); var pinned = _serializer.Deserialize(pinnedState) ?? new PinnedEventSelection(); - var query = ApplyScope(new RepositoryQuery(), appFilter, operation.Selection) + var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection) .FilterExpression(pinned.EventFilter) .Include(e => e.Id) .SortExpression($"-{EventIndex.Alias.Date}"); @@ -111,7 +112,7 @@ public async Task GetPageAsync(BulkOperation operation, strin if (pinned.ExcludedStackIds.Length > 0) query.ExcludeStack(pinned.ExcludedStackIds); - // Paging only needs the next ids; the total was counted once when the operation was created. + // Paging only needs the next ids; the total was counted once when the task was created. var options = new CommandOptions().SearchAfterToken(cursor, _serializer).PageLimit(size).TrackTotalHits(false); var results = await _eventRepository.FindAsync(query, options); @@ -130,24 +131,24 @@ private static RepositoryQuery ApplyScope(RepositoryQuery GetAppFilterAsync(BulkOperation operation) + private async Task GetAppFilterAsync(BackgroundTask task) { - var organization = await _organizationRepository.GetByIdAsync(operation.OrganizationId, o => o.Cache()); + var organization = await _organizationRepository.GetByIdAsync(task.OrganizationId, o => o.Cache()); if (organization is null) return null; - var validation = await _validator.ValidateQueryAsync(operation.Selection.Filter); + var validation = await _validator.ValidateQueryAsync(task.Bulk!.Selection.Filter); AppFilter appFilter; - if (!String.IsNullOrEmpty(operation.ProjectId)) + if (!String.IsNullOrEmpty(task.ProjectId)) { - var project = await _projectRepository.GetByIdAsync(operation.ProjectId, o => o.Cache()); + var project = await _projectRepository.GetByIdAsync(task.ProjectId, o => o.Cache()); if (project is null || !String.Equals(project.OrganizationId, organization.Id, StringComparison.Ordinal)) return null; @@ -162,7 +163,7 @@ private static RepositoryQuery ApplyScope(RepositoryQueryWhat the stack level filter resolved to when the operation was created.
+ /// What the stack level filter resolved to when the task was created. private sealed class PinnedEventSelection { public string? EventFilter { get; set; } diff --git a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs index fafdb340a4..eb783ab7e4 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs @@ -1,4 +1,5 @@ using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; namespace Exceptionless.Core.Services.BulkOperations; @@ -15,7 +16,7 @@ public interface IBulkOperationHandler /// The kind of item the action targets. Selects the that pages ids. BulkOperationEntity Entity { get; } - /// Destructive actions are limited to one active operation per project. + /// Destructive actions are limited to one active task per project. bool IsDestructive { get; } /// Checks that the actor may run this action against the organization and project. @@ -24,14 +25,14 @@ public interface IBulkOperationHandler /// Checks the action specific parameters and the selection shape. Task ValidateAsync(BulkOperationRequest request); - /// A short, human readable description shown while the operation runs. - string Describe(BulkOperation operation); + /// A short, human readable description shown while the task runs. + string Describe(BackgroundTask task); /// /// Executes one batch of ids through the same code path as the single-item action, so side effects stay identical. - /// Ids that cannot be processed (missing, or outside the operation's organization and project) must be reported as failed. + /// Ids that cannot be processed (missing, or outside the task's organization and project) must be reported as failed. /// - Task ExecuteBatchAsync(BulkOperation operation, IReadOnlyCollection ids, CancellationToken cancellationToken); + Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken); } /// Pages the ids that match a filter selection. One implementation per . @@ -43,17 +44,17 @@ public interface IBulkSelectionResolver Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection); /// - /// Runs once when the operation is created. Counts what matches and pins every part of the selection that could - /// otherwise change while the operation runs (for events, the stack level filter), so the selection stays exactly - /// what the user confirmed and pages stay cheap. The pinned state is stored with the operation for its lifetime. + /// Runs once when the task is created. Counts what matches and pins every part of the selection that could + /// otherwise change while the task runs (for events, the stack level filter), so the selection stays exactly + /// what the user confirmed and pages stay cheap. The pinned state is stored with the task for its lifetime. /// - Task PrepareAsync(BulkOperation operation, CancellationToken cancellationToken); + Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken); /// /// Returns the next page of ids after . The cursor is a value based position, so /// paging stays correct while earlier items are being deleted or changed. Paging must not count totals. /// - Task GetPageAsync(BulkOperation operation, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken); + Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken); } public record BulkSelectionPreparation(BulkOperationCheckResult Check, long Total = 0, string? PinnedState = null) diff --git a/src/Exceptionless.Core/Utility/SampleDataService.cs b/src/Exceptionless.Core/Utility/SampleDataService.cs index 49b1d8feba..49587a4c69 100644 --- a/src/Exceptionless.Core/Utility/SampleDataService.cs +++ b/src/Exceptionless.Core/Utility/SampleDataService.cs @@ -2,8 +2,10 @@ using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services.BackgroundTasks; using Foundatio.Jobs; using Foundatio.Queues; using Foundatio.Repositories; @@ -21,6 +23,7 @@ public class SampleDataService private readonly BillingPlans _billingPlans; private readonly TimeProvider _timeProvider; private readonly IUserRepository _userRepository; + private readonly BackgroundTaskService _backgroundTaskService; private readonly ILogger _logger; public const string TEST_USER_EMAIL = "admin@exceptionless.test"; @@ -48,6 +51,7 @@ public SampleDataService( IUserRepository userRepository, ITokenRepository tokenRepository, IQueue workItemQueue, + BackgroundTaskService backgroundTaskService, BillingManager billingManager, BillingPlans billingPlans, TimeProvider timeProvider, @@ -59,6 +63,7 @@ ILoggerFactory loggerFactory _userRepository = userRepository; _tokenRepository = tokenRepository; _workItemQueue = workItemQueue; + _backgroundTaskService = backgroundTaskService; _billingManager = billingManager; _billingPlans = billingPlans; _timeProvider = timeProvider; @@ -287,17 +292,28 @@ await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem _logger.LogInformation("Enqueued sample event generation: {EventCount} events over {DaysBack} days", eventCount, daysBack); } - public async Task EnqueueSampleEventsAsync(string organizationId, string projectId, int eventCount = 250, int daysBack = 7) + /// + /// Queues sample event generation for one project and returns the id of the background task that tracks it. + /// + public async Task EnqueueSampleEventsAsync(Project project, string createdByUserId, int eventCount = 250, int daysBack = 7) { - string workItemId = await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem + var task = await _backgroundTaskService.CreateAndEnqueueAsync(new BackgroundTask { - OrganizationId = organizationId, - ProjectId = projectId, + OrganizationId = project.OrganizationId, + ProjectId = project.Id, + Type = BackgroundTaskTypes.GenerateProjectSampleData, + Description = $"Generate sample data for {project.Name}", + CreatedByUserId = createdByUserId + }, t => new GenerateSampleEventsWorkItem + { + OrganizationId = project.OrganizationId, + ProjectId = project.Id, EventCount = eventCount, - DaysBack = daysBack + DaysBack = daysBack, + TaskId = t.Id }); - _logger.LogInformation("Enqueued sample event generation for project {ProjectId}: {EventCount} events over {DaysBack} days", projectId, eventCount, daysBack); - return workItemId; + _logger.LogInformation("Enqueued sample event generation for project {ProjectId}: {EventCount} events over {DaysBack} days", project.Id, eventCount, daysBack); + return task.Id; } } diff --git a/src/Exceptionless.Web/Api/ApiEndpoints.cs b/src/Exceptionless.Web/Api/ApiEndpoints.cs index cd547e9e34..4063b21e59 100644 --- a/src/Exceptionless.Web/Api/ApiEndpoints.cs +++ b/src/Exceptionless.Web/Api/ApiEndpoints.cs @@ -16,7 +16,7 @@ public static WebApplication MapApiEndpoints(this WebApplication app) app.MapWebHookEndpoints(); app.MapStripeEndpoints(); app.MapSavedViewEndpoints(); - app.MapBulkOperationEndpoints(); + app.MapBackgroundTaskEndpoints(); app.MapUserEndpoints(); app.MapProjectEndpoints(); app.MapOrganizationEndpoints(); diff --git a/src/Exceptionless.Web/Api/Endpoints/BulkOperationEndpoints.cs b/src/Exceptionless.Web/Api/Endpoints/BackgroundTaskEndpoints.cs similarity index 58% rename from src/Exceptionless.Web/Api/Endpoints/BulkOperationEndpoints.cs rename to src/Exceptionless.Web/Api/Endpoints/BackgroundTaskEndpoints.cs index c320374218..3af9e51308 100644 --- a/src/Exceptionless.Web/Api/Endpoints/BulkOperationEndpoints.cs +++ b/src/Exceptionless.Web/Api/Endpoints/BackgroundTaskEndpoints.cs @@ -6,24 +6,24 @@ using Exceptionless.Web.Utility.OpenApi; using Foundatio.Mediator; using Microsoft.AspNetCore.Mvc; -using BulkOperationMessages = Exceptionless.Web.Api.Messages; +using TaskMessages = Exceptionless.Web.Api.Messages; using HttpIResult = Microsoft.AspNetCore.Http.IResult; namespace Exceptionless.Web.Api.Endpoints; -public static class BulkOperationEndpoints +public static class BackgroundTaskEndpoints { - public static IEndpointRouteBuilder MapBulkOperationEndpoints(this IEndpointRouteBuilder endpoints) + public static IEndpointRouteBuilder MapBackgroundTaskEndpoints(this IEndpointRouteBuilder endpoints) { var group = endpoints.MapGroup("api/v2") .RequireAuthorization(AuthorizationRoles.UserPolicy) .AddEndpointFilter() - .WithTags("BulkOperation"); + .WithTags("BackgroundTask"); group.MapPost("bulk-operations", async (IMediator mediator, IMediatorResultMapper resultMapper, [FromBody] NewBulkOperation operation) - => (await mediator.InvokeAsync>>(new BulkOperationMessages.CreateBulkOperation(operation))).ToHttpResult(resultMapper)) + => (await mediator.InvokeAsync>>(new TaskMessages.CreateBulkOperation(operation))).ToHttpResult(resultMapper)) .Accepts("application/json", "application/*+json") - .Produces(StatusCodes.Status202Accepted) + .Produces(StatusCodes.Status202Accepted) .ProducesProblem(StatusCodes.Status403Forbidden) .ProducesProblem(StatusCodes.Status404NotFound) .ProducesProblem(StatusCodes.Status409Conflict) @@ -33,8 +33,8 @@ public static IEndpointRouteBuilder MapBulkOperationEndpoints(this IEndpointRout .WithDescription(""" Starts an action, such as `events.delete`, on everything that matches a selection. The selection is either a list of ids or a filter with a time range. The time range is resolved to absolute UTC dates when the operation is created, and events - stored after that moment are never included. The operation runs in the background; poll it or listen for - `BulkOperationUpdated` messages to follow its progress. + stored after that moment are never included. The operation runs in the background and is returned as a task; poll + `GET api/v2/tasks/{id}` or listen for `BackgroundTaskUpdated` messages to follow its progress. """) .WithMetadata(new EndpointDocumentation { RequestBodyDescription = "The bulk operation to start.", @@ -48,27 +48,28 @@ public static IEndpointRouteBuilder MapBulkOperationEndpoints(this IEndpointRout } }); - group.MapGet("bulk-operations/{id:objectid}", async (string id, IMediator mediator, IMediatorResultMapper resultMapper) - => (await mediator.InvokeAsync>(new BulkOperationMessages.GetBulkOperationById(id))).ToHttpResult(resultMapper)) - .WithName("GetBulkOperationById") - .Produces() + group.MapGet("tasks/{id:objectid}", async (string id, IMediator mediator, IMediatorResultMapper resultMapper) + => (await mediator.InvokeAsync>(new TaskMessages.GetBackgroundTaskById(id))).ToHttpResult(resultMapper)) + .WithName("GetBackgroundTaskById") + .Produces() .ProducesProblem(StatusCodes.Status404NotFound) .WithSummary("Get by id") + .WithDescription("Returns the progress of a task, such as a bulk action, a project data reset or sample data generation. Tasks are kept for seven days.") .WithMetadata(new EndpointDocumentation { ParameterDescriptions = new() { - ["id"] = "The identifier of the bulk operation.", + ["id"] = "The identifier of the task.", }, ResponseDescriptions = new() { - ["404"] = "The bulk operation could not be found.", + ["404"] = "The task could not be found.", } }); - group.MapGet("organizations/{organizationId:objectid}/bulk-operations", async (string organizationId, IMediator mediator, IMediatorResultMapper resultMapper) - => (await mediator.InvokeAsync>>(new BulkOperationMessages.GetBulkOperationsByOrganization(organizationId))).ToHttpResult(resultMapper)) - .Produces>() + group.MapGet("organizations/{organizationId:objectid}/tasks", async (string organizationId, IMediator mediator, IMediatorResultMapper resultMapper) + => (await mediator.InvokeAsync>>(new TaskMessages.GetBackgroundTasksByOrganization(organizationId))).ToHttpResult(resultMapper)) + .Produces>() .ProducesProblem(StatusCodes.Status404NotFound) .WithSummary("Get by organization") - .WithDescription("Returns the active and recently finished bulk operations of the organization, newest first. Finished operations are kept for seven days.") + .WithDescription("Returns the active and recently finished tasks of the organization, newest first. Finished tasks are kept for seven days.") .WithMetadata(new EndpointDocumentation { ParameterDescriptions = new() { ["organizationId"] = "The identifier of the organization.", @@ -78,19 +79,21 @@ public static IEndpointRouteBuilder MapBulkOperationEndpoints(this IEndpointRout } }); - group.MapPost("bulk-operations/{id:objectid}/cancel", async (string id, IMediator mediator, IMediatorResultMapper resultMapper) - => (await mediator.InvokeAsync>>(new BulkOperationMessages.CancelBulkOperation(id))).ToHttpResult(resultMapper)) - .Produces(StatusCodes.Status202Accepted) + group.MapPost("tasks/{id:objectid}/cancel", async (string id, IMediator mediator, IMediatorResultMapper resultMapper) + => (await mediator.InvokeAsync>>(new TaskMessages.CancelBackgroundTask(id))).ToHttpResult(resultMapper)) + .Produces(StatusCodes.Status202Accepted) .ProducesProblem(StatusCodes.Status404NotFound) + .ProducesProblem(StatusCodes.Status422UnprocessableEntity) .WithSummary("Cancel") - .WithDescription("Requests cancellation. A running operation stops after the batch it is processing; work already done is not undone. Cancelling a finished operation has no effect.") + .WithDescription("Requests cancellation. A running bulk action stops after the batch it is processing; work already done is not undone. Cancelling a finished task has no effect. Tasks that cannot be stopped, such as a project data reset, are rejected.") .WithMetadata(new EndpointDocumentation { ParameterDescriptions = new() { - ["id"] = "The identifier of the bulk operation.", + ["id"] = "The identifier of the task.", }, ResponseDescriptions = new() { ["202"] = "Accepted", - ["404"] = "The bulk operation could not be found.", + ["404"] = "The task could not be found.", + ["422"] = "The task cannot be cancelled.", } }); diff --git a/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs similarity index 66% rename from src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs rename to src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs index 902817478d..1e59ecec4a 100644 --- a/src/Exceptionless.Web/Api/Handlers/BulkOperationHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs @@ -1,9 +1,11 @@ using Exceptionless.Core; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Core.Services.BulkOperations; using Exceptionless.Web.Api.Infrastructure; using Exceptionless.Web.Api.Messages; @@ -16,7 +18,8 @@ namespace Exceptionless.Web.Api.Handlers; -public class BulkOperationHandler( +public class BackgroundTaskHandler( + BackgroundTaskService backgroundTaskService, BulkOperationService bulkOperationService, IEnumerable resolvers, IOrganizationRepository organizationRepository, @@ -24,14 +27,14 @@ public class BulkOperationHandler( AppOptions appOptions, TimeProvider timeProvider, IHttpContextAccessor httpContextAccessor, - ILogger logger) + ILogger logger) { private const int MaxIds = 1000; private static readonly ICollection _allowedDateFields = new List { EventIndex.Alias.Date }; private HttpContext HttpContext => httpContextAccessor.HttpContext ?? throw new InvalidOperationException("HttpContext is unavailable."); - public async Task>> Handle(CreateBulkOperation message) + public async Task>> Handle(CreateBulkOperation message) { var request = HttpContext.Request; var model = message.Operation; @@ -69,20 +72,23 @@ public async Task>> Handle(CreateBulk var authorized = await handler.AuthorizeAsync(bulkRequest); if (!authorized.IsOk) - return ToResult>(authorized); + return ToResult>(authorized); var validated = await handler.ValidateAsync(bulkRequest); if (!validated.IsOk) - return ToResult>(validated); + return ToResult>(validated); - var operation = new BulkOperation + var task = new BackgroundTask { OrganizationId = organization.Id, ProjectId = project?.Id, Type = handler.Type, - Selection = selection, - Parameters = new Dictionary(parameters), - CreatedByUserId = user.Id + CreatedByUserId = user.Id, + Bulk = new BulkOperationDetails + { + Selection = selection, + Parameters = new Dictionary(parameters) + } }; if (selection.Kind is BulkOperationSelectionKind.Filter) @@ -93,65 +99,68 @@ public async Task>> Handle(CreateBulk var selectionCheck = await resolver.ValidateSelectionAsync(organization, selection); if (!selectionCheck.IsOk) - return ToResult>(selectionCheck); + return ToResult>(selectionCheck); } // Counting and pinning the selection (and describing the operation) happens while it is created. - var created = await bulkOperationService.CreateAsync(operation); + var created = await bulkOperationService.CreateAsync(task); if (created.Rejection is not null) - return ToResult>(created.Rejection); + return ToResult>(created.Rejection); if (created.Conflict is not null) return Result.Conflict("Another destructive operation is already running for this project. Wait for it to finish or cancel it first."); - if (created.Operation is null) + if (created.Task is null) return Result.Conflict("Another bulk operation is being started for this organization. Please try again."); using var _ = logger.BeginScope(new ExceptionlessState().Organization(organization.Id).Tag("BulkOperation").Identity(user.EmailAddress)); - logger.LogInformation("User {User} started bulk operation {OperationId} {Type} for {Total} items", user.Id, created.Operation.Id, created.Operation.Type, created.Operation.Total); + logger.LogInformation("User {User} started bulk operation {TaskId} {Type} for {Total} items", user.Id, created.Task.Id, created.Task.Type, created.Task.Total); - return new AcceptedResponse(MapToViewModel(created.Operation)); + return new AcceptedResponse(MapToViewModel(created.Task)); } - public async Task> Handle(GetBulkOperationById message) + public async Task> Handle(GetBackgroundTaskById message) { - var operation = await GetAccessibleAsync(message.Id); - if (operation is null) - return Result.NotFound("Bulk operation not found."); + var task = await GetAccessibleAsync(message.Id); + if (task is null) + return Result.NotFound("Task not found."); - return MapToViewModel(operation); + return MapToViewModel(task); } - public async Task>> Handle(GetBulkOperationsByOrganization message) + public async Task>> Handle(GetBackgroundTasksByOrganization message) { if (!HttpContext.Request.CanAccessOrganization(message.OrganizationId)) return Result.NotFound("Organization not found."); - var operations = await bulkOperationService.GetByOrganizationAsync(message.OrganizationId); - return Result>.Success(operations.Select(MapToViewModel).ToList()); + var tasks = await backgroundTaskService.GetByOrganizationAsync(message.OrganizationId); + return Result>.Success(tasks.Select(MapToViewModel).ToList()); } - public async Task>> Handle(CancelBulkOperation message) + public async Task>> Handle(CancelBackgroundTask message) { - var operation = await GetAccessibleAsync(message.Id); - if (operation is null) - return Result.NotFound("Bulk operation not found."); + var task = await GetAccessibleAsync(message.Id); + if (task is null) + return Result.NotFound("Task not found."); + + var cancelled = await backgroundTaskService.CancelAsync(task.Id); + if (cancelled.Task is null) + return Result.NotFound("Task not found."); - var cancelled = await bulkOperationService.CancelAsync(operation.Id); - if (cancelled is null) - return Result.NotFound("Bulk operation not found."); + if (cancelled.IsNotCancellable) + return Result.Invalid(ValidationError.Create("task", "This task cannot be cancelled.")); - logger.LogInformation("User {User} requested cancellation of bulk operation {OperationId}", HttpContext.Request.GetUser().Id, cancelled.Id); - return new AcceptedResponse(MapToViewModel(cancelled)); + logger.LogInformation("User {User} requested cancellation of task {TaskId}", HttpContext.Request.GetUser().Id, cancelled.Task.Id); + return new AcceptedResponse(MapToViewModel(cancelled.Task)); } - private async Task GetAccessibleAsync(string id) + private async Task GetAccessibleAsync(string id) { - var operation = await bulkOperationService.GetAsync(id); - if (operation is null || !HttpContext.Request.CanAccessOrganization(operation.OrganizationId)) + var task = await backgroundTaskService.GetAsync(id); + if (task is null || !HttpContext.Request.CanAccessOrganization(task.OrganizationId)) return null; - return operation; + return task; } private (BulkOperationSelection? Selection, Result? Result) ResolveSelection(NewBulkOperationSelection model, Organization organization) @@ -215,35 +224,38 @@ private static Result ToResult(BulkOperationCheckResult check) }; } - private static ViewBulkOperation MapToViewModel(BulkOperation operation) + private static ViewBackgroundTask MapToViewModel(BackgroundTask task) { - return new ViewBulkOperation + var selection = task.Bulk?.Selection; + return new ViewBackgroundTask { - Id = operation.Id, - OrganizationId = operation.OrganizationId, - ProjectId = operation.ProjectId, - Type = operation.Type, - Description = operation.Description, - Status = operation.Status, - Selection = new ViewBulkOperationSelection + Id = task.Id, + OrganizationId = task.OrganizationId, + ProjectId = task.ProjectId, + Type = task.Type, + Description = task.Description, + Status = task.Status, + Progress = task.Progress, + Total = task.Total, + Processed = task.Processed, + Succeeded = task.Succeeded, + Failed = task.Failed, + CanCancel = task.CanCancel, + Selection = selection is null ? null : new ViewBulkOperationSelection { - Kind = operation.Selection.Kind, - IdCount = operation.Selection.Kind is BulkOperationSelectionKind.Ids ? operation.Selection.Ids?.Count : null, - Filter = operation.Selection.Filter, - Time = operation.Selection.Time, - UtcStart = operation.Selection.UtcStart, - UtcEnd = operation.Selection.UtcEnd + Kind = selection.Kind, + IdCount = selection.Kind is BulkOperationSelectionKind.Ids ? selection.Ids?.Count : null, + Filter = selection.Filter, + Time = selection.Time, + UtcStart = selection.UtcStart, + UtcEnd = selection.UtcEnd }, - Total = operation.Total, - Processed = operation.Processed, - Succeeded = operation.Succeeded, - Failed = operation.Failed, - CreatedByUserId = operation.CreatedByUserId, - CreatedUtc = operation.CreatedUtc, - StartedUtc = operation.StartedUtc, - CompletedUtc = operation.CompletedUtc, - UpdatedUtc = operation.UpdatedUtc, - Message = operation.Message + CreatedByUserId = task.CreatedByUserId, + CreatedUtc = task.CreatedUtc, + StartedUtc = task.StartedUtc, + CompletedUtc = task.CompletedUtc, + UpdatedUtc = task.UpdatedUtc, + Message = task.Message }; } } diff --git a/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs b/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs index 3705100472..be9a73d13c 100644 --- a/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/ProjectHandler.cs @@ -3,10 +3,12 @@ using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Queries; using Exceptionless.Core.Services; +using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Core.Utility; using Exceptionless.Web.Api.Infrastructure; using Exceptionless.Web.Api.Messages; @@ -37,6 +39,7 @@ public class ProjectHandler( SlackService slackService, IOAuthProviderClient oauthProviderClient, SampleDataService sampleDataService, + BackgroundTaskService backgroundTaskService, ApiMapper mapper, ITextSerializer serializer, AppOptions options, @@ -223,8 +226,8 @@ public async Task> Handle(GenerateProjectSampleData if (project is null) return Result.NotFound("Project not found."); - string workItemId = await sampleDataService.EnqueueSampleEventsAsync(project.OrganizationId, project.Id); - return new WorkInProgressResult([workItemId]); + string taskId = await sampleDataService.EnqueueSampleEventsAsync(project, GetCurrentUser(message.Context).Id); + return new WorkInProgressResult([taskId]); } public async Task> Handle(ResetProjectData message) @@ -233,13 +236,22 @@ public async Task> Handle(ResetProjectData message) if (project is null) return Result.NotFound("Project not found."); - string workItemId = await workItemQueue.EnqueueAsync(new ResetProjectDataWorkItem + // The returned id is the background task that tracks the reset, so the progress panel can follow it. + var task = await backgroundTaskService.CreateAndEnqueueAsync(new BackgroundTask { OrganizationId = project.OrganizationId, - ProjectId = project.Id + ProjectId = project.Id, + Type = BackgroundTaskTypes.ResetProjectData, + Description = $"Reset data for {project.Name}", + CreatedByUserId = GetCurrentUser(message.Context).Id + }, t => new ResetProjectDataWorkItem + { + OrganizationId = project.OrganizationId, + ProjectId = project.Id, + TaskId = t.Id }); - return new WorkInProgressResult([workItemId]); + return new WorkInProgressResult([task.Id]); } public async Task>> Handle(GetProjectNotificationSettings message) diff --git a/src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs b/src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs new file mode 100644 index 0000000000..26024a5207 --- /dev/null +++ b/src/Exceptionless.Web/Api/Messages/BackgroundTaskMessages.cs @@ -0,0 +1,8 @@ +using Exceptionless.Web.Models; + +namespace Exceptionless.Web.Api.Messages; + +public record CreateBulkOperation(NewBulkOperation Operation); +public record GetBackgroundTaskById(string Id); +public record GetBackgroundTasksByOrganization(string OrganizationId); +public record CancelBackgroundTask(string Id); diff --git a/src/Exceptionless.Web/Api/Messages/BulkOperationMessages.cs b/src/Exceptionless.Web/Api/Messages/BulkOperationMessages.cs deleted file mode 100644 index 54a2db3aa7..0000000000 --- a/src/Exceptionless.Web/Api/Messages/BulkOperationMessages.cs +++ /dev/null @@ -1,8 +0,0 @@ -using Exceptionless.Web.Models; - -namespace Exceptionless.Web.Api.Messages; - -public record CreateBulkOperation(NewBulkOperation Operation); -public record GetBulkOperationById(string Id); -public record GetBulkOperationsByOrganization(string OrganizationId); -public record CancelBulkOperation(string Id); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts index 3112cb20c9..bbc3efc31f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/api.ts @@ -7,14 +7,6 @@ export enum StackStatus { Discarded = "discarded", } -export enum BulkOperationStatus { - Queued = "Queued", - Running = "Running", - Completed = "Completed", - Failed = "Failed", - Cancelled = "Cancelled", -} - export enum BulkOperationSelectionKind { Ids = "Ids", Filter = "Filter", @@ -28,6 +20,14 @@ export enum BillingStatus { Unpaid = 4, } +export enum BackgroundTaskStatus { + Queued = "Queued", + Running = "Running", + Completed = "Completed", + Failed = "Failed", + Cancelled = "Cancelled", +} + export interface AdminAssistantOrganizationUsage { organization_id: string; organization_name: string; @@ -855,18 +855,26 @@ export interface UserSavedViewOrderPreference { saved_view_ids: string[]; } -export interface ViewBulkOperation { +export interface ViewBackgroundTask { /** @pattern ^[a-fA-F0-9]{24}$ */ id: string; /** @pattern ^[a-fA-F0-9]{24}$ */ organization_id: string; /** @pattern ^[a-fA-F0-9]{24}$ */ project_id?: null | string; + /** What the task does, e.g. events.delete, projects.reset-data or projects.sample-data. */ type: string; description: string; - status: BulkOperationStatus; - selection: ViewBulkOperationSelection; - /** @format int64 */ + status: BackgroundTaskStatus; + /** + * Percent complete, 0 to 100. + * @format int32 + */ + progress: number; + /** + * The item counts. All zero for tasks that only report a percent. + * @format int64 + */ total: number; /** @format int64 */ processed: number; @@ -874,6 +882,10 @@ export interface ViewBulkOperation { succeeded: number; /** @format int64 */ failed: number; + /** Whether the task can be stopped with the cancel endpoint. */ + can_cancel: boolean; + /** What a bulk action applies to. Only set for bulk actions. */ + selection?: null | ViewBulkOperationSelection; /** @pattern ^[a-fA-F0-9]{24}$ */ created_by_user_id: string; /** @format date-time */ @@ -887,6 +899,7 @@ export interface ViewBulkOperation { message?: null | string; } +/** What a bulk action applies to. Only set for bulk actions. */ export interface ViewBulkOperationSelection { kind: BulkOperationSelectionKind; /** @@ -902,7 +915,7 @@ export interface ViewBulkOperationSelection { */ utc_start?: null | string; /** - * The absolute end of the resolved time window, never later than when the operation was created. + * The absolute end of the resolved time window, never later than when the task was created. * @format date-time */ utc_end?: null | string; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts index cf92fbbc21..e55cd3691c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/generated/schemas.ts @@ -27,13 +27,6 @@ export const StackStatusSchema = zodEnum([ "ignored", "discarded", ]); -export const BulkOperationStatusSchema = zodEnum([ - "Queued", - "Running", - "Completed", - "Failed", - "Cancelled", -]); export const BulkOperationSelectionKindSchema = zodEnum(["Ids", "Filter"]); export const BillingStatusSchema = union([ literal(0), @@ -42,6 +35,13 @@ export const BillingStatusSchema = union([ literal(3), literal(4), ]); +export const BackgroundTaskStatusSchema = zodEnum([ + "Queued", + "Running", + "Completed", + "Failed", + "Cancelled", +]); export const AdminAssistantOrganizationUsageSchema = object({ organization_id: string().min(1, "Organization id is required"), @@ -1001,7 +1001,7 @@ export type UserSavedViewOrderPreferenceFormData = Infer< typeof UserSavedViewOrderPreferenceSchema >; -export const ViewBulkOperationSchema = object({ +export const ViewBackgroundTaskSchema = object({ id: string() .length(24, "Id must be exactly 24 characters") .regex(/^[a-fA-F0-9]{24}$/, "Id has invalid format"), @@ -1015,12 +1015,14 @@ export const ViewBulkOperationSchema = object({ .optional(), type: string().min(1, "Type is required"), description: string().min(1, "Description is required"), - status: BulkOperationStatusSchema, - selection: lazy(() => ViewBulkOperationSelectionSchema), + status: BackgroundTaskStatusSchema, + progress: int32(), total: int(), processed: int(), succeeded: int(), failed: int(), + can_cancel: boolean(), + selection: lazy(() => ViewBulkOperationSelectionSchema).optional(), created_by_user_id: string() .length(24, "Created by user id must be exactly 24 characters") .regex(/^[a-fA-F0-9]{24}$/, "Created by user id has invalid format"), @@ -1030,7 +1032,7 @@ export const ViewBulkOperationSchema = object({ updated_utc: iso.datetime(), message: string().min(1, "Message is required").nullable().optional(), }); -export type ViewBulkOperationFormData = Infer; +export type ViewBackgroundTaskFormData = Infer; export const ViewBulkOperationSelectionSchema = object({ kind: BulkOperationSelectionKindSchema, diff --git a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs index 4db79ecaa7..ebf136094a 100644 --- a/src/Exceptionless.Web/Hubs/MessageBusBroker.cs +++ b/src/Exceptionless.Web/Hubs/MessageBusBroker.cs @@ -38,7 +38,7 @@ await Task.WhenAll( _subscriber.SubscribeAsync(OnEntityChangedAsync, shutdownToken), _subscriber.SubscribeAsync(OnPlanChangedAsync, shutdownToken), _subscriber.SubscribeAsync(OnPlanOverageAsync, shutdownToken), - _subscriber.SubscribeAsync(OnBulkOperationUpdatedAsync, shutdownToken), + _subscriber.SubscribeAsync(OnBackgroundTaskUpdatedAsync, shutdownToken), _subscriber.SubscribeAsync(OnUserMembershipChangedAsync, shutdownToken), _subscriber.SubscribeAsync(OnReleaseNotificationAsync, shutdownToken), _subscriber.SubscribeAsync(OnSystemNotificationAsync, shutdownToken) @@ -159,12 +159,12 @@ private Task OnPlanOverageAsync(PlanOverage planOverage, CancellationToken cance return Task.CompletedTask; } - internal Task OnBulkOperationUpdatedAsync(BulkOperationUpdated bulkOperationUpdated, CancellationToken cancellationToken = default) + internal Task OnBackgroundTaskUpdatedAsync(BackgroundTaskUpdated backgroundTaskUpdated, CancellationToken cancellationToken = default) { - if (bulkOperationUpdated is not null) + if (backgroundTaskUpdated is not null) { - _logger.LogTrace("Sending bulk operation {OperationId} update to organization: {Organization}", bulkOperationUpdated.Id, bulkOperationUpdated.OrganizationId); - return GroupSendAsync(bulkOperationUpdated.OrganizationId, bulkOperationUpdated); + _logger.LogTrace("Sending background task {TaskId} update to organization: {Organization}", backgroundTaskUpdated.Id, backgroundTaskUpdated.OrganizationId); + return GroupSendAsync(backgroundTaskUpdated.OrganizationId, backgroundTaskUpdated); } return Task.CompletedTask; diff --git a/src/Exceptionless.Web/Models/BulkOperation/ViewBulkOperation.cs b/src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs similarity index 64% rename from src/Exceptionless.Web/Models/BulkOperation/ViewBulkOperation.cs rename to src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs index 72cab11289..b309fef872 100644 --- a/src/Exceptionless.Web/Models/BulkOperation/ViewBulkOperation.cs +++ b/src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs @@ -1,9 +1,10 @@ using Exceptionless.Core.Attributes; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; namespace Exceptionless.Web.Models; -public record ViewBulkOperation +public record ViewBackgroundTask { [ObjectId] public string Id { get; set; } = null!; @@ -14,16 +15,27 @@ public record ViewBulkOperation [ObjectId] public string? ProjectId { get; set; } + /// What the task does, e.g. events.delete, projects.reset-data or projects.sample-data. public string Type { get; set; } = null!; + public string Description { get; set; } = null!; - public BulkOperationStatus Status { get; set; } - public ViewBulkOperationSelection Selection { get; set; } = null!; + public BackgroundTaskStatus Status { get; set; } + + /// Percent complete, 0 to 100. + public int Progress { get; set; } + /// The item counts. All zero for tasks that only report a percent. public long Total { get; set; } public long Processed { get; set; } public long Succeeded { get; set; } public long Failed { get; set; } + /// Whether the task can be stopped with the cancel endpoint. + public bool CanCancel { get; set; } + + /// What a bulk action applies to. Only set for bulk actions. + public ViewBulkOperationSelection? Selection { get; set; } + [ObjectId] public string CreatedByUserId { get; set; } = null!; @@ -47,6 +59,6 @@ public record ViewBulkOperationSelection /// The absolute start of the resolved time window. public DateTime? UtcStart { get; set; } - /// The absolute end of the resolved time window, never later than when the operation was created. + /// The absolute end of the resolved time window, never later than when the task was created. public DateTime? UtcEnd { get; set; } } diff --git a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json index 20858e3b00..613918c1fb 100644 --- a/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json +++ b/tests/Exceptionless.Tests/Api/Data/endpoint-manifest.json @@ -850,35 +850,7 @@ "route": "/api/v2/bulk-operations", "displayName": "HTTP: POST api/v2/bulk-operations", "tags": [ - "BulkOperation" - ], - "allowAnonymous": false, - "authorizationPolicies": [ - "UserPolicy" - ], - "authorizationRoles": [], - "authenticationSchemes": [] - }, - { - "method": "GET", - "route": "/api/v2/bulk-operations/{id:objectid}", - "displayName": "HTTP: GET api/v2/bulk-operations/{id:objectid}", - "tags": [ - "BulkOperation" - ], - "allowAnonymous": false, - "authorizationPolicies": [ - "UserPolicy" - ], - "authorizationRoles": [], - "authenticationSchemes": [] - }, - { - "method": "POST", - "route": "/api/v2/bulk-operations/{id:objectid}/cancel", - "displayName": "HTTP: POST api/v2/bulk-operations/{id:objectid}/cancel", - "tags": [ - "BulkOperation" + "BackgroundTask" ], "allowAnonymous": false, "authorizationPolicies": [ @@ -1500,20 +1472,6 @@ "authorizationRoles": [], "authenticationSchemes": [] }, - { - "method": "GET", - "route": "/api/v2/organizations/{organizationId:objectid}/bulk-operations", - "displayName": "HTTP: GET api/v2/organizations/{organizationId:objectid}/bulk-operations", - "tags": [ - "BulkOperation" - ], - "allowAnonymous": false, - "authorizationPolicies": [ - "UserPolicy" - ], - "authorizationRoles": [], - "authenticationSchemes": [] - }, { "method": "GET", "route": "/api/v2/organizations/{organizationId:objectid}/events", @@ -1711,6 +1669,20 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "GET", + "route": "/api/v2/organizations/{organizationId:objectid}/tasks", + "displayName": "HTTP: GET api/v2/organizations/{organizationId:objectid}/tasks", + "tags": [ + "BackgroundTask" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "GET", "route": "/api/v2/organizations/{organizationId:objectid}/tokens", @@ -2690,6 +2662,34 @@ "authorizationRoles": [], "authenticationSchemes": [] }, + { + "method": "GET", + "route": "/api/v2/tasks/{id:objectid}", + "displayName": "HTTP: GET api/v2/tasks/{id:objectid}", + "tags": [ + "BackgroundTask" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, + { + "method": "POST", + "route": "/api/v2/tasks/{id:objectid}/cancel", + "displayName": "HTTP: POST api/v2/tasks/{id:objectid}/cancel", + "tags": [ + "BackgroundTask" + ], + "allowAnonymous": false, + "authorizationPolicies": [ + "UserPolicy" + ], + "authorizationRoles": [], + "authenticationSchemes": [] + }, { "method": "POST", "route": "/api/v2/tokens", diff --git a/tests/Exceptionless.Tests/Api/Data/openapi.json b/tests/Exceptionless.Tests/Api/Data/openapi.json index a2b561d3e4..f73e56b520 100644 --- a/tests/Exceptionless.Tests/Api/Data/openapi.json +++ b/tests/Exceptionless.Tests/Api/Data/openapi.json @@ -3995,10 +3995,10 @@ "/api/v2/bulk-operations": { "post": { "tags": [ - "BulkOperation" + "BackgroundTask" ], "summary": "Create", - "description": "Starts an action, such as \u0060events.delete\u0060, on everything that matches a selection. The selection is either a list of ids\nor a filter with a time range. The time range is resolved to absolute UTC dates when the operation is created, and events\nstored after that moment are never included. The operation runs in the background; poll it or listen for\n\u0060BulkOperationUpdated\u0060 messages to follow its progress.", + "description": "Starts an action, such as \u0060events.delete\u0060, on everything that matches a selection. The selection is either a list of ids\nor a filter with a time range. The time range is resolved to absolute UTC dates when the operation is created, and events\nstored after that moment are never included. The operation runs in the background and is returned as a task; poll\n\u0060GET api/v2/tasks/{id}\u0060 or listen for \u0060BackgroundTaskUpdated\u0060 messages to follow its progress.", "requestBody": { "description": "The bulk operation to start.", "content": { @@ -4021,7 +4021,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ViewBulkOperation" + "$ref": "#/components/schemas/ViewBackgroundTask" } } } @@ -4079,18 +4079,19 @@ } } }, - "/api/v2/bulk-operations/{id}": { + "/api/v2/tasks/{id}": { "get": { "tags": [ - "BulkOperation" + "BackgroundTask" ], "summary": "Get by id", - "operationId": "GetBulkOperationById", + "description": "Returns the progress of a task, such as a bulk action, a project data reset or sample data generation. Tasks are kept for seven days.", + "operationId": "GetBackgroundTaskById", "parameters": [ { "name": "id", "in": "path", - "description": "The identifier of the bulk operation.", + "description": "The identifier of the task.", "required": true, "schema": { "pattern": "^[a-zA-Z\\d]{24,36}$", @@ -4104,13 +4105,13 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ViewBulkOperation" + "$ref": "#/components/schemas/ViewBackgroundTask" } } } }, "404": { - "description": "The bulk operation could not be found.", + "description": "The task could not be found.", "content": { "application/problem\u002Bjson": { "schema": { @@ -4122,13 +4123,13 @@ } } }, - "/api/v2/organizations/{organizationId}/bulk-operations": { + "/api/v2/organizations/{organizationId}/tasks": { "get": { "tags": [ - "BulkOperation" + "BackgroundTask" ], "summary": "Get by organization", - "description": "Returns the active and recently finished bulk operations of the organization, newest first. Finished operations are kept for seven days.", + "description": "Returns the active and recently finished tasks of the organization, newest first. Finished tasks are kept for seven days.", "parameters": [ { "name": "organizationId", @@ -4149,7 +4150,7 @@ "schema": { "type": "array", "items": { - "$ref": "#/components/schemas/ViewBulkOperation" + "$ref": "#/components/schemas/ViewBackgroundTask" } } } @@ -4168,18 +4169,18 @@ } } }, - "/api/v2/bulk-operations/{id}/cancel": { + "/api/v2/tasks/{id}/cancel": { "post": { "tags": [ - "BulkOperation" + "BackgroundTask" ], "summary": "Cancel", - "description": "Requests cancellation. A running operation stops after the batch it is processing; work already done is not undone. Cancelling a finished operation has no effect.", + "description": "Requests cancellation. A running bulk action stops after the batch it is processing; work already done is not undone. Cancelling a finished task has no effect. Tasks that cannot be stopped, such as a project data reset, are rejected.", "parameters": [ { "name": "id", "in": "path", - "description": "The identifier of the bulk operation.", + "description": "The identifier of the task.", "required": true, "schema": { "pattern": "^[a-zA-Z\\d]{24,36}$", @@ -4193,13 +4194,23 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ViewBulkOperation" + "$ref": "#/components/schemas/ViewBackgroundTask" } } } }, "404": { - "description": "The bulk operation could not be found.", + "description": "The task could not be found.", + "content": { + "application/problem\u002Bjson": { + "schema": { + "$ref": "#/components/schemas/ProblemDetails" + } + } + } + }, + "422": { + "description": "The task cannot be cancelled.", "content": { "application/problem\u002Bjson": { "schema": { @@ -12263,6 +12274,22 @@ } } }, + "BackgroundTaskStatus": { + "enum": [ + "Queued", + "Running", + "Completed", + "Failed", + "Cancelled" + ], + "x-enumNames": [ + "Queued", + "Running", + "Completed", + "Failed", + "Cancelled" + ] + }, "BillingPlan": { "required": [ "id", @@ -12342,22 +12369,6 @@ "Filter" ] }, - "BulkOperationStatus": { - "enum": [ - "Queued", - "Running", - "Completed", - "Failed", - "Cancelled" - ], - "x-enumNames": [ - "Queued", - "Running", - "Completed", - "Failed", - "Cancelled" - ] - }, "ChangePasswordModel": { "required": [ "current_password", @@ -14664,18 +14675,19 @@ } } }, - "ViewBulkOperation": { + "ViewBackgroundTask": { "required": [ "id", "organization_id", "type", "description", "status", - "selection", + "progress", "total", "processed", "succeeded", "failed", + "can_cancel", "created_by_user_id", "created_utc", "updated_utc" @@ -14704,19 +14716,23 @@ ] }, "type": { - "type": "string" + "type": "string", + "description": "What the task does, e.g. events.delete, projects.reset-data or projects.sample-data." }, "description": { "type": "string" }, "status": { - "$ref": "#/components/schemas/BulkOperationStatus" + "$ref": "#/components/schemas/BackgroundTaskStatus" }, - "selection": { - "$ref": "#/components/schemas/ViewBulkOperationSelection" + "progress": { + "type": "integer", + "description": "Percent complete, 0 to 100.", + "format": "int32" }, "total": { "type": "integer", + "description": "The item counts. All zero for tasks that only report a percent.", "format": "int64" }, "processed": { @@ -14731,6 +14747,21 @@ "type": "integer", "format": "int64" }, + "can_cancel": { + "type": "boolean", + "description": "Whether the task can be stopped with the cancel endpoint." + }, + "selection": { + "oneOf": [ + { + "type": "null" + }, + { + "description": "What a bulk action applies to. Only set for bulk actions.", + "$ref": "#/components/schemas/ViewBulkOperationSelection" + } + ] + }, "created_by_user_id": { "maxLength": 24, "minLength": 24, @@ -14809,10 +14840,11 @@ "null", "string" ], - "description": "The absolute end of the resolved time window, never later than when the operation was created.", + "description": "The absolute end of the resolved time window, never later than when the task was created.", "format": "date-time" } - } + }, + "description": "What a bulk action applies to. Only set for bulk actions." }, "ViewCurrentUser": { "required": [ @@ -15697,7 +15729,7 @@ "name": "SavedView" }, { - "name": "BulkOperation" + "name": "BackgroundTask" }, { "name": "User" @@ -15712,4 +15744,4 @@ "name": "Source Map" } ] -} +} \ No newline at end of file diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs index edc2307ffd..7f5f995cea 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -1,11 +1,13 @@ using System.Collections.Concurrent; using Exceptionless.Core.Messaging.Models; using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Queries.Validation; using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Base; +using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Core.Services.BulkOperations; using Exceptionless.Core.Utility; using Exceptionless.Tests.Extensions; @@ -18,6 +20,10 @@ using Foundatio.Repositories.Options; using Foundatio.Repositories.Utility; using Foundatio.Serializer; +using Foundatio.Caching; +using Foundatio.Lock; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; using Moq; using Xunit; @@ -57,12 +63,12 @@ public async Task Create_FilterMatchingTooManyStacks_IsRejectedWithTheRealMessag var result = await service.CreateAsync(NewRecordingOperation("events.delete")); // Assert - Assert.Null(result.Operation); + Assert.Null(result.Task); Assert.NotNull(result.Rejection); Assert.Equal(BulkOperationCheckStatus.Invalid, result.Rejection.Status); Assert.Equal("selection.filter", result.Rejection.Field); Assert.Equal(message, result.Rejection.Message); - Assert.Empty(await service.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); } [Fact] @@ -99,31 +105,31 @@ public async Task Create_MoreThanFiveHundredOperations_StillSeesTheNewestActiveO // Arrange var service = CreateService(new RecordingHandler(), new RecordingHandler("test.destructive", true)); for (int index = 0; index < 500; index++) - Assert.NotNull((await service.CreateAsync(NewIdsOperation("test.record"))).Operation); + Assert.NotNull((await service.CreateAsync(NewIdsOperation("test.record"))).Task); // Act var newest = await service.CreateAsync(NewIdsOperation("test.destructive")); var blocked = await service.CreateAsync(NewIdsOperation("test.destructive")); - var list = await service.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID); + var list = await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID); // Assert - Assert.NotNull(newest.Operation); - Assert.Null(blocked.Operation); - Assert.Equal(newest.Operation.Id, blocked.Conflict?.Id); - Assert.Equal(BulkOperationService.ListLimit, list.Count); - Assert.Equal(newest.Operation.Id, list[0].Id); + Assert.NotNull(newest.Task); + Assert.Null(blocked.Task); + Assert.Equal(newest.Task.Id, blocked.Conflict?.Id); + Assert.Equal(BackgroundTaskService.ListLimit, list.Count); + Assert.Equal(newest.Task.Id, list[0].Id); } [Fact] public async Task Execute_RecordMissingWhenTheJobRuns_PublishesAFinalFailedUpdate() { // Arrange - var messages = new ConcurrentQueue(); - await GetService().SubscribeAsync(m => messages.Enqueue(m), TestCancellationToken); + var messages = new ConcurrentQueue(); + await GetService().SubscribeAsync(m => messages.Enqueue(m), TestCancellationToken); var workItem = new BulkOperationWorkItem { Description = "Delete 7 events matching filter", - OperationId = ObjectId.GenerateNewId().ToString(), + TaskId = ObjectId.GenerateNewId().ToString(), OrganizationId = SampleDataService.TEST_ORG_ID, Type = "events.delete" }; @@ -134,10 +140,10 @@ public async Task Execute_RecordMissingWhenTheJobRuns_PublishesAFinalFailedUpdat // Assert var update = Assert.Single(messages); - Assert.Equal(workItem.OperationId, update.Id); + Assert.Equal(workItem.TaskId, update.Id); Assert.Equal(SampleDataService.TEST_ORG_ID, update.OrganizationId); Assert.Equal("Delete 7 events matching filter", update.Description); - Assert.Equal(BulkOperationStatus.Failed, update.Status); + Assert.Equal(BackgroundTaskStatus.Failed, update.Status); Assert.False(String.IsNullOrEmpty(update.Message)); } @@ -160,20 +166,20 @@ public async Task Execute_TemporaryErrorMidOperation_IsRetriedFromTheSavedCursor // Act var thrown = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); long continuations = await GetEnqueuedWorkItemCountAsync() - enqueuedBefore; - var interrupted = await service.GetAsync(created.Id); + var interrupted = await _taskService.GetAsync(created.Id); await RunOperationAsync(service, created.Id, 2, TestCancellationToken); - var finished = await service.GetAsync(created.Id); + var finished = await _taskService.GetAsync(created.Id); // Assert Assert.Null(thrown); Assert.Equal(1, continuations); Assert.NotNull(interrupted); - Assert.Equal(BulkOperationStatus.Running, interrupted.Status); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); Assert.Equal(2, interrupted.Processed); - Assert.Equal(1, interrupted.RetryCount); + Assert.Equal(1, interrupted.Bulk!.RetryCount); Assert.NotNull(finished); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); - Assert.Equal(0, finished.RetryCount); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); + Assert.Equal(0, finished.Bulk!.RetryCount); Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); } @@ -190,15 +196,15 @@ public async Task Execute_TemporaryErrorsKeepHappening_FailsAfterTheRetriesAreUs var first = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); var second = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); var third = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); - var finished = await service.GetAsync(created.Id); + var finished = await _taskService.GetAsync(created.Id); // Assert Assert.Null(first); Assert.Null(second); Assert.Null(third); Assert.NotNull(finished); - Assert.Equal(BulkOperationStatus.Failed, finished.Status); - Assert.Equal(3, finished.RetryCount); + Assert.Equal(BackgroundTaskStatus.Failed, finished.Status); + Assert.Equal(3, finished.Bulk!.RetryCount); } [Fact] @@ -219,17 +225,17 @@ public async Task Execute_TemporaryErrorsBetweenProgress_ContinueInNewRunsUntilD var thrown = new List(); // Act - for (int run = 0; run < 10 && (await service.GetAsync(created.Id))!.IsActive; run++) + for (int run = 0; run < 10 && (await _taskService.GetAsync(created.Id))!.IsActive; run++) thrown.Add(await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 1, TestCancellationToken))); long continuations = await GetEnqueuedWorkItemCountAsync() - enqueuedBefore; - var finished = await service.GetAsync(created.Id); + var finished = await _taskService.GetAsync(created.Id); // Assert Assert.All(thrown, Assert.Null); Assert.Equal(thrown.Count - 1, continuations); Assert.NotNull(finished); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); } @@ -246,28 +252,28 @@ public async Task Execute_RestartAfterTheLastIdBatchBeforeCompletion_DoesNotStar var recorder = new RecordingHandler(); var service = CreateService(recorder); var operation = NewRecordingOperation("test.record"); - operation.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = events.Select(e => e.Id).ToList() }; + operation.Bulk!.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = events.Select(e => e.Id).ToList() }; var result = await service.CreateAsync(operation); - var created = result.Operation; + var created = result.Task; Assert.NotNull(created); await RunOperationAsync(service, created.Id, 2, TestCancellationToken); int batchesBefore = recorder.Batches.Count; // The worker stopped after saving the last batch but before saving the completed status. - await service.UpdateAsync(created.Id, current => + await _taskService.UpdateAsync(created.Id, current => { - current.Status = BulkOperationStatus.Running; + current.Status = BackgroundTaskStatus.Running; current.CompletedUtc = null; return true; }); // Act await RunOperationAsync(service, created.Id, 2, TestCancellationToken); - var finished = await service.GetAsync(created.Id); + var finished = await _taskService.GetAsync(created.Id); // Assert Assert.Equal(batchesBefore, recorder.Batches.Count); Assert.NotNull(finished); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(5, finished.Processed); } @@ -279,27 +285,29 @@ public async Task Create_WhenTheQueueRejectsTheWorkItem_FailsTheOperationSoItDoe var unavailableQueue = new Mock>(); unavailableQueue.SetupGet(q => q.Serializer).Returns(GetService>().Serializer); unavailableQueue.Setup(q => q.EnqueueAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new TimeoutException("Queue unavailable")); + var failingTasks = new BackgroundTaskService(GetService(), GetService(), GetService(), unavailableQueue.Object, + GetService(), new BackgroundTaskOptions(), TimeProvider, GetService>()); var recorder = new RecordingHandler("test.destroy", isDestructive: true); - var failingService = CreateService([recorder], [GetService()], unavailableQueue.Object); + var failingService = CreateService([recorder], [GetService()], failingTasks); var service = CreateService([recorder], [GetService()]); // Act var thrown = await Record.ExceptionAsync(async () => await failingService.CreateAsync(NewRecordingOperation("test.destroy"))); - var orphan = (await service.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)).Single(); + var orphan = (await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)).Single(); var retry = await service.CreateAsync(NewRecordingOperation("test.destroy")); // Assert Assert.IsType(thrown); - Assert.Equal(BulkOperationStatus.Failed, orphan.Status); - Assert.NotNull(retry.Operation); - Assert.Equal(BulkOperationStatus.Queued, retry.Operation.Status); + Assert.Equal(BackgroundTaskStatus.Failed, orphan.Status); + Assert.NotNull(retry.Task); + Assert.Equal(BackgroundTaskStatus.Queued, retry.Task.Status); } [Fact] public void Handler_RenewsTheQueueEntryAndLockOnEveryBatch() { // Arrange - var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(CreateService(new RecordingHandler()), [GetService()], + var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(CreateService(new RecordingHandler()), _taskService, [GetService()], new BulkOperationOptions(), GetService(), TimeProvider, Log); // Act @@ -325,13 +333,13 @@ public async Task Execute_ErrorThatRetryingCannotFix_FailsImmediately() // Act var thrown = await Record.ExceptionAsync(() => RunOperationAsync(service, created.Id, 2, TestCancellationToken)); - var finished = await service.GetAsync(created.Id); + var finished = await _taskService.GetAsync(created.Id); // Assert Assert.Null(thrown); Assert.NotNull(finished); - Assert.Equal(BulkOperationStatus.Failed, finished.Status); - Assert.Equal(0, finished.RetryCount); + Assert.Equal(BackgroundTaskStatus.Failed, finished.Status); + Assert.Equal(0, finished.Bulk!.RetryCount); } [Fact] @@ -347,25 +355,25 @@ public async Task Execute_JobRestartsMidDelete_ResumesAndDeletesEachEventExactly var deleteHandler = new InterceptingHandler(GetService()); var service = CreateService(deleteHandler); var result = await service.CreateAsync(NewRecordingOperation("events.delete")); - Assert.NotNull(result.Operation); + Assert.NotNull(result.Task); using var shutdown = new CancellationTokenSource(); deleteHandler.AfterBatch = () => shutdown.Cancel(); // Act - await RunOperationAsync(service, result.Operation.Id, 2, shutdown.Token); - var interrupted = await service.GetAsync(result.Operation.Id); + await RunOperationAsync(service, result.Task.Id, 2, shutdown.Token); + var interrupted = await _taskService.GetAsync(result.Task.Id); var remainingWhileInterrupted = await GetRemainingMessagesAsync(events); deleteHandler.AfterBatch = null; - await RunOperationAsync(service, result.Operation.Id, 2, TestCancellationToken); - var finished = await service.GetAsync(result.Operation.Id); + await RunOperationAsync(service, result.Task.Id, 2, TestCancellationToken); + var finished = await _taskService.GetAsync(result.Task.Id); // Assert Assert.NotNull(interrupted); - Assert.Equal(BulkOperationStatus.Running, interrupted.Status); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); Assert.Equal(2, interrupted.Processed); Assert.Equal(3, remainingWhileInterrupted.Count); Assert.NotNull(finished); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(5, finished.Processed); Assert.Equal(5, finished.Succeeded); Assert.Equal(0, finished.Failed); @@ -378,19 +386,19 @@ public async Task Cancel_RunningOperationThatStoppedReportingProgress_CancelsImm // Arrange var service = CreateService(new RecordingHandler()); var created = await CreateRecordingOperationAsync(service); - await service.UpdateAsync(created.Id, operation => + await _taskService.UpdateAsync(created.Id, operation => { - operation.Status = BulkOperationStatus.Running; + operation.Status = BackgroundTaskStatus.Running; return true; }); - TimeProvider.Advance(BulkOperationService.StaleAfter + TimeSpan.FromMinutes(1)); + TimeProvider.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); // Act - var cancelled = await service.CancelAsync(created.Id); + var cancelled = (await _taskService.CancelAsync(created.Id)).Task; // Assert Assert.NotNull(cancelled); - Assert.Equal(BulkOperationStatus.Cancelled, cancelled.Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, cancelled.Status); Assert.NotNull(cancelled.CompletedUtc); } @@ -402,12 +410,12 @@ public async Task Cancel_FinishedOperation_ChangesNothing() await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); // Act - var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("bulk-operations", created.Id, "cancel").StatusCodeShouldBeAccepted()); + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", created.Id, "cancel").StatusCodeShouldBeAccepted()); // Assert - Assert.Equal(BulkOperationStatus.Completed, response?.Status); - Assert.Equal(BulkOperationStatus.Completed, (await GetOperationAsync(created.Id)).Status); - Assert.False(await _bulkOperationService.IsCancelRequestedAsync(created.Id)); + Assert.Equal(BackgroundTaskStatus.Completed, response?.Status); + Assert.Equal(BackgroundTaskStatus.Completed, (await GetOperationAsync(created.Id)).Status); + Assert.False(await _taskService.IsCancelRequestedAsync(created.Id)); } private EventBulkSelectionResolver CreateResolver(IEventRepository eventRepository) @@ -416,10 +424,10 @@ private EventBulkSelectionResolver CreateResolver(IEventRepository eventReposito GetService(), GetService(), GetService()); } - private BulkOperation NewIdsOperation(string type) + private BackgroundTask NewIdsOperation(string type) { var operation = NewRecordingOperation(type); - operation.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [SampleDataService.TEST_ORG_ID] }; + operation.Bulk!.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [SampleDataService.TEST_ORG_ID] }; return operation; } @@ -434,9 +442,9 @@ private sealed class InterceptingHandler(IBulkOperationHandler inner) : IBulkOpe public Task AuthorizeAsync(BulkOperationRequest request) => inner.AuthorizeAsync(request); public Task ValidateAsync(BulkOperationRequest request) => inner.ValidateAsync(request); - public string Describe(BulkOperation operation) => inner.Describe(operation); + public string Describe(BackgroundTask operation) => inner.Describe(operation); - public async Task ExecuteBatchAsync(BulkOperation operation, IReadOnlyCollection ids, CancellationToken cancellationToken) + public async Task ExecuteBatchAsync(BackgroundTask operation, IReadOnlyCollection ids, CancellationToken cancellationToken) { var result = await inner.ExecuteBatchAsync(operation, ids, cancellationToken); AfterBatch?.Invoke(); diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index 131740da00..d663cb6262 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -1,10 +1,12 @@ using System.Net; using Exceptionless.Core.Messaging.Models; using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Repositories; using Exceptionless.Core.Services; +using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Core.Services.BulkOperations; using Exceptionless.Core.Utility; using Exceptionless.Tests.Extensions; @@ -29,12 +31,14 @@ public partial class BulkOperationEndpointTests : IntegrationTestsBase { private readonly IEventRepository _eventRepository; private readonly BulkOperationService _bulkOperationService; + private readonly BackgroundTaskService _taskService; private readonly WorkItemJob _workItemJob; public BulkOperationEndpointTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { _eventRepository = GetService(); _bulkOperationService = GetService(); + _taskService = GetService(); _workItemJob = GetService(); } @@ -62,11 +66,11 @@ public async Task Create_FilterSelection_DeletesEverythingMatchingThroughTheQueu var finished = await GetOperationAsync(created.Id); // Assert - Assert.Equal(BulkOperationStatus.Queued, created.Status); + Assert.Equal(BackgroundTaskStatus.Queued, created.Status); Assert.Equal("events.delete", created.Type); Assert.Equal(3, created.Total); Assert.Equal("Delete 3 events matching filter", created.Description); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(3, finished.Processed); Assert.Equal(3, finished.Succeeded); Assert.Equal(0, finished.Failed); @@ -95,7 +99,7 @@ public async Task Create_FilterWithRelativeDates_PinsThemAndDeletesOnlyTheMatche Assert.DoesNotContain("now", pinned, StringComparison.OrdinalIgnoreCase); Assert.Contains("||-2h", pinned); Assert.Equal(1, created.Total); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(1, finished.Succeeded); Assert.Equal(["outside"], (await GetRemainingMessagesAsync(events)).ToArray()); } @@ -161,10 +165,10 @@ public async Task Create_FilterSelection_ResolvesTimeWindowToAbsoluteDatesCapped await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); // Assert - Assert.NotNull(created.Selection.UtcEnd); - Assert.True(created.Selection.UtcEnd <= created.CreatedUtc, "The window end must not be later than the moment the operation was created."); - Assert.True(created.Selection.UtcStart < created.Selection.UtcEnd); - Assert.Equal(created.Selection.UtcStart, reloaded.Selection.UtcStart); + Assert.NotNull(created.Selection!.UtcEnd); + Assert.True(created.Selection!.UtcEnd <= created.CreatedUtc, "The window end must not be later than the moment the operation was created."); + Assert.True(created.Selection!.UtcStart < created.Selection!.UtcEnd); + Assert.Equal(created.Selection!.UtcStart, reloaded.Selection!.UtcStart); Assert.Equal(created.Selection.UtcEnd, reloaded.Selection.UtcEnd); Assert.Equal(["future dated"], (await GetRemainingMessagesAsync(events)).ToArray()); } @@ -184,7 +188,7 @@ public async Task Create_FilterSelection_DoesNotIncludeEventsStoredAfterCreation // Assert var finished = await GetOperationAsync(created.Id); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(1, finished.Succeeded); Assert.NotNull(await _eventRepository.GetByIdAsync(lateEvents.Single().Id)); } @@ -211,7 +215,7 @@ public async Task Create_IdSelection_DeletesTheIdsThroughTheSameSideEffectsAsThe // Assert var finished = await GetOperationAsync(created.Id); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(2, finished.Succeeded); Assert.Null(await _eventRepository.GetByIdAsync(events[0].Id)); Assert.Null(await _eventRepository.GetByIdAsync(events[1].Id)); @@ -292,8 +296,8 @@ await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPath("bulk-o .Content(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")).StatusCodeShouldBeNotFound()); // Assert - Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); - Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); } [Fact] @@ -307,7 +311,7 @@ public async Task Create_WithoutAUserSession_IsRejected() await SendRequestAsync(r => r.Post().AsTestOrganizationClientUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeForbidden()); // Assert - Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); } [Fact] @@ -317,7 +321,7 @@ public async Task Create_GlobalAdmin_CanStartOperationForAnyOrganization() var request = FilterRequest(SampleDataService.FREE_ORG_ID, null, ""); // Act - var created = await SendRequestAsAsync(r => r.Post().AsGlobalAdminUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeAccepted()); + var created = await SendRequestAsAsync(r => r.Post().AsGlobalAdminUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeAccepted()); // Assert Assert.NotNull(created); @@ -331,14 +335,14 @@ public async Task Get_OperationOfAnotherOrganization_IsNotFoundForListGetAndCanc var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); // Act - await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("bulk-operations", created.Id).StatusCodeShouldBeNotFound()); - await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("bulk-operations", created.Id, "cancel").StatusCodeShouldBeNotFound()); - await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "bulk-operations").StatusCodeShouldBeNotFound()); - var asAdmin = await SendRequestAsAsync(r => r.AsGlobalAdminUser().AppendPaths("bulk-operations", created.Id).StatusCodeShouldBeOk()); + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("tasks", created.Id).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("tasks", created.Id, "cancel").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "tasks").StatusCodeShouldBeNotFound()); + var asAdmin = await SendRequestAsAsync(r => r.AsGlobalAdminUser().AppendPaths("tasks", created.Id).StatusCodeShouldBeOk()); // Assert Assert.Equal(created.Id, asAdmin?.Id); - Assert.Equal(BulkOperationStatus.Queued, (await GetOperationAsync(created.Id)).Status); + Assert.Equal(BackgroundTaskStatus.Queued, (await GetOperationAsync(created.Id)).Status); } [Fact] @@ -351,14 +355,14 @@ public async Task GetByOrganization_ReturnsActiveAndRecentOperationsNewestFirst( var second = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); // Act - var list = await SendRequestAsAsync>(r => r.AsTestOrganizationUser() - .AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "bulk-operations").StatusCodeShouldBeOk()); + var list = await SendRequestAsAsync>(r => r.AsTestOrganizationUser() + .AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "tasks").StatusCodeShouldBeOk()); // Assert Assert.NotNull(list); Assert.Equal([second.Id, first.Id], list.Select(o => o.Id).ToArray()); - Assert.Equal(BulkOperationStatus.Queued, list.First().Status); - Assert.Equal(BulkOperationStatus.Completed, list.Last().Status); + Assert.Equal(BackgroundTaskStatus.Queued, list.First().Status); + Assert.Equal(BackgroundTaskStatus.Completed, list.Last().Status); } [Fact] @@ -389,12 +393,12 @@ public async Task Create_AfterTheActiveOperationFinishesOrIsCancelled_IsAllowedA var second = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); // Act - var cancelled = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("bulk-operations", second.Id, "cancel").StatusCodeShouldBeAccepted()); + var cancelled = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", second.Id, "cancel").StatusCodeShouldBeAccepted()); var third = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); // Assert - Assert.Equal(BulkOperationStatus.Completed, (await GetOperationAsync(first.Id)).Status); - Assert.Equal(BulkOperationStatus.Cancelled, cancelled?.Status); + Assert.Equal(BackgroundTaskStatus.Completed, (await GetOperationAsync(first.Id)).Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, cancelled?.Status); Assert.NotEqual(second.Id, third.Id); } @@ -405,14 +409,14 @@ public async Task Create_WhileTheActiveOperationHasStoppedReportingProgress_Reti var stale = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); // Act - TimeProvider.Advance(BulkOperationService.StaleAfter + TimeSpan.FromMinutes(1)); + TimeProvider.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); var next = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID, "")); var retired = await GetOperationAsync(stale.Id); // Assert - Assert.Equal(BulkOperationStatus.Queued, next.Status); + Assert.Equal(BackgroundTaskStatus.Queued, next.Status); // A delayed delivery of the stale operation must not run alongside its replacement. - Assert.Equal(BulkOperationStatus.Failed, retired.Status); + Assert.Equal(BackgroundTaskStatus.Failed, retired.Status); } [Fact] @@ -440,7 +444,7 @@ public async Task Create_InvalidRequests_AreRejectedWithoutStartingAnything() await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeUnprocessableEntity()); // Assert - Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); } [Fact] @@ -453,7 +457,7 @@ public async Task Create_PremiumFilterOnFreePlan_RequiresUpgrade() await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeUpgradeRequired()); // Assert - Assert.Empty(await _bulkOperationService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); } [Fact] @@ -464,12 +468,12 @@ public async Task Cancel_QueuedOperation_CancelsImmediatelyAndTheJobDoesNothing( var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); // Act - var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("bulk-operations", created.Id, "cancel").StatusCodeShouldBeAccepted()); + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", created.Id, "cancel").StatusCodeShouldBeAccepted()); await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); // Assert - Assert.Equal(BulkOperationStatus.Cancelled, response?.Status); - Assert.Equal(BulkOperationStatus.Cancelled, (await GetOperationAsync(created.Id)).Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, response?.Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, (await GetOperationAsync(created.Id)).Status); Assert.NotNull(await _eventRepository.GetByIdAsync(events.Single().Id)); } @@ -483,27 +487,27 @@ public async Task Execute_MoreMatchesThanOneBatch_ProcessesEveryBatchAndPublishe for (int i = 0; i < 5; i++) d.Event().TestProject().Date(now.AddMinutes(-i - 1)); }); - var messages = new System.Collections.Concurrent.ConcurrentQueue(); - await GetService().SubscribeAsync(m => messages.Enqueue(m), TestCancellationToken); + var messages = new System.Collections.Concurrent.ConcurrentQueue(); + await GetService().SubscribeAsync(m => messages.Enqueue(m), TestCancellationToken); var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "")); // Act await RunOperationAsync(_bulkOperationService, created.Id, batchSize: 2, TestCancellationToken); - await WaitForAsync(() => messages.Any(m => m.Status == BulkOperationStatus.Completed)); + await WaitForAsync(() => messages.Any(m => m.Status == BackgroundTaskStatus.Completed)); // Assert var finished = await GetOperationAsync(created.Id); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(5, finished.Processed); Assert.Equal(5, finished.Succeeded); Assert.Empty(await GetRemainingMessagesAsync(events)); var forOperation = messages.Where(m => m.Id == created.Id).ToList(); - Assert.Contains(forOperation, m => m.Status == BulkOperationStatus.Running && m.Processed == 2); - Assert.Contains(forOperation, m => m.Status == BulkOperationStatus.Running && m.Processed == 4); - Assert.Contains(forOperation, m => m.Status == BulkOperationStatus.Running && m.Processed == 5); + Assert.Contains(forOperation, m => m.Status == BackgroundTaskStatus.Running && m.Processed == 2); + Assert.Contains(forOperation, m => m.Status == BackgroundTaskStatus.Running && m.Processed == 4); + Assert.Contains(forOperation, m => m.Status == BackgroundTaskStatus.Running && m.Processed == 5); var last = forOperation.OrderBy(m => m.UpdatedUtc).ThenBy(m => m.Processed).Last(); - Assert.Equal(BulkOperationStatus.Completed, last.Status); + Assert.Equal(BackgroundTaskStatus.Completed, last.Status); Assert.All(forOperation, m => Assert.Equal(SampleDataService.TEST_ORG_ID, m.OrganizationId)); } @@ -525,18 +529,18 @@ public async Task Execute_JobRestartsMidOperation_ResumesAfterTheSavedCursorWith // Act await RunOperationAsync(service, created.Id, batchSize: 2, shutdown.Token); - var interrupted = await service.GetAsync(created.Id); + var interrupted = await _taskService.GetAsync(created.Id); recorder.AfterBatch = null; await RunOperationAsync(service, created.Id, batchSize: 2, TestCancellationToken); - var finished = await service.GetAsync(created.Id); + var finished = await _taskService.GetAsync(created.Id); // Assert Assert.NotNull(interrupted); - Assert.Equal(BulkOperationStatus.Running, interrupted.Status); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); Assert.Equal(2, interrupted.Processed); - Assert.NotNull(interrupted.Cursor); + Assert.NotNull(interrupted.Bulk!.Cursor); Assert.NotNull(finished); - Assert.Equal(BulkOperationStatus.Completed, finished.Status); + Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(5, finished.Processed); Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), recorder.Batches.SelectMany(b => b).ToArray()); Assert.Equal([2, 2, 1], recorder.Batches.Select(b => b.Count).ToArray()); @@ -555,15 +559,15 @@ await CreateDataAsync(d => var recorder = new RecordingHandler(); var service = CreateService(recorder); var created = await CreateRecordingOperationAsync(service); - recorder.AfterBatch = () => service.CancelAsync(created.Id).GetAwaiter().GetResult(); + recorder.AfterBatch = () => _taskService.CancelAsync(created.Id).GetAwaiter().GetResult(); // Act await RunOperationAsync(service, created.Id, batchSize: 2, TestCancellationToken); // Assert - var finished = await service.GetAsync(created.Id); + var finished = await _taskService.GetAsync(created.Id); Assert.NotNull(finished); - Assert.Equal(BulkOperationStatus.Cancelled, finished.Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, finished.Status); Assert.Equal(2, finished.Processed); Assert.NotNull(finished.CompletedUtc); Assert.Single(recorder.Batches); @@ -583,15 +587,15 @@ public async Task Execute_NonDestructiveOperationsDoNotBlockEachOtherOrDestructi var blocked = await service.CreateAsync(NewRecordingOperation("test.destructive")); // Assert - Assert.NotNull(first.Operation); - Assert.NotNull(second.Operation); - Assert.NotNull(destructive.Operation); - Assert.Null(blocked.Operation); - Assert.Equal(destructive.Operation.Id, blocked.Conflict?.Id); + Assert.NotNull(first.Task); + Assert.NotNull(second.Task); + Assert.NotNull(destructive.Task); + Assert.Null(blocked.Task); + Assert.Equal(destructive.Task.Id, blocked.Conflict?.Id); } [Fact] - public async Task OnBulkOperationUpdatedAsync_SendsOnlyToTheOperationsOrganization() + public async Task OnBackgroundTaskUpdatedAsync_SendsOnlyToTheOperationsOrganization() { // Arrange var broker = GetService(); @@ -607,13 +611,13 @@ public async Task OnBulkOperationUpdatedAsync_SendsOnlyToTheOperationsOrganizati try { // Act - await broker.OnBulkOperationUpdatedAsync(new BulkOperationUpdated + await broker.OnBackgroundTaskUpdatedAsync(new BackgroundTaskUpdated { Id = "537650f3b77efe23a47914f9", OrganizationId = SampleDataService.TEST_ORG_ID, Type = "events.delete", Description = "Delete events matching filter", - Status = BulkOperationStatus.Running, + Status = BackgroundTaskStatus.Running, Total = 10, Processed = 4 }, TestCancellationToken); @@ -622,7 +626,7 @@ await broker.OnBulkOperationUpdatedAsync(new BulkOperationUpdated // Assert string sent = Assert.Single(ownSocket.SentMessages); - Assert.Contains("\"type\":\"BulkOperationUpdated\"", sent); + Assert.Contains("\"type\":\"BackgroundTaskUpdated\"", sent); Assert.Contains("\"processed\":4", sent); Assert.Empty(otherSocket.SentMessages); } @@ -646,9 +650,9 @@ private static NewBulkOperation FilterRequest(string organizationId, string? pro }; } - private async Task CreateOperationAsync(NewBulkOperation request, Action? authorize = null) + private async Task CreateOperationAsync(NewBulkOperation request, Action? authorize = null) { - var created = await SendRequestAsAsync(r => + var created = await SendRequestAsAsync(r => { r.Post().AppendPath("bulk-operations").Content(request).StatusCodeShouldBeAccepted(); if (authorize is null) @@ -661,9 +665,9 @@ private async Task CreateOperationAsync(NewBulkOperation requ return created; } - private async Task GetOperationAsync(string id) + private async Task GetOperationAsync(string id) { - var operation = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("bulk-operations", id).StatusCodeShouldBeOk()); + var operation = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", id).StatusCodeShouldBeOk()); Assert.NotNull(operation); return operation; } @@ -687,48 +691,50 @@ private BulkOperationService CreateService(params IBulkOperationHandler[] handle return CreateService(handlers, [GetService()]); } - private BulkOperationService CreateService(IBulkOperationHandler[] handlers, IBulkSelectionResolver[] resolvers, IQueue? workItemQueue = null) + private BulkOperationService CreateService(IBulkOperationHandler[] handlers, IBulkSelectionResolver[] resolvers, BackgroundTaskService? taskService = null) { - return new BulkOperationService(GetService(), GetService(), GetService(), workItemQueue ?? GetService>(), - handlers, resolvers, TimeProvider, GetService>()); + return new BulkOperationService(taskService ?? _taskService, GetService(), GetService(), handlers, resolvers, TimeProvider); } - private async Task CreateRecordingOperationAsync(BulkOperationService service) + private async Task CreateRecordingOperationAsync(BulkOperationService service) { var result = await service.CreateAsync(NewRecordingOperation("test.record")); - Assert.NotNull(result.Operation); - return result.Operation; + Assert.NotNull(result.Task); + return result.Task; } - private BulkOperation NewRecordingOperation(string type) + private BackgroundTask NewRecordingOperation(string type) { var now = TimeProvider.GetUtcNow().UtcDateTime; - return new BulkOperation + return new BackgroundTask { OrganizationId = SampleDataService.TEST_ORG_ID, ProjectId = SampleDataService.TEST_PROJECT_ID, Type = type, Description = "Record events", CreatedByUserId = "537650f3b77efe23a47914f8", - Selection = new BulkOperationSelection + Bulk = new BulkOperationDetails { - Kind = BulkOperationSelectionKind.Filter, - Filter = String.Empty, - UtcStart = now.AddDays(-7), - UtcEnd = now, - UtcCreatedBefore = now + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Filter, + Filter = String.Empty, + UtcStart = now.AddDays(-7), + UtcEnd = now, + UtcCreatedBefore = now + } } }; } private Task RunOperationAsync(BulkOperationService service, string operationId, int batchSize, CancellationToken shutdownToken) { - return RunWorkItemAsync(service, new BulkOperationWorkItem { OperationId = operationId, OrganizationId = SampleDataService.TEST_ORG_ID }, batchSize, shutdownToken); + return RunWorkItemAsync(service, new BulkOperationWorkItem { TaskId = operationId, OrganizationId = SampleDataService.TEST_ORG_ID }, batchSize, shutdownToken); } private Task RunWorkItemAsync(BulkOperationService service, BulkOperationWorkItem workItem, int batchSize, CancellationToken shutdownToken) { - var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(service, [GetService()], + var handler = new Exceptionless.Core.Jobs.WorkItemHandlers.BulkOperationWorkItemHandler(service, _taskService, [GetService()], new BulkOperationOptions { BatchSize = batchSize, RetryDelay = TimeSpan.Zero }, GetService(), TimeProvider, Log); var context = new WorkItemContext(workItem, "test-job", EmptyLock.Empty, shutdownToken, static (_, _) => Task.CompletedTask); @@ -751,9 +757,9 @@ private sealed class RecordingHandler(string type = "test.record", bool isDestru public Task AuthorizeAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); public Task ValidateAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); - public string Describe(BulkOperation operation) => "Record events"; + public string Describe(BackgroundTask operation) => "Record events"; - public Task ExecuteBatchAsync(BulkOperation operation, IReadOnlyCollection ids, CancellationToken cancellationToken) + public Task ExecuteBatchAsync(BackgroundTask operation, IReadOnlyCollection ids, CancellationToken cancellationToken) { var failure = Failure?.Invoke(++_calls); if (failure is not null) From 18d15d1990c4e01bfac276f0280d37f11de860c9 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 3 Nov 2026 23:17:26 -0600 Subject: [PATCH 12/42] Show every organization task in the progress panel Rename the bulk operations feature to tasks and read the generic task endpoints and BackgroundTaskUpdated messages. Percent-only tasks show their percent and message, Cancel only appears when the task can be cancelled, and Reset data and Generate sample data refresh the panel. --- .../e2e/tests/bulk-delete-matching.e2e.ts | 2 +- .../features/bulk-operations/api.svelte.ts | 114 ------------- .../components/bulk-operation-item.svelte | 82 --------- .../bulk-operation-item.svelte.test.ts | 132 --------------- .../components/bulk-operation-panel.svelte | 93 ----------- .../bulk-operation-panel.svelte.test.ts | 150 ----------------- .../features/bulk-operations/models.test.ts | 134 --------------- .../lib/features/bulk-operations/models.ts | 67 -------- .../events-bulk-actions-dropdown-menu.svelte | 4 +- ...-bulk-actions-dropdown-menu.svelte.test.ts | 2 +- .../src/lib/features/projects/api.svelte.ts | 13 +- .../src/lib/features/tasks/api.svelte.ts | 121 ++++++++++++++ .../{bulk-operations => tasks}/api.test.ts | 36 ++-- .../tasks/components/task-item.svelte | 97 +++++++++++ .../tasks/components/task-item.svelte.test.ts | 158 ++++++++++++++++++ .../tasks/components/task-panel.svelte | 93 +++++++++++ .../components/task-panel.svelte.test.ts | 147 ++++++++++++++++ .../src/lib/features/tasks/models.test.ts | 139 +++++++++++++++ .../src/lib/features/tasks/models.ts | 74 ++++++++ .../src/lib/features/websockets/models.ts | 16 +- .../ClientApp/src/routes/(app)/+layout.svelte | 4 +- 21 files changed, 875 insertions(+), 803 deletions(-) delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.svelte.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.test.ts delete mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts rename src/Exceptionless.Web/ClientApp/src/lib/features/{bulk-operations => tasks}/api.test.ts (67%) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts index ea074f91e0..8d6ff533c1 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts @@ -76,7 +76,7 @@ test('deleting everything matching the filter runs in the background and shows p .click(); // Assert - const panel = page.getByRole('region', { name: 'Bulk operations' }); + const panel = page.getByRole('region', { name: 'Background tasks' }); await expect(panel).toContainText(/Delete \d+ events matching filter/, { timeout: 30_000 }); await expect(panel).toContainText(`Completed · ${EVENT_COUNT} processed`, { timeout: 60_000 }); await expect(table.getByRole('checkbox', { name: 'Select row' })).toHaveCount(0, { timeout: 30_000 }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.svelte.ts deleted file mode 100644 index 6cf26ede53..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.svelte.ts +++ /dev/null @@ -1,114 +0,0 @@ -import type { QueryClient } from '@tanstack/svelte-query'; - -import { type ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; -import { createMutation, createQuery, useQueryClient } from '@tanstack/svelte-query'; - -import type { BulkOperationUpdated } from '#features/websockets/models.ts'; - -import { accessToken } from '#features/auth/index.svelte.ts'; -import { queryKeys as eventQueryKeys } from '#features/events/api.svelte.ts'; -import { queryKeys as stackQueryKeys } from '#features/stacks/api.svelte.ts'; - -import { applyBulkOperationUpdate, isBulkOperationActive, type NewBulkOperation, type ViewBulkOperation } from './models'; - -/** While an operation is active the list is polled as well, so progress still moves when push messages are unavailable. */ -export const ACTIVE_OPERATION_POLL_INTERVAL_MS = 3000; - -export const queryKeys = { - cancel: (id: string | undefined) => [...queryKeys.type, id, 'cancel'] as const, - organization: (organizationId: string | undefined) => [...queryKeys.type, 'organization', organizationId] as const, - post: () => [...queryKeys.type, 'post'] as const, - type: ['BulkOperation'] as const -}; - -export interface GetBulkOperationsRequest { - route: { - organizationId: string | undefined; - }; -} - -export function cancelBulkOperation() { - const queryClient = useQueryClient(); - - return createMutation(() => ({ - enabled: () => !!accessToken.current, - mutationFn: async (id: string) => { - const client = useFetchClient(); - const response = await client.postJSON(`bulk-operations/${id}/cancel`); - - return response.data!; - }, - mutationKey: queryKeys.cancel(undefined), - onSuccess: (operation) => { - upsertBulkOperation(queryClient, operation); - } - })); -} - -export function getBulkOperationsQuery(request: GetBulkOperationsRequest) { - return createQuery(() => ({ - enabled: () => !!accessToken.current && !!request.route.organizationId, - queryFn: async ({ signal }: { signal: AbortSignal }) => { - const client = useFetchClient(); - const response = await client.getJSON(`organizations/${request.route.organizationId}/bulk-operations`, { - signal - }); - - return response.data ?? []; - }, - queryKey: queryKeys.organization(request.route.organizationId), - refetchInterval: (query) => (query.state.data?.some((operation) => isBulkOperationActive(operation.status)) ? ACTIVE_OPERATION_POLL_INTERVAL_MS : false) - })); -} - -/** Refreshes everything a finished bulk operation may have changed. */ -export async function invalidateBulkOperationAffectedQueries(queryClient: QueryClient) { - await Promise.all([ - queryClient.invalidateQueries({ - queryKey: eventQueryKeys.type - }), - queryClient.invalidateQueries({ - queryKey: stackQueryKeys.type - }) - ]); -} - -export function postBulkOperation() { - const queryClient = useQueryClient(); - - return createMutation(() => ({ - enabled: () => !!accessToken.current, - mutationFn: async (operation: NewBulkOperation) => { - const client = useFetchClient(); - const response = await client.postJSON('bulk-operations', operation); - - return response.data!; - }, - mutationKey: queryKeys.post(), - onSuccess: (operation) => { - upsertBulkOperation(queryClient, operation); - } - })); -} - -/** Applies a pushed update to the cached list. Unknown operations (started elsewhere) trigger a refetch instead. */ -export function updateBulkOperationFromMessage(queryClient: QueryClient, update: BulkOperationUpdated) { - const key = queryKeys.organization(update.organization_id); - const current = queryClient.getQueryData(key); - const next = current ? applyBulkOperationUpdate(current, update) : undefined; - if (!next) { - void queryClient.invalidateQueries({ - queryKey: key - }); - return; - } - - queryClient.setQueryData(key, next); -} - -function upsertBulkOperation(queryClient: QueryClient, operation: ViewBulkOperation) { - queryClient.setQueryData(queryKeys.organization(operation.organization_id), (current) => [ - operation, - ...(current ?? []).filter((existing) => existing.id !== operation.id) - ]); -} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte deleted file mode 100644 index 972c8be697..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte +++ /dev/null @@ -1,82 +0,0 @@ - - -
-
- -
- {announcement} -

{operation.description}

-

- {#if operation.status === BulkOperationStatus.Queued} - Waiting to start - {:else if operation.status === BulkOperationStatus.Running} - of - {:else if operation.status === BulkOperationStatus.Completed} - Completed · processed{#if operation.failed > 0}, failed{/if} - {:else if operation.status === BulkOperationStatus.Cancelled} - Cancelled after of - {:else} - Failed after of {#if operation.message}{` · ${operation.message}`}{/if} - {/if} -

-
- {#if isActive} - - {:else} - - {/if} -
- {#if isActive || operation.status === BulkOperationStatus.Completed} - - {/if} -
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts deleted file mode 100644 index 239d39277b..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-item.svelte.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { cleanup, fireEvent, render, screen } from '@testing-library/svelte'; -import { afterEach, describe, expect, it, vi } from 'vitest'; - -import { BulkOperationSelectionKind, BulkOperationStatus, type ViewBulkOperation } from '../models'; -import BulkOperationItem from './bulk-operation-item.svelte'; - -function createOperation(overrides: Partial = {}): ViewBulkOperation { - return { - created_by_user_id: 'user-id', - created_utc: '2026-01-01T10:00:00Z', - description: 'Delete 1,234 events matching filter', - failed: 0, - id: 'operation-1', - organization_id: 'organization-id', - processed: 500, - selection: { kind: BulkOperationSelectionKind.Filter }, - status: BulkOperationStatus.Running, - succeeded: 500, - total: 1234, - type: 'events.delete', - updated_utc: '2026-01-01T10:00:00Z', - ...overrides - }; -} - -function statusText(): string { - return (document.querySelector('[data-slot="bulk-operation-progress"]')?.textContent ?? '').replace(/\s+/g, ' ').trim(); -} - -describe('BulkOperationItem', () => { - afterEach(cleanup); - - it('shows the description, x of y progress and a Cancel button while running', async () => { - // Arrange - const onCancel = vi.fn(); - const operation = createOperation(); - render(BulkOperationItem, { props: { onCancel, onDismiss: vi.fn(), operation } }); - - // Act - await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); - - // Assert - expect(screen.getByText('Delete 1,234 events matching filter')).toBeTruthy(); - expect(statusText()).toBe('500 of 1,234'); - expect(screen.getByRole('progressbar').getAttribute('aria-valuenow')).toBe('41'); - expect(onCancel).toHaveBeenCalledWith(operation); - expect(screen.queryByRole('button', { name: /^Dismiss / })).toBeNull(); - }); - - it('names the operation on its buttons and announces only state changes, not every progress update', async () => { - // Arrange - const { rerender } = render(BulkOperationItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation({ processed: 100 }) } }); - const announcement = () => screen.getByRole('status').textContent; - const initial = announcement(); - - // Act - await rerender({ operation: createOperation({ processed: 900 }) }); - const afterProgress = announcement(); - await rerender({ operation: createOperation({ processed: 1234, status: BulkOperationStatus.Completed }) }); - - // Assert - expect(initial).toBe('Delete 1,234 events matching filter: in progress'); - expect(afterProgress).toBe(initial); - expect(announcement()).toBe('Delete 1,234 events matching filter: completed'); - expect(screen.getByRole('button', { name: 'Dismiss Delete 1,234 events matching filter' })).toBeTruthy(); - }); - - it('labels Cancel with the operation it cancels', () => { - // Arrange & Act - render(BulkOperationItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation() } }); - - // Assert - expect(screen.getByRole('button', { name: 'Cancel Delete 1,234 events matching filter' })).toBeTruthy(); - }); - - it('shows the queued state without a count', () => { - // Arrange & Act - render(BulkOperationItem, { - props: { onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation({ processed: 0, status: BulkOperationStatus.Queued }) } - }); - - // Assert - expect(statusText()).toBe('Waiting to start'); - expect(screen.getByRole('button', { name: /^Cancel / })).toBeTruthy(); - }); - - it('disables Cancel once cancellation was requested', () => { - // Arrange & Act - render(BulkOperationItem, { props: { isCancelling: true, onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation() } }); - - // Assert - expect((screen.getByRole('button', { name: /^Cancel / }) as HTMLButtonElement).disabled).toBe(true); - }); - - it('shows a completed operation with its failures and a Dismiss button', async () => { - // Arrange - const onDismiss = vi.fn(); - const operation = createOperation({ failed: 2, processed: 1234, status: BulkOperationStatus.Completed, succeeded: 1232 }); - render(BulkOperationItem, { props: { onCancel: vi.fn(), onDismiss, operation } }); - - // Act - await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); - - // Assert - expect(statusText()).toBe('Completed · 1,232 processed, 2 failed'); - expect(screen.queryByRole('button', { name: /^Cancel / })).toBeNull(); - expect(onDismiss).toHaveBeenCalledWith(operation); - }); - - it('shows how far a cancelled operation got', () => { - // Arrange & Act - render(BulkOperationItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), operation: createOperation({ status: BulkOperationStatus.Cancelled }) } }); - - // Assert - expect(statusText()).toBe('Cancelled after 500 of 1,234'); - expect(screen.queryByRole('progressbar')).toBeNull(); - }); - - it('shows why an operation failed', () => { - // Arrange & Act - render(BulkOperationItem, { - props: { - onCancel: vi.fn(), - onDismiss: vi.fn(), - operation: createOperation({ message: 'The operation stopped because of an unexpected error.', status: BulkOperationStatus.Failed }) - } - }); - - // Assert - expect(statusText()).toBe('Failed after 500 of 1,234 · The operation stopped because of an unexpected error.'); - }); -}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte deleted file mode 100644 index 26e6378b54..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte +++ /dev/null @@ -1,93 +0,0 @@ - - -{#if visibleOperations.length > 0} -
- {#each visibleOperations as operation (operation.id)} - - {/each} -
-{/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts deleted file mode 100644 index b70700dee7..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/components/bulk-operation-panel.svelte.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/svelte'; -import { tick } from 'svelte'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -import { BulkOperationSelectionKind, BulkOperationStatus, type ViewBulkOperation } from '../models'; - -const mocks = vi.hoisted(() => ({ - cancelMutateAsync: vi.fn(), - invalidateAffected: vi.fn(), - queryClient: { id: 'query-client' }, - updateFromMessage: vi.fn() -})); - -vi.mock('@tanstack/svelte-query', () => ({ useQueryClient: () => mocks.queryClient })); -vi.mock('svelte-sonner', () => ({ toast: { error: vi.fn(), success: vi.fn() } })); -vi.mock('#features/organizations/context.svelte.ts', () => ({ organization: { current: 'organization-id' } })); -vi.mock('../api.svelte', async () => { - const { SvelteMap } = await import('svelte/reactivity'); - const state = new SvelteMap(); - - return { - __state: state, - cancelBulkOperation: () => ({ mutateAsync: mocks.cancelMutateAsync }), - getBulkOperationsQuery: () => ({ - get data() { - return state.get('operations'); - } - }), - invalidateBulkOperationAffectedQueries: mocks.invalidateAffected, - updateBulkOperationFromMessage: mocks.updateFromMessage - }; -}); - -import * as api from '../api.svelte'; -import BulkOperationPanel from './bulk-operation-panel.svelte'; - -const state = (api as unknown as { __state: Map }).__state; - -function createOperation(overrides: Partial = {}): ViewBulkOperation { - const now = new Date().toISOString(); - return { - created_by_user_id: 'user-id', - created_utc: now, - description: 'Delete 50 events matching filter', - failed: 0, - id: 'operation-1', - organization_id: 'organization-id', - processed: 20, - selection: { kind: BulkOperationSelectionKind.Filter }, - status: BulkOperationStatus.Running, - succeeded: 20, - total: 50, - type: 'events.delete', - updated_utc: now, - ...overrides - }; -} - -describe('BulkOperationPanel', () => { - beforeEach(() => { - state.clear(); - mocks.cancelMutateAsync.mockResolvedValue(undefined); - }); - - afterEach(cleanup); - - it('hydrates running operations from the list after a reload and ignores stale finished ones', async () => { - // Arrange - state.set('operations', [ - createOperation(), - createOperation({ id: 'long-finished', status: BulkOperationStatus.Completed, updated_utc: '2020-01-01T00:00:00Z' }) - ]); - - // Act - render(BulkOperationPanel); - await tick(); - - // Assert - expect(screen.getAllByText('Delete 50 events matching filter')).toHaveLength(1); - expect(document.querySelector('[data-slot="bulk-operation-progress"]')?.textContent?.replace(/\s+/g, ' ').trim()).toBe('20 of 50'); - expect(mocks.invalidateAffected).not.toHaveBeenCalled(); - }); - - it('applies pushed updates for its organization and ignores other organizations', async () => { - // Arrange - state.set('operations', [createOperation()]); - render(BulkOperationPanel); - const update = { id: 'operation-1', organization_id: 'organization-id', processed: 30, status: BulkOperationStatus.Running }; - - // Act - document.dispatchEvent(new CustomEvent('BulkOperationUpdated', { detail: update })); - document.dispatchEvent(new CustomEvent('BulkOperationUpdated', { detail: { ...update, organization_id: 'another-organization' } })); - - // Assert - expect(mocks.updateFromMessage).toHaveBeenCalledTimes(1); - expect(mocks.updateFromMessage).toHaveBeenCalledWith(mocks.queryClient, update); - }); - - it('refreshes events and stacks once when a running operation finishes', async () => { - // Arrange - state.set('operations', [createOperation()]); - render(BulkOperationPanel); - await tick(); - - // Act - state.set('operations', [createOperation({ processed: 50, status: BulkOperationStatus.Completed, succeeded: 50 })]); - await tick(); - state.set('operations', [ - createOperation({ processed: 50, status: BulkOperationStatus.Completed, succeeded: 50, updated_utc: new Date().toISOString() }) - ]); - await tick(); - - // Assert - expect(mocks.invalidateAffected).toHaveBeenCalledTimes(1); - expect(mocks.invalidateAffected).toHaveBeenCalledWith(mocks.queryClient); - expect(document.querySelector('[data-slot="bulk-operation-progress"]')?.textContent).toContain('Completed'); - }); - - it('requests cancellation of a running operation', async () => { - // Arrange - state.set('operations', [createOperation()]); - render(BulkOperationPanel); - - // Act - await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); - - // Assert - await waitFor(() => expect(mocks.cancelMutateAsync).toHaveBeenCalledWith('operation-1')); - }); - - it('removes a finished operation when it is dismissed', async () => { - // Arrange - state.set('operations', [createOperation({ status: BulkOperationStatus.Completed })]); - render(BulkOperationPanel); - - // Act - await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); - - // Assert - expect(screen.queryByText('Delete 50 events matching filter')).toBeNull(); - }); - - it('renders nothing when there is nothing to show', () => { - // Arrange & Act - render(BulkOperationPanel); - - // Assert - expect(screen.queryByRole('region', { name: 'Bulk operations' })).toBeNull(); - }); -}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.test.ts deleted file mode 100644 index 71bf1a2cb6..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { - applyBulkOperationUpdate, - BulkOperationSelectionKind, - BulkOperationStatus, - FINISHED_OPERATION_VISIBLE_MS, - getProgressPercent, - selectVisibleOperations, - type ViewBulkOperation -} from './models'; - -function createOperation(overrides: Partial = {}): ViewBulkOperation { - return { - created_by_user_id: 'user-id', - created_utc: '2026-01-01T10:00:00Z', - description: 'Delete events matching filter', - failed: 0, - id: 'operation-1', - organization_id: 'organization-id', - processed: 0, - selection: { kind: BulkOperationSelectionKind.Filter }, - status: BulkOperationStatus.Running, - succeeded: 0, - total: 100, - type: 'events.delete', - updated_utc: '2026-01-01T10:00:00Z', - ...overrides - }; -} - -function createUpdate(overrides: Record = {}) { - return { - description: 'Delete events matching filter', - failed: 0, - id: 'operation-1', - organization_id: 'organization-id', - processed: 0, - status: BulkOperationStatus.Running, - succeeded: 0, - total: 100, - type: 'events.delete', - updated_utc: '2026-01-01T10:00:05Z', - ...overrides - }; -} - -describe('applyBulkOperationUpdate', () => { - it('patches the progress of a known operation', () => { - // Arrange - const operations = [createOperation(), createOperation({ id: 'operation-2' })]; - - // Act - const result = applyBulkOperationUpdate(operations, createUpdate({ failed: 1, processed: 40, succeeded: 39 })); - - // Assert - expect(result?.[0]).toMatchObject({ failed: 1, processed: 40, succeeded: 39 }); - expect(result?.[1]).toBe(operations[1]); - }); - - it('ignores a message that is older than what is already known', () => { - // Arrange - const operations = [createOperation({ processed: 60, updated_utc: '2026-01-01T10:00:10Z' })]; - - // Act - const result = applyBulkOperationUpdate(operations, createUpdate({ processed: 20 })); - - // Assert - expect(result).toBe(operations); - }); - - it('reports an unknown operation so the list can be reloaded', () => { - // Arrange - const operations = [createOperation()]; - - // Act - const result = applyBulkOperationUpdate(operations, createUpdate({ id: 'started-elsewhere', status: BulkOperationStatus.Queued, total: 5 })); - - // Assert - expect(result).toBeUndefined(); - }); -}); - -describe('getProgressPercent', () => { - it.each([ - [{ processed: 0, status: BulkOperationStatus.Queued, total: 0 }, 0], - [{ processed: 50, status: BulkOperationStatus.Running, total: 200 }, 25], - [{ processed: 5, status: BulkOperationStatus.Running, total: 3 }, 100], - [{ processed: 3, status: BulkOperationStatus.Completed, total: 0 }, 100] - ])('maps %j to %i percent', (operation, expected) => { - // Arrange & Act - const percent = getProgressPercent(operation); - - // Assert - expect(percent).toBe(expected); - }); -}); - -describe('selectVisibleOperations', () => { - const now = new Date('2026-01-01T10:30:00Z').getTime(); - - it('keeps active operations and recently finished ones, newest first', () => { - // Arrange - const active = createOperation({ created_utc: '2026-01-01T09:00:00Z', id: 'active', updated_utc: '2026-01-01T09:00:00Z' }); - const recent = createOperation({ - created_utc: '2026-01-01T10:20:00Z', - id: 'recent', - status: BulkOperationStatus.Completed, - updated_utc: new Date(now - FINISHED_OPERATION_VISIBLE_MS + 1000).toISOString() - }); - const old = createOperation({ - id: 'old', - status: BulkOperationStatus.Completed, - updated_utc: new Date(now - FINISHED_OPERATION_VISIBLE_MS - 1000).toISOString() - }); - - // Act - const result = selectVisibleOperations([active, old, recent], new Set(), now); - - // Assert - expect(result.map((operation) => operation.id)).toEqual(['recent', 'active']); - }); - - it('hides dismissed operations', () => { - // Arrange - const operation = createOperation({ id: 'dismissed' }); - - // Act - const result = selectVisibleOperations([operation], new Set(['dismissed']), now); - - // Assert - expect(result).toEqual([]); - }); -}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.ts deleted file mode 100644 index b26afc3d72..0000000000 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/models.ts +++ /dev/null @@ -1,67 +0,0 @@ -import type { BulkOperationUpdated } from '#features/websockets/models.ts'; - -import { BulkOperationSelectionKind, BulkOperationStatus, type NewBulkOperation, type ViewBulkOperation } from '#generated/api.ts'; - -export { BulkOperationSelectionKind, BulkOperationStatus, type NewBulkOperation, type ViewBulkOperation }; - -export const EVENTS_DELETE_OPERATION = 'events.delete'; - -/** Finished operations stay visible in the panel for this long, unless dismissed. */ -export const FINISHED_OPERATION_VISIBLE_MS = 2 * 60 * 1000; - -export function applyBulkOperationUpdate(operations: ViewBulkOperation[], update: BulkOperationUpdated): undefined | ViewBulkOperation[] { - const index = operations.findIndex((operation) => operation.id === update.id); - if (index < 0) { - return undefined; - } - - const current = operations[index]!; - if (new Date(update.updated_utc).getTime() < new Date(current.updated_utc).getTime()) { - // Messages can arrive out of order; never move an operation backwards. - return operations; - } - - const next = [...operations]; - next[index] = { - ...current, - description: update.description, - failed: update.failed, - message: update.message ?? null, - processed: update.processed, - status: update.status, - succeeded: update.succeeded, - total: update.total, - updated_utc: update.updated_utc - }; - - return next; -} - -export function getProgressPercent(operation: Pick): number { - if (operation.status === BulkOperationStatus.Completed) { - return 100; - } - - if (operation.total <= 0) { - return 0; - } - - return Math.min(100, Math.round((operation.processed / operation.total) * 100)); -} - -export function isBulkOperationActive(status: BulkOperationStatus): boolean { - return status === BulkOperationStatus.Queued || status === BulkOperationStatus.Running; -} - -/** Active operations plus recently finished ones the user has not dismissed, newest first. */ -export function selectVisibleOperations(operations: ViewBulkOperation[], dismissedIds: ReadonlySet, now: number): ViewBulkOperation[] { - return operations - .filter((operation) => { - if (dismissedIds.has(operation.id)) { - return false; - } - - return isBulkOperationActive(operation.status) || now - new Date(operation.updated_utc).getTime() < FINISHED_OPERATION_VISIBLE_MS; - }) - .sort((a, b) => new Date(b.created_utc).getTime() - new Date(a.created_utc).getTime()); -} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte index 131b8be99d..b439556ae7 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte @@ -11,9 +11,9 @@ import { Button } from '#comp/ui/button/index.ts'; import * as DropdownMenu from '#comp/ui/dropdown-menu/index.ts'; - import { postBulkOperation } from '#features/bulk-operations/api.svelte.ts'; - import { BulkOperationSelectionKind, EVENTS_DELETE_OPERATION } from '#features/bulk-operations/models.ts'; import { deleteEvent } from '#features/events/api.svelte.ts'; + import { postBulkOperation } from '#features/tasks/api.svelte.ts'; + import { BulkOperationSelectionKind, EVENTS_DELETE_OPERATION } from '#features/tasks/models.ts'; import type { AllMatchingTarget } from './all-matching-selection.svelte'; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts index c1df772563..b09c74fdd1 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts @@ -8,7 +8,7 @@ const startBulkOperation = vi.hoisted(() => vi.fn()); const postBulkOperation = vi.hoisted(() => vi.fn(() => ({ mutateAsync: startBulkOperation }))); vi.mock('#features/events/api.svelte.ts', () => ({ deleteEvent })); -vi.mock('#features/bulk-operations/api.svelte.ts', () => ({ postBulkOperation })); +vi.mock('#features/tasks/api.svelte.ts', () => ({ postBulkOperation })); vi.mock('svelte-sonner', () => ({ toast })); import EventsBulkActionsDropdownMenu from './events-bulk-actions-dropdown-menu.svelte'; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts index 28a3a0f4c9..fe2492701c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts @@ -9,6 +9,7 @@ import { accessToken } from '#features/auth/index.svelte.ts'; import { queryKeys as eventQueryKeys } from '#features/events/api.svelte.ts'; import { fetchApiJson } from '#features/shared/api/api.svelte.ts'; import { queryKeys as stackQueryKeys } from '#features/stacks/api.svelte.ts'; +import { invalidateTaskQueries } from '#features/tasks/api.svelte.ts'; export async function invalidateProjectQueries(queryClient: QueryClient, message: WebSocketMessageValue<'ProjectChanged'>) { const { id, organization_id } = message; @@ -360,6 +361,8 @@ export function deleteSourceMapMutation(request: SourceMapRequest) { } export function generateSampleData(request: GenerateSampleDataRequest) { + const queryClient = useQueryClient(); + return createMutation(() => ({ enabled: () => !!accessToken.current && !!request.route.id, mutationFn: async () => { @@ -370,7 +373,9 @@ export function generateSampleData(request: GenerateSampleDataRequest) { return response.data!; }, - mutationKey: queryKeys.generateSampleData(request.route.id) + mutationKey: queryKeys.generateSampleData(request.route.id), + // The request starts a background task; show it in the progress panel. + onSuccess: () => invalidateTaskQueries(queryClient) })); } @@ -707,6 +712,8 @@ export function putProjectIntegrationNotificationSettings(request: PutProjectInt } export function resetData(request: ResetDataRequest) { + const queryClient = useQueryClient(); + return createMutation(() => ({ enabled: () => !!accessToken.current && !!request.route.id, mutationFn: async () => { @@ -717,7 +724,9 @@ export function resetData(request: ResetDataRequest) { return response.data!; }, - mutationKey: queryKeys.resetData(request.route.id) + mutationKey: queryKeys.resetData(request.route.id), + // The request starts a background task; show it in the progress panel. + onSuccess: () => invalidateTaskQueries(queryClient) })); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts new file mode 100644 index 0000000000..05c5b64e79 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts @@ -0,0 +1,121 @@ +import type { QueryClient } from '@tanstack/svelte-query'; + +import { type ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; +import { createMutation, createQuery, useQueryClient } from '@tanstack/svelte-query'; + +import type { BackgroundTaskUpdated } from '#features/websockets/models.ts'; + +import { accessToken } from '#features/auth/index.svelte.ts'; +import { queryKeys as eventQueryKeys } from '#features/events/api.svelte.ts'; +import { queryKeys as stackQueryKeys } from '#features/stacks/api.svelte.ts'; + +import { applyTaskUpdate, isTaskActive, type NewBulkOperation, type ViewBackgroundTask } from './models'; + +/** While a task is active the list is polled as well, so progress still moves when push messages are unavailable. */ +export const ACTIVE_TASK_POLL_INTERVAL_MS = 3000; + +export const queryKeys = { + cancel: (id: string | undefined) => [...queryKeys.type, id, 'cancel'] as const, + organization: (organizationId: string | undefined) => [...queryKeys.type, 'organization', organizationId] as const, + post: () => [...queryKeys.type, 'post'] as const, + type: ['BackgroundTask'] as const +}; + +export interface GetTasksRequest { + route: { + organizationId: string | undefined; + }; +} + +export function cancelTask() { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current, + mutationFn: async (id: string) => { + const client = useFetchClient(); + const response = await client.postJSON(`tasks/${id}/cancel`); + + return response.data!; + }, + mutationKey: queryKeys.cancel(undefined), + onSuccess: (task) => { + upsertTask(queryClient, task); + } + })); +} + +export function getTasksQuery(request: GetTasksRequest) { + return createQuery(() => ({ + enabled: () => !!accessToken.current && !!request.route.organizationId, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const client = useFetchClient(); + const response = await client.getJSON(`organizations/${request.route.organizationId}/tasks`, { + signal + }); + + return response.data ?? []; + }, + queryKey: queryKeys.organization(request.route.organizationId), + refetchInterval: (query) => (query.state.data?.some((task) => isTaskActive(task.status)) ? ACTIVE_TASK_POLL_INTERVAL_MS : false) + })); +} + +/** Refreshes everything a finished bulk task may have changed. */ +export async function invalidateTaskAffectedQueries(queryClient: QueryClient) { + await Promise.all([ + queryClient.invalidateQueries({ + queryKey: eventQueryKeys.type + }), + queryClient.invalidateQueries({ + queryKey: stackQueryKeys.type + }) + ]); +} + +/** Refetches the task lists, e.g. after a request that starts a task. */ +export function invalidateTaskQueries(queryClient: QueryClient) { + return queryClient.invalidateQueries({ + queryKey: queryKeys.type + }); +} + +export function postBulkOperation() { + const queryClient = useQueryClient(); + + return createMutation(() => ({ + enabled: () => !!accessToken.current, + mutationFn: async (task: NewBulkOperation) => { + const client = useFetchClient(); + const response = await client.postJSON('bulk-operations', task); + + return response.data!; + }, + mutationKey: queryKeys.post(), + onSuccess: (task) => { + upsertTask(queryClient, task); + } + })); +} + +/** Applies a pushed update to the cached list. Unknown tasks (started elsewhere) trigger a refetch instead. */ +export function updateTaskFromMessage(queryClient: QueryClient, update: BackgroundTaskUpdated) { + const key = queryKeys.organization(update.organization_id); + const current = queryClient.getQueryData(key); + const next = current ? applyTaskUpdate(current, update) : undefined; + if (!next) { + void queryClient.invalidateQueries({ + queryKey: key + }); + return; + } + + queryClient.setQueryData(key, next); +} + +function upsertTask(queryClient: QueryClient, task: ViewBackgroundTask) { + queryClient.setQueryData(queryKeys.organization(task.organization_id), (current) => [ + task, + ...(current ?? []).filter((existing) => existing.id !== task.id) + ]); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts similarity index 67% rename from src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.test.ts rename to src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts index 8ecf1a2916..dc1d798c55 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/bulk-operations/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { BulkOperationSelectionKind, BulkOperationStatus, type ViewBulkOperation } from './models'; +import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from './models'; vi.mock('#features/auth/index.svelte.ts', () => ({ accessToken: { current: 'access-token' } })); vi.mock('#features/events/api.svelte.ts', () => ({ queryKeys: { type: ['PersistentEvent'] } })); @@ -8,19 +8,21 @@ vi.mock('#features/stacks/api.svelte.ts', () => ({ queryKeys: { type: ['Stack'] vi.mock('@foundatiofx/fetchclient', () => ({ useFetchClient: vi.fn() })); vi.mock('@tanstack/svelte-query', () => ({ createMutation: vi.fn(), createQuery: vi.fn(), useQueryClient: vi.fn() })); -import { invalidateBulkOperationAffectedQueries, queryKeys, updateBulkOperationFromMessage } from './api.svelte'; +import { invalidateTaskAffectedQueries, queryKeys, updateTaskFromMessage } from './api.svelte'; -function createOperation(overrides: Partial = {}): ViewBulkOperation { +function createTask(overrides: Partial = {}): ViewBackgroundTask { return { + can_cancel: true, created_by_user_id: 'user-id', created_utc: '2026-01-01T10:00:00Z', description: 'Delete events matching filter', failed: 0, - id: 'operation-1', + id: 'task-1', organization_id: 'organization-id', processed: 0, + progress: 0, selection: { kind: BulkOperationSelectionKind.Filter }, - status: BulkOperationStatus.Running, + status: BackgroundTaskStatus.Running, succeeded: 0, total: 100, type: 'events.delete', @@ -31,12 +33,14 @@ function createOperation(overrides: Partial = {}): ViewBulkOp function createUpdate(id: string) { return { + can_cancel: true, description: 'Delete events matching filter', failed: 0, id, organization_id: 'organization-id', processed: 75, - status: BulkOperationStatus.Running, + progress: 75, + status: BackgroundTaskStatus.Running, succeeded: 75, total: 100, type: 'events.delete', @@ -44,33 +48,33 @@ function createUpdate(id: string) { }; } -describe('updateBulkOperationFromMessage', () => { +describe('updateTaskFromMessage', () => { const queryClient = { getQueryData: vi.fn(), invalidateQueries: vi.fn(), setQueryData: vi.fn() }; beforeEach(() => { vi.clearAllMocks(); }); - it('writes the progress of a known operation into the cached list', () => { + it('writes the progress of a known task into the cached list', () => { // Arrange - queryClient.getQueryData.mockReturnValue([createOperation()]); + queryClient.getQueryData.mockReturnValue([createTask()]); // Act - updateBulkOperationFromMessage(queryClient as never, createUpdate('operation-1')); + updateTaskFromMessage(queryClient as never, createUpdate('task-1')); // Assert expect(queryClient.setQueryData).toHaveBeenCalledWith(queryKeys.organization('organization-id'), [ - expect.objectContaining({ id: 'operation-1', processed: 75 }) + expect.objectContaining({ id: 'task-1', processed: 75 }) ]); expect(queryClient.invalidateQueries).not.toHaveBeenCalled(); }); - it('reloads the list when the operation was started somewhere else', () => { + it('reloads the list when the task was started somewhere else', () => { // Arrange - queryClient.getQueryData.mockReturnValue([createOperation()]); + queryClient.getQueryData.mockReturnValue([createTask()]); // Act - updateBulkOperationFromMessage(queryClient as never, createUpdate('started-elsewhere')); + updateTaskFromMessage(queryClient as never, createUpdate('started-elsewhere')); // Assert expect(queryClient.setQueryData).not.toHaveBeenCalled(); @@ -78,13 +82,13 @@ describe('updateBulkOperationFromMessage', () => { }); }); -describe('invalidateBulkOperationAffectedQueries', () => { +describe('invalidateTaskAffectedQueries', () => { it('refreshes event and stack queries', async () => { // Arrange const queryClient = { invalidateQueries: vi.fn().mockResolvedValue(undefined) }; // Act - await invalidateBulkOperationAffectedQueries(queryClient as never); + await invalidateTaskAffectedQueries(queryClient as never); // Assert expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: ['PersistentEvent'] }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte new file mode 100644 index 0000000000..234872d1e4 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte @@ -0,0 +1,97 @@ + + +
+
+ +
+ {announcement} +

{task.description}

+

+ {#if task.status === BackgroundTaskStatus.Queued} + Waiting to start + {:else if task.status === BackgroundTaskStatus.Running} + {#if counts} + of + {:else} + {percent}%{#if task.message}{` · ${task.message}`}{/if} + {/if} + {:else if task.status === BackgroundTaskStatus.Completed} + {#if counts} + Completed · processed{#if task.failed > 0}, failed{/if} + {:else} + Completed{#if task.message}{` · ${task.message}`}{/if} + {/if} + {:else if task.status === BackgroundTaskStatus.Cancelled} + {#if counts} + Cancelled after of + {:else} + Cancelled + {/if} + {:else if counts} + Failed after of {#if task.message}{` · ${task.message}`}{/if} + {:else} + Failed{#if task.message}{` · ${task.message}`}{/if} + {/if} +

+
+ {#if isActive} + {#if task.can_cancel} + + {/if} + {:else} + + {/if} +
+ {#if isActive || task.status === BackgroundTaskStatus.Completed} + + {/if} +
diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts new file mode 100644 index 0000000000..cad36780f5 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts @@ -0,0 +1,158 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from '../models'; +import TaskItem from './task-item.svelte'; + +function createTask(overrides: Partial = {}): ViewBackgroundTask { + return { + can_cancel: true, + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Delete 1,234 events matching filter', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 500, + progress: 0, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BackgroundTaskStatus.Running, + succeeded: 500, + total: 1234, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:00Z', + ...overrides + }; +} + +function statusText(): string { + return (document.querySelector('[data-slot="task-progress"]')?.textContent ?? '').replace(/\s+/g, ' ').trim(); +} + +describe('TaskItem', () => { + afterEach(cleanup); + + it('shows the description, x of y progress and a Cancel button while running', async () => { + // Arrange + const onCancel = vi.fn(); + const task = createTask(); + render(TaskItem, { props: { onCancel, onDismiss: vi.fn(), task } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); + + // Assert + expect(screen.getByText('Delete 1,234 events matching filter')).toBeTruthy(); + expect(statusText()).toBe('500 of 1,234'); + expect(screen.getByRole('progressbar').getAttribute('aria-valuenow')).toBe('41'); + expect(onCancel).toHaveBeenCalledWith(task); + expect(screen.queryByRole('button', { name: /^Dismiss / })).toBeNull(); + }); + + it('names the task on its buttons and announces only state changes, not every progress update', async () => { + // Arrange + const { rerender } = render(TaskItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ processed: 100 }) } }); + const announcement = () => screen.getByRole('status').textContent; + const initial = announcement(); + + // Act + await rerender({ task: createTask({ processed: 900 }) }); + const afterProgress = announcement(); + await rerender({ task: createTask({ processed: 1234, status: BackgroundTaskStatus.Completed }) }); + + // Assert + expect(initial).toBe('Delete 1,234 events matching filter: in progress'); + expect(afterProgress).toBe(initial); + expect(announcement()).toBe('Delete 1,234 events matching filter: completed'); + expect(screen.getByRole('button', { name: 'Dismiss Delete 1,234 events matching filter' })).toBeTruthy(); + }); + + it('labels Cancel with the task it cancels', () => { + // Arrange & Act + render(TaskItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() } }); + + // Assert + expect(screen.getByRole('button', { name: 'Cancel Delete 1,234 events matching filter' })).toBeTruthy(); + }); + + it('shows the queued state without a count', () => { + // Arrange & Act + render(TaskItem, { + props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ processed: 0, status: BackgroundTaskStatus.Queued }) } + }); + + // Assert + expect(statusText()).toBe('Waiting to start'); + expect(screen.getByRole('button', { name: /^Cancel / })).toBeTruthy(); + }); + + it('disables Cancel once cancellation was requested', () => { + // Arrange & Act + render(TaskItem, { props: { isCancelling: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() } }); + + // Assert + expect((screen.getByRole('button', { name: /^Cancel / }) as HTMLButtonElement).disabled).toBe(true); + }); + + it('shows a completed task with its failures and a Dismiss button', async () => { + // Arrange + const onDismiss = vi.fn(); + const task = createTask({ failed: 2, processed: 1234, status: BackgroundTaskStatus.Completed, succeeded: 1232 }); + render(TaskItem, { props: { onCancel: vi.fn(), onDismiss, task } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + + // Assert + expect(statusText()).toBe('Completed · 1,232 processed, 2 failed'); + expect(screen.queryByRole('button', { name: /^Cancel / })).toBeNull(); + expect(onDismiss).toHaveBeenCalledWith(task); + }); + + it('shows how far a cancelled task got', () => { + // Arrange & Act + render(TaskItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ status: BackgroundTaskStatus.Cancelled }) } }); + + // Assert + expect(statusText()).toBe('Cancelled after 500 of 1,234'); + expect(screen.queryByRole('progressbar')).toBeNull(); + }); + + it('shows why a task failed', () => { + // Arrange & Act + render(TaskItem, { + props: { + onCancel: vi.fn(), + onDismiss: vi.fn(), + task: createTask({ message: 'The task stopped because of an unexpected error.', status: BackgroundTaskStatus.Failed }) + } + }); + + // Assert + expect(statusText()).toBe('Failed after 500 of 1,234 · The task stopped because of an unexpected error.'); + }); + + it('shows the percent and message of a task that has no item counts', () => { + // Arrange & Act + render(TaskItem, { + props: { + onCancel: vi.fn(), + onDismiss: vi.fn(), + task: createTask({ can_cancel: false, message: 'Events removed: 12', processed: 0, progress: 50, total: 0, type: 'projects.reset-data' }) + } + }); + + // Assert + expect(statusText()).toBe('50% · Events removed: 12'); + expect(screen.getByRole('progressbar').getAttribute('aria-valuenow')).toBe('50'); + }); + + it('does not offer Cancel for a running task that cannot be cancelled', () => { + // Arrange & Act + render(TaskItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ can_cancel: false }) } }); + + // Assert + expect(screen.queryByRole('button', { name: /^Cancel / })).toBeNull(); + expect(screen.queryByRole('button', { name: /^Dismiss / })).toBeNull(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte new file mode 100644 index 0000000000..f1f7f071e5 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte @@ -0,0 +1,93 @@ + + +{#if visibleTasks.length > 0} +
+ {#each visibleTasks as task (task.id)} + + {/each} +
+{/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts new file mode 100644 index 0000000000..eb05a3e6a6 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts @@ -0,0 +1,147 @@ +import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { tick } from 'svelte'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from '../models'; + +const mocks = vi.hoisted(() => ({ + cancelMutateAsync: vi.fn(), + invalidateAffected: vi.fn(), + queryClient: { id: 'query-client' }, + updateFromMessage: vi.fn() +})); + +vi.mock('@tanstack/svelte-query', () => ({ useQueryClient: () => mocks.queryClient })); +vi.mock('svelte-sonner', () => ({ toast: { error: vi.fn(), success: vi.fn() } })); +vi.mock('#features/organizations/context.svelte.ts', () => ({ organization: { current: 'organization-id' } })); +vi.mock('../api.svelte', async () => { + const { SvelteMap } = await import('svelte/reactivity'); + const state = new SvelteMap(); + + return { + __state: state, + cancelTask: () => ({ mutateAsync: mocks.cancelMutateAsync }), + getTasksQuery: () => ({ + get data() { + return state.get('tasks'); + } + }), + invalidateTaskAffectedQueries: mocks.invalidateAffected, + updateTaskFromMessage: mocks.updateFromMessage + }; +}); + +import * as api from '../api.svelte'; +import TaskPanel from './task-panel.svelte'; + +const state = (api as unknown as { __state: Map }).__state; + +function createTask(overrides: Partial = {}): ViewBackgroundTask { + const now = new Date().toISOString(); + return { + can_cancel: true, + created_by_user_id: 'user-id', + created_utc: now, + description: 'Delete 50 events matching filter', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 20, + progress: 0, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BackgroundTaskStatus.Running, + succeeded: 20, + total: 50, + type: 'events.delete', + updated_utc: now, + ...overrides + }; +} + +describe('TaskPanel', () => { + beforeEach(() => { + state.clear(); + mocks.cancelMutateAsync.mockResolvedValue(undefined); + }); + + afterEach(cleanup); + + it('hydrates running tasks from the list after a reload and ignores stale finished ones', async () => { + // Arrange + state.set('tasks', [createTask(), createTask({ id: 'long-finished', status: BackgroundTaskStatus.Completed, updated_utc: '2020-01-01T00:00:00Z' })]); + + // Act + render(TaskPanel); + await tick(); + + // Assert + expect(screen.getAllByText('Delete 50 events matching filter')).toHaveLength(1); + expect(document.querySelector('[data-slot="task-progress"]')?.textContent?.replace(/\s+/g, ' ').trim()).toBe('20 of 50'); + expect(mocks.invalidateAffected).not.toHaveBeenCalled(); + }); + + it('applies pushed updates for its organization and ignores other organizations', async () => { + // Arrange + state.set('tasks', [createTask()]); + render(TaskPanel); + const update = { id: 'task-1', organization_id: 'organization-id', processed: 30, status: BackgroundTaskStatus.Running }; + + // Act + document.dispatchEvent(new CustomEvent('BackgroundTaskUpdated', { detail: update })); + document.dispatchEvent(new CustomEvent('BackgroundTaskUpdated', { detail: { ...update, organization_id: 'another-organization' } })); + + // Assert + expect(mocks.updateFromMessage).toHaveBeenCalledTimes(1); + expect(mocks.updateFromMessage).toHaveBeenCalledWith(mocks.queryClient, update); + }); + + it('refreshes events and stacks once when a running task finishes', async () => { + // Arrange + state.set('tasks', [createTask()]); + render(TaskPanel); + await tick(); + + // Act + state.set('tasks', [createTask({ processed: 50, status: BackgroundTaskStatus.Completed, succeeded: 50 })]); + await tick(); + state.set('tasks', [createTask({ processed: 50, status: BackgroundTaskStatus.Completed, succeeded: 50, updated_utc: new Date().toISOString() })]); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(1); + expect(mocks.invalidateAffected).toHaveBeenCalledWith(mocks.queryClient); + expect(document.querySelector('[data-slot="task-progress"]')?.textContent).toContain('Completed'); + }); + + it('requests cancellation of a running task', async () => { + // Arrange + state.set('tasks', [createTask()]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); + + // Assert + await waitFor(() => expect(mocks.cancelMutateAsync).toHaveBeenCalledWith('task-1')); + }); + + it('removes a finished task when it is dismissed', async () => { + // Arrange + state.set('tasks', [createTask({ status: BackgroundTaskStatus.Completed })]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + + // Assert + expect(screen.queryByText('Delete 50 events matching filter')).toBeNull(); + }); + + it('renders nothing when there is nothing to show', () => { + // Arrange & Act + render(TaskPanel); + + // Assert + expect(screen.queryByRole('region', { name: 'Bulk tasks' })).toBeNull(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts new file mode 100644 index 0000000000..789f8a82d2 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest'; + +import { + applyTaskUpdate, + BackgroundTaskStatus, + BulkOperationSelectionKind, + FINISHED_TASK_VISIBLE_MS, + getProgressPercent, + selectVisibleTasks, + type ViewBackgroundTask +} from './models'; + +function createTask(overrides: Partial = {}): ViewBackgroundTask { + return { + can_cancel: true, + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Delete events matching filter', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 0, + progress: 0, + selection: { kind: BulkOperationSelectionKind.Filter }, + status: BackgroundTaskStatus.Running, + succeeded: 0, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:00Z', + ...overrides + }; +} + +function createUpdate(overrides: Record = {}) { + return { + can_cancel: true, + description: 'Delete events matching filter', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 0, + progress: 0, + status: BackgroundTaskStatus.Running, + succeeded: 0, + total: 100, + type: 'events.delete', + updated_utc: '2026-01-01T10:00:05Z', + ...overrides + }; +} + +describe('applyTaskUpdate', () => { + it('patches the progress of a known task', () => { + // Arrange + const tasks = [createTask(), createTask({ id: 'task-2' })]; + + // Act + const result = applyTaskUpdate(tasks, createUpdate({ failed: 1, processed: 40, succeeded: 39 })); + + // Assert + expect(result?.[0]).toMatchObject({ failed: 1, processed: 40, succeeded: 39 }); + expect(result?.[1]).toBe(tasks[1]); + }); + + it('ignores a message that is older than what is already known', () => { + // Arrange + const tasks = [createTask({ processed: 60, updated_utc: '2026-01-01T10:00:10Z' })]; + + // Act + const result = applyTaskUpdate(tasks, createUpdate({ processed: 20 })); + + // Assert + expect(result).toBe(tasks); + }); + + it('reports an unknown task so the list can be reloaded', () => { + // Arrange + const tasks = [createTask()]; + + // Act + const result = applyTaskUpdate(tasks, createUpdate({ id: 'started-elsewhere', status: BackgroundTaskStatus.Queued, total: 5 })); + + // Assert + expect(result).toBeUndefined(); + }); +}); + +describe('getProgressPercent', () => { + it.each([ + [{ processed: 0, progress: 0, status: BackgroundTaskStatus.Queued, total: 0 }, 0], + [{ processed: 50, progress: 0, status: BackgroundTaskStatus.Running, total: 200 }, 25], + [{ processed: 5, progress: 0, status: BackgroundTaskStatus.Running, total: 3 }, 100], + [{ processed: 3, progress: 0, status: BackgroundTaskStatus.Completed, total: 0 }, 100], + [{ processed: 0, progress: 42, status: BackgroundTaskStatus.Running, total: 0 }, 42] + ])('maps %j to %i percent', (task, expected) => { + // Arrange & Act + const percent = getProgressPercent(task); + + // Assert + expect(percent).toBe(expected); + }); +}); + +describe('selectVisibleTasks', () => { + const now = new Date('2026-01-01T10:30:00Z').getTime(); + + it('keeps active tasks and recently finished ones, newest first', () => { + // Arrange + const active = createTask({ created_utc: '2026-01-01T09:00:00Z', id: 'active', updated_utc: '2026-01-01T09:00:00Z' }); + const recent = createTask({ + created_utc: '2026-01-01T10:20:00Z', + id: 'recent', + status: BackgroundTaskStatus.Completed, + updated_utc: new Date(now - FINISHED_TASK_VISIBLE_MS + 1000).toISOString() + }); + const old = createTask({ + id: 'old', + status: BackgroundTaskStatus.Completed, + updated_utc: new Date(now - FINISHED_TASK_VISIBLE_MS - 1000).toISOString() + }); + + // Act + const result = selectVisibleTasks([active, old, recent], new Set(), now); + + // Assert + expect(result.map((task) => task.id)).toEqual(['recent', 'active']); + }); + + it('hides dismissed tasks', () => { + // Arrange + const task = createTask({ id: 'dismissed' }); + + // Act + const result = selectVisibleTasks([task], new Set(['dismissed']), now); + + // Assert + expect(result).toEqual([]); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts new file mode 100644 index 0000000000..334ecdb53b --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts @@ -0,0 +1,74 @@ +import type { BackgroundTaskUpdated } from '#features/websockets/models.ts'; + +import { BackgroundTaskStatus, BulkOperationSelectionKind, type NewBulkOperation, type ViewBackgroundTask } from '#generated/api.ts'; + +export { BackgroundTaskStatus, BulkOperationSelectionKind, type NewBulkOperation, type ViewBackgroundTask }; + +export const EVENTS_DELETE_OPERATION = 'events.delete'; + +/** Finished tasks stay visible in the panel for this long, unless dismissed. */ +export const FINISHED_TASK_VISIBLE_MS = 2 * 60 * 1000; + +export function applyTaskUpdate(tasks: ViewBackgroundTask[], update: BackgroundTaskUpdated): undefined | ViewBackgroundTask[] { + const index = tasks.findIndex((task) => task.id === update.id); + if (index < 0) { + return undefined; + } + + const current = tasks[index]!; + if (new Date(update.updated_utc).getTime() < new Date(current.updated_utc).getTime()) { + // Messages can arrive out of order; never move an task backwards. + return tasks; + } + + const next = [...tasks]; + next[index] = { + ...current, + can_cancel: update.can_cancel, + description: update.description, + failed: update.failed, + message: update.message ?? null, + processed: update.processed, + progress: update.progress, + status: update.status, + succeeded: update.succeeded, + total: update.total, + updated_utc: update.updated_utc + }; + + return next; +} + +/** Tasks that report item counts show those; tasks that only report a percent (total of zero) show the percent. */ +export function getProgressPercent(task: Pick): number { + if (task.status === BackgroundTaskStatus.Completed) { + return 100; + } + + if (task.total <= 0) { + return Math.min(100, Math.max(0, task.progress ?? 0)); + } + + return Math.min(100, Math.round((task.processed / task.total) * 100)); +} + +export function hasCounts(task: Pick): boolean { + return task.total > 0; +} + +export function isTaskActive(status: BackgroundTaskStatus): boolean { + return status === BackgroundTaskStatus.Queued || status === BackgroundTaskStatus.Running; +} + +/** Active tasks plus recently finished ones the user has not dismissed, newest first. */ +export function selectVisibleTasks(tasks: ViewBackgroundTask[], dismissedIds: ReadonlySet, now: number): ViewBackgroundTask[] { + return tasks + .filter((task) => { + if (dismissedIds.has(task.id)) { + return false; + } + + return isTaskActive(task.status) || now - new Date(task.updated_utc).getTime() < FINISHED_TASK_VISIBLE_MS; + }) + .sort((a, b) => new Date(b.created_utc).getTime() - new Date(a.created_utc).getTime()); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts index e6e694de72..787564c9c8 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts @@ -1,4 +1,4 @@ -import type { BulkOperationStatus } from '#generated/api.ts'; +import type { BackgroundTaskStatus } from '#generated/api.ts'; export enum ChangeType { Added = 0, @@ -6,15 +6,17 @@ export enum ChangeType { Removed = 2 } -export interface BulkOperationUpdated { +export interface BackgroundTaskUpdated { + can_cancel: boolean; description: string; failed: number; id: string; message?: string; organization_id: string; processed: number; + progress: number; project_id?: string; - status: BulkOperationStatus; + status: BackgroundTaskStatus; succeeded: number; total: number; type: string; @@ -67,10 +69,10 @@ export interface WebSocketMessage { } export type WebSocketMessageType = - 'BulkOperationUpdated' | 'PlanChanged' | 'PlanOverage' | 'ReleaseNotification' | 'SystemNotification' | 'UserMembershipChanged' | `${string}Changed`; + 'BackgroundTaskUpdated' | 'PlanChanged' | 'PlanOverage' | 'ReleaseNotification' | 'SystemNotification' | 'UserMembershipChanged' | `${string}Changed`; -export type WebSocketMessageValue = T extends 'BulkOperationUpdated' - ? BulkOperationUpdated +export type WebSocketMessageValue = T extends 'BackgroundTaskUpdated' + ? BackgroundTaskUpdated : T extends 'PlanChanged' ? PlanChanged : T extends 'PlanOverage' @@ -93,7 +95,7 @@ export function isPlanOverageType(message: { message: unknown; type: WebSocketMe export function isWebSocketMessageType(type: string): type is WebSocketMessageType { return ( - (['BulkOperationUpdated', 'PlanChanged', 'PlanOverage', 'UserMembershipChanged', 'ReleaseNotification', 'SystemNotification'] as const).includes( + (['BackgroundTaskUpdated', 'PlanChanged', 'PlanOverage', 'UserMembershipChanged', 'ReleaseNotification', 'SystemNotification'] as const).includes( type as Exclude ) || type.endsWith('Changed') ); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index 567083e123..39a5445607 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -29,7 +29,6 @@ import { getIntercomTokenQuery } from '#features/auth/api.svelte.ts'; import { accessToken, gotoLogin } from '#features/auth/index.svelte.ts'; import { ChangePlanDialogHost, UpgradeRequiredDialog } from '#features/billing/index.ts'; - import BulkOperationPanel from '#features/bulk-operations/components/bulk-operation-panel.svelte'; import { createOrganizationEventNotificationRefresher, invalidatePersistentEventQueries, @@ -63,6 +62,7 @@ import { savedViewHref } from '#features/saved-views/slugs.ts'; import { appKeyboardShortcuts, isKeyboardShortcut } from '#features/shared/keyboard-shortcuts.ts'; import { createProjectStackNotificationRefresher, invalidateStackQueries, type ProjectStackNotificationRefresher } from '#features/stacks/api.svelte.ts'; + import TaskPanel from '#features/tasks/components/task-panel.svelte'; import { invalidateTokenQueries } from '#features/tokens/api.svelte.ts'; import { getMeQuery, invalidateUserQueries } from '#features/users/api.svelte.ts'; import { getGravatarFromCurrentUser } from '#features/users/gravatar.svelte.ts'; @@ -1009,7 +1009,7 @@ - + {/if} Date: Tue, 3 Nov 2026 23:28:44 -0600 Subject: [PATCH 13/42] Add background task backend tests Cover the task service (storage, index cleanup, cancel rules, lock retries), bulk conflict and enqueue-failure rules, the event selection resolver, the tracking work item handler, the tasks endpoints and the websocket broker. --- .../Endpoints/BackgroundTaskEndpointTests.cs | 303 +++++++++++ .../Endpoints/BulkOperationEndpointTests.cs | 45 -- ...kgroundTaskTrackingWorkItemHandlerTests.cs | 241 +++++++++ .../Models/BackgroundTaskTests.cs | 45 ++ .../BackgroundTaskServiceTests.cs | 493 ++++++++++++++++++ .../BulkOperationServiceTests.cs | 266 ++++++++++ .../EventBulkSelectionResolverTests.cs | 233 +++++++++ .../Utility/BackgroundTaskTestHarness.cs | 79 +++ 8 files changed, 1660 insertions(+), 45 deletions(-) create mode 100644 tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs create mode 100644 tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs create mode 100644 tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs create mode 100644 tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs create mode 100644 tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs create mode 100644 tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs create mode 100644 tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs new file mode 100644 index 0000000000..f48042c6a8 --- /dev/null +++ b/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs @@ -0,0 +1,303 @@ +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Extensions; +using Exceptionless.Tests.Hubs; +using Exceptionless.Tests.Utility; +using Exceptionless.Web.Api.Results; +using Exceptionless.Web.Hubs; +using Exceptionless.Web.Models; +using Foundatio.Jobs; +using Foundatio.Repositories; +using Foundatio.Repositories.Utility; +using Xunit; + +namespace Exceptionless.Tests.Api.Endpoints; + +public class BackgroundTaskEndpointTests : IntegrationTestsBase +{ + private readonly BackgroundTaskService _taskService; + private readonly WorkItemJob _workItemJob; + + public BackgroundTaskEndpointTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _taskService = GetService(); + _workItemJob = GetService(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + await GetService().CreateDataAsync(); + } + + [Fact] + public async Task Get_Task_ReturnsItsProgressAndOnlyBulkTasksHaveASelection() + { + // Arrange + var plain = await AddTaskAsync(canCancel: false, progress: 40); + var bulk = await AddTaskAsync(canCancel: true, bulk: true); + + // Act + var plainView = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", plain.Id).StatusCodeShouldBeOk()); + var bulkView = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", bulk.Id).StatusCodeShouldBeOk()); + + // Assert + Assert.Equal(plain.Id, plainView!.Id); + Assert.Equal(BackgroundTaskTypes.ResetProjectData, plainView.Type); + Assert.Equal(BackgroundTaskStatus.Queued, plainView.Status); + Assert.Equal(40, plainView.Progress); + Assert.False(plainView.CanCancel); + Assert.Null(plainView.Selection); + Assert.True(bulkView!.CanCancel); + Assert.Equal(BulkOperationSelectionKind.Ids, bulkView.Selection!.Kind); + Assert.Equal(2, bulkView.Selection.IdCount); + } + + [Fact] + public async Task GetByOrganization_MixedTaskTypes_AreListedNewestFirst() + { + // Arrange + var first = await AddTaskAsync(canCancel: false); + TimeProvider.Advance(TimeSpan.FromSeconds(5)); + var second = await AddTaskAsync(canCancel: true, bulk: true); + await AddTaskAsync(canCancel: false, organizationId: SampleDataService.FREE_ORG_ID); + + // Act + var list = await SendRequestAsAsync>(r => r.AsTestOrganizationUser() + .AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "tasks").StatusCodeShouldBeOk()); + + // Assert + Assert.Equal([second.Id, first.Id], list!.Select(t => t.Id).ToArray()); + } + + [Fact] + public async Task Get_TaskOfAnotherOrganization_IsNotFoundForGetListAndCancelButVisibleToGlobalAdmins() + { + // Arrange + var task = await AddTaskAsync(canCancel: true); + + // Act + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("tasks", task.Id).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.AsFreeOrganizationUser().AppendPaths("organizations", SampleDataService.TEST_ORG_ID, "tasks").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", ObjectId.GenerateNewId().ToString()).StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", ObjectId.GenerateNewId().ToString(), "cancel").StatusCodeShouldBeNotFound()); + var asAdmin = await SendRequestAsAsync(r => r.AsGlobalAdminUser().AppendPaths("tasks", task.Id).StatusCodeShouldBeOk()); + + // Assert + Assert.Equal(task.Id, asAdmin!.Id); + Assert.Equal(BackgroundTaskStatus.Queued, (await _taskService.GetAsync(task.Id))!.Status); + } + + [Fact] + public async Task Tasks_WithoutAUserSession_AreRejected() + { + // Arrange + var task = await AddTaskAsync(canCancel: true); + + // Act + await SendRequestAsync(r => r.AppendPaths("tasks", task.Id).StatusCodeShouldBeUnauthorized()); + await SendRequestAsync(r => r.AsTestOrganizationClientUser().AppendPaths("tasks", task.Id).StatusCodeShouldBeForbidden()); + await SendRequestAsync(r => r.Post().AsTestOrganizationClientUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeForbidden()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Queued, (await _taskService.GetAsync(task.Id))!.Status); + } + + [Fact] + public async Task Cancel_QueuedCancellableTask_IsAcceptedAndCancelledImmediately() + { + // Arrange + var task = await AddTaskAsync(canCancel: true); + + // Act + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeAccepted()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, response!.Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, (await _taskService.GetAsync(task.Id))!.Status); + } + + [Fact] + public async Task Cancel_ActiveTaskThatCannotBeCancelled_IsUnprocessableAndKeepsRunning() + { + // Arrange + var task = await AddTaskAsync(canCancel: false); + await _taskService.UpdateAsync(task.Id, current => + { + current.Start(TimeProvider.GetUtcNow().UtcDateTime); + return true; + }); + + // Act + await SendRequestAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeUnprocessableEntity()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Running, (await _taskService.GetAsync(task.Id))!.Status); + Assert.False(await _taskService.IsCancelRequestedAsync(task.Id)); + } + + [Fact] + public async Task Cancel_FinishedTaskThatCannotBeCancelled_IsAcceptedAndChangesNothing() + { + // Arrange + var task = await AddTaskAsync(canCancel: false); + await _taskService.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", TimeProvider.GetUtcNow().UtcDateTime); + return true; + }); + + // Act + var response = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser().AppendPaths("tasks", task.Id, "cancel").StatusCodeShouldBeAccepted()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, response!.Status); + } + + [Fact] + public async Task ResetData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => d.Event().TestProject().Date(now.AddHours(-1))); + + // Act + var accepted = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "reset-data").StatusCodeShouldBeAccepted()); + string taskId = Assert.Single(accepted!.Workers); + var queued = await GetTaskAsync(taskId); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var completed = await GetTaskAsync(taskId); + + // Assert + Assert.Equal(BackgroundTaskTypes.ResetProjectData, queued.Type); + Assert.Equal(BackgroundTaskStatus.Queued, queued.Status); + Assert.False(queued.CanCancel); + Assert.Equal(SampleDataService.TEST_PROJECT_ID, queued.ProjectId); + Assert.StartsWith("Reset data for ", queued.Description); + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(100, completed.Progress); + Assert.NotNull(completed.StartedUtc); + Assert.NotNull(completed.CompletedUtc); + await RefreshDataAsync(); + Assert.Empty(await GetService().GetByIdsAsync(events.Select(e => e.Id).ToArray(), o => o.Cache(false))); + } + + [Fact] + public async Task SampleData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() + { + // Act + var accepted = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "sample-data").StatusCodeShouldBeAccepted()); + string taskId = Assert.Single(accepted!.Workers); + var queued = await GetTaskAsync(taskId); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var completed = await GetTaskAsync(taskId); + + // Assert + Assert.Equal(BackgroundTaskTypes.GenerateProjectSampleData, queued.Type); + Assert.Equal(BackgroundTaskStatus.Queued, queued.Status); + Assert.False(queued.CanCancel); + Assert.StartsWith("Generate sample data for ", queued.Description); + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(100, completed.Progress); + } + + [Fact] + public async Task ResetAndSampleData_ProjectOfAnotherOrganization_IsNotFoundAndTracksNothing() + { + // Act + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "reset-data").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "sample-data").StatusCodeShouldBeNotFound()); + + // Assert + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); + Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.FREE_ORG_ID)); + } + + [Fact] + public async Task OnBackgroundTaskUpdatedAsync_SendsOnlyToTheTasksOrganization() + { + // Arrange + var broker = GetService(); + var connectionMapping = GetService(); + var connectionManager = GetService(); + var ownSocket = new TestWebSocket(); + var otherSocket = new TestWebSocket(); + string ownConnection = connectionManager.AddWebSocket(ownSocket); + string otherConnection = connectionManager.AddWebSocket(otherSocket); + await connectionMapping.GroupAddAsync(SampleDataService.TEST_ORG_ID, ownConnection); + await connectionMapping.GroupAddAsync(SampleDataService.FREE_ORG_ID, otherConnection); + + try + { + // Act + await broker.OnBackgroundTaskUpdatedAsync(new BackgroundTaskUpdated + { + Id = "537650f3b77efe23a47914f9", + OrganizationId = SampleDataService.TEST_ORG_ID, + Type = BackgroundTaskTypes.ResetProjectData, + Description = "Reset data for Test", + Status = BackgroundTaskStatus.Running, + Progress = 40, + CanCancel = false + }, TestCancellationToken); + + await WaitForAsync(() => ownSocket.SentMessages.Count > 0); + + // Assert + string sent = Assert.Single(ownSocket.SentMessages); + Assert.Contains("\"type\":\"BackgroundTaskUpdated\"", sent); + Assert.Contains("\"progress\":40", sent); + Assert.Contains("\"can_cancel\":false", sent); + Assert.Empty(otherSocket.SentMessages); + } + finally + { + await connectionMapping.GroupRemoveAsync(SampleDataService.TEST_ORG_ID, ownConnection); + await connectionMapping.GroupRemoveAsync(SampleDataService.FREE_ORG_ID, otherConnection); + await connectionManager.RemoveWebSocketAsync(ownConnection); + await connectionManager.RemoveWebSocketAsync(otherConnection); + } + } + + private async Task AddTaskAsync(bool canCancel, int progress = 0, bool bulk = false, string organizationId = SampleDataService.TEST_ORG_ID) + { + var task = new BackgroundTask + { + OrganizationId = organizationId, + ProjectId = organizationId == SampleDataService.TEST_ORG_ID ? SampleDataService.TEST_PROJECT_ID : SampleDataService.FREE_PROJECT_ID, + Type = bulk ? "events.delete" : BackgroundTaskTypes.ResetProjectData, + Description = bulk ? "Delete 2 selected events" : "Reset data for Test", + Progress = progress, + CanCancel = canCancel, + CreatedByUserId = "537650f3b77efe23a47914f8", + Bulk = bulk ? new BulkOperationDetails + { + Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [ObjectId.GenerateNewId().ToString(), ObjectId.GenerateNewId().ToString()] } + } : null + }; + + await _taskService.AddAsync(task); + return task; + } + + private async Task GetTaskAsync(string id) + { + var task = await SendRequestAsAsync(r => r.AsTestOrganizationUser().AppendPaths("tasks", id).StatusCodeShouldBeOk()); + Assert.NotNull(task); + return task; + } + + private static async Task WaitForAsync(Func condition) + { + var deadline = DateTime.UtcNow.AddSeconds(10); + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(25); + } +} diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index d663cb6262..1ac49a18b6 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -594,51 +594,6 @@ public async Task Execute_NonDestructiveOperationsDoNotBlockEachOtherOrDestructi Assert.Equal(destructive.Task.Id, blocked.Conflict?.Id); } - [Fact] - public async Task OnBackgroundTaskUpdatedAsync_SendsOnlyToTheOperationsOrganization() - { - // Arrange - var broker = GetService(); - var connectionMapping = GetService(); - var connectionManager = GetService(); - var ownSocket = new TestWebSocket(); - var otherSocket = new TestWebSocket(); - string ownConnection = connectionManager.AddWebSocket(ownSocket); - string otherConnection = connectionManager.AddWebSocket(otherSocket); - await connectionMapping.GroupAddAsync(SampleDataService.TEST_ORG_ID, ownConnection); - await connectionMapping.GroupAddAsync(SampleDataService.FREE_ORG_ID, otherConnection); - - try - { - // Act - await broker.OnBackgroundTaskUpdatedAsync(new BackgroundTaskUpdated - { - Id = "537650f3b77efe23a47914f9", - OrganizationId = SampleDataService.TEST_ORG_ID, - Type = "events.delete", - Description = "Delete events matching filter", - Status = BackgroundTaskStatus.Running, - Total = 10, - Processed = 4 - }, TestCancellationToken); - - await WaitForAsync(() => ownSocket.SentMessages.Count > 0); - - // Assert - string sent = Assert.Single(ownSocket.SentMessages); - Assert.Contains("\"type\":\"BackgroundTaskUpdated\"", sent); - Assert.Contains("\"processed\":4", sent); - Assert.Empty(otherSocket.SentMessages); - } - finally - { - await connectionMapping.GroupRemoveAsync(SampleDataService.TEST_ORG_ID, ownConnection); - await connectionMapping.GroupRemoveAsync(SampleDataService.FREE_ORG_ID, otherConnection); - await connectionManager.RemoveWebSocketAsync(ownConnection); - await connectionManager.RemoveWebSocketAsync(otherConnection); - } - } - private static NewBulkOperation FilterRequest(string organizationId, string? projectId, string filter, string time = "[now-7d TO now]") { return new NewBulkOperation diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs new file mode 100644 index 0000000000..c900f94611 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs @@ -0,0 +1,241 @@ +using Exceptionless.Core.Jobs.WorkItemHandlers; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Utility; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Exceptionless.Tests.Jobs.WorkItemHandlers; + +public class BackgroundTaskTrackingWorkItemHandlerTests : TestWithLoggingBase, IDisposable +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private const string ProjectId = "537650f3b77efe23a47914f6"; + private readonly FakeTimeProvider _time = new(new DateTimeOffset(2026, 11, 3, 4, 5, 6, TimeSpan.Zero)); + private readonly BackgroundTaskTestHarness _harness; + + public BackgroundTaskTrackingWorkItemHandlerTests(ITestOutputHelper output) : base(output) + { + _harness = new BackgroundTaskTestHarness(_time, Log); + } + + public void Dispose() + { + _harness.Dispose(); + } + + [Fact] + public async Task HandleItemAsync_ProgressReports_UpdateTheTaskAndStillReachTheJobCallback() + { + // Arrange + var task = await AddTaskAsync(); + var reported = new List<(int Progress, string? Message)>(); + BackgroundTask? duringWork = null; + var handler = CreateHandler(async context => + { + await context.ReportProgressAsync(0, "Starting"); + _time.Advance(TimeSpan.FromSeconds(30)); + await context.ReportProgressAsync(50, "Halfway"); + duringWork = await _harness.Tasks.GetAsync(task.Id); + }); + + // Act + await handler.HandleItemAsync(CreateContext(task.Id, (progress, message) => + { + reported.Add((progress, message)); + return Task.CompletedTask; + })); + + // Assert + Assert.Equal([(0, "Starting"), (50, "Halfway")], reported); + Assert.Equal(BackgroundTaskStatus.Running, duringWork!.Status); + Assert.Equal(50, duringWork.Progress); + Assert.Equal("Halfway", duringWork.Message); + Assert.NotNull(duringWork.StartedUtc); + Assert.Contains(_harness.Publisher.Updates, u => u.Status == BackgroundTaskStatus.Running && u.Progress == 50); + } + + [Fact] + public async Task HandleItemAsync_WorkSucceeds_CompletesTheTaskAtOneHundredPercentWithTheLastMessage() + { + // Arrange + var task = await AddTaskAsync(); + var handler = CreateHandler(async context => + { + await context.ReportProgressAsync(50, "Events removed: 3"); + await context.ReportProgressAsync(100, "Events removed: 3, stacks removed: 1"); + }); + + // Act + await handler.HandleItemAsync(CreateContext(task.Id)); + var stored = await _harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, stored!.Status); + Assert.Equal(100, stored.Progress); + Assert.Equal("Events removed: 3, stacks removed: 1", stored.Message); + Assert.NotNull(stored.CompletedUtc); + Assert.Empty(await _harness.Tasks.GetActiveIdsAsync(OrganizationId)); + Assert.Equal(BackgroundTaskStatus.Completed, _harness.Publisher.Updates[^1].Status); + } + + [Fact] + public async Task HandleItemAsync_WorkThrows_FailsTheTaskAndRethrowsThenARetryBringsItBackToRunningAndCompletes() + { + // Arrange + var task = await AddTaskAsync(); + int calls = 0; + BackgroundTask? duringRetry = null; + IReadOnlyList? activeDuringRetry = null; + var handler = CreateHandler(async context => + { + if (++calls == 1) + throw new InvalidOperationException("Elasticsearch unavailable"); + + duringRetry = await _harness.Tasks.GetAsync(task.Id); + activeDuringRetry = await _harness.Tasks.GetActiveIdsAsync(OrganizationId); + await context.ReportProgressAsync(100, "Done"); + }); + + // Act + var thrown = await Record.ExceptionAsync(() => handler.HandleItemAsync(CreateContext(task.Id))); + var failed = await _harness.Tasks.GetAsync(task.Id); + var activeAfterFailure = await _harness.Tasks.GetActiveIdsAsync(OrganizationId); + await handler.HandleItemAsync(CreateContext(task.Id)); + var completed = await _harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.IsType(thrown); + Assert.Equal(BackgroundTaskStatus.Failed, failed!.Status); + Assert.Contains("Elasticsearch unavailable", failed.Message); + Assert.Empty(activeAfterFailure); + Assert.Equal(BackgroundTaskStatus.Running, duringRetry!.Status); + Assert.Null(duringRetry.CompletedUtc); + Assert.Equal([task.Id], activeDuringRetry!.ToArray()); + Assert.Equal(BackgroundTaskStatus.Completed, completed!.Status); + Assert.Empty(await _harness.Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Fact] + public async Task HandleItemAsync_ResultIsNotASuccess_PassesItToTheJobAndKeepsTheTaskRunning() + { + // Arrange + var task = await AddTaskAsync(); + var handler = CreateHandler(context => + { + context.Result = JobResult.CancelledWithMessage("Shutting down"); + return Task.CompletedTask; + }); + var context = CreateContext(task.Id); + + // Act + await handler.HandleItemAsync(context); + var stored = await _harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.False(context.Result.IsSuccess); + Assert.Equal("Shutting down", context.Result.Message); + Assert.Equal(BackgroundTaskStatus.Running, stored!.Status); + } + + [Fact] + public async Task HandleItemAsync_WorkItemWithoutATaskId_PassesStraightThroughWithoutTouchingAnyTask() + { + // Arrange + var other = await AddTaskAsync(); + WorkItemContext? received = null; + var handler = CreateHandler(async context => + { + received = context; + await context.ReportProgressAsync(10, "Working"); + }); + var context = new WorkItemContext(new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = ProjectId }, "job", EmptyLock.Empty, TestCancellationToken, (_, _) => Task.CompletedTask); + + // Act + await handler.HandleItemAsync(context); + var untouched = await _harness.Tasks.GetAsync(other.Id); + + // Assert + Assert.Same(context, received); + Assert.Equal(BackgroundTaskStatus.Queued, untouched!.Status); + Assert.Empty(_harness.Publisher.Updates); + } + + [Fact] + public async Task HandleItemAsync_TaskRecordIsGone_StillRunsTheWork() + { + // Arrange + bool ran = false; + var handler = CreateHandler(async context => + { + await context.ReportProgressAsync(100, "Done"); + ran = true; + }); + + // Act + await handler.HandleItemAsync(CreateContext(Foundatio.Repositories.Utility.ObjectId.GenerateNewId().ToString())); + + // Assert + Assert.True(ran); + Assert.Empty(_harness.Publisher.Updates); + } + + [Fact] + public async Task HandleItemAsync_TaskWasCancelled_DoesNotRevive() + { + // Arrange + var task = await AddTaskAsync(canCancel: true); + await _harness.Tasks.CancelAsync(task.Id); + var handler = CreateHandler(context => context.ReportProgressAsync(100, "Done")); + + // Act + await handler.HandleItemAsync(CreateContext(task.Id)); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, (await _harness.Tasks.GetAsync(task.Id))!.Status); + } + + [Fact] + public void Handler_DelegatesTheJobSettingsToTheWrappedHandler() + { + // Arrange + var inner = new DelegateWorkItemHandler(_ => Task.CompletedTask) { AutoRenewLockOnProgress = true }; + var handler = new BackgroundTaskTrackingWorkItemHandler(inner, _harness.Tasks, _time); + + // Act + handler.AutoRenewLockOnProgress = false; + + // Assert + Assert.False(inner.AutoRenewLockOnProgress); + Assert.Same(inner.Log, handler.Log); + } + + private async Task AddTaskAsync(bool canCancel = false) + { + var task = new BackgroundTask + { + OrganizationId = OrganizationId, + ProjectId = ProjectId, + Type = BackgroundTaskTypes.ResetProjectData, + Description = "Reset data for Test", + CreatedByUserId = "537650f3b77efe23a47914f8", + CanCancel = canCancel + }; + await _harness.Tasks.AddAsync(task); + return task; + } + + private BackgroundTaskTrackingWorkItemHandler CreateHandler(Func work) + { + return new BackgroundTaskTrackingWorkItemHandler(new DelegateWorkItemHandler(work), _harness.Tasks, _time); + } + + private WorkItemContext CreateContext(string taskId, Func? progress = null) + { + var workItem = new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = ProjectId, TaskId = taskId }; + return new WorkItemContext(workItem, "job", EmptyLock.Empty, TestCancellationToken, progress ?? ((_, _) => Task.CompletedTask)); + } +} diff --git a/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs b/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs new file mode 100644 index 0000000000..dcec573cb3 --- /dev/null +++ b/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs @@ -0,0 +1,45 @@ +using Exceptionless.Core.Models.BackgroundTasks; +using Xunit; + +namespace Exceptionless.Tests.Models; + +public class BackgroundTaskTests +{ + private static readonly DateTime UtcNow = new(2026, 11, 3, 4, 5, 6, DateTimeKind.Utc); + + [Fact] + public void Start_CalledTwice_KeepsTheFirstStartTime() + { + // Arrange + var task = new BackgroundTask(); + + // Act + task.Start(UtcNow); + task.Start(UtcNow.AddMinutes(5)); + + // Assert + Assert.Equal(BackgroundTaskStatus.Running, task.Status); + Assert.Equal(UtcNow, task.StartedUtc); + } + + [Theory] + [InlineData(BackgroundTaskStatus.Completed, 100)] + [InlineData(BackgroundTaskStatus.Failed, 40)] + [InlineData(BackgroundTaskStatus.Cancelled, 40)] + public void Finish_AnyOutcome_StopsTheTaskAndOnlyCompletionReachesOneHundredPercent(BackgroundTaskStatus status, int expectedProgress) + { + // Arrange + var task = new BackgroundTask { Progress = 40 }; + task.Start(UtcNow); + + // Act + task.Finish(status, "Done", UtcNow.AddMinutes(1)); + + // Assert + Assert.False(task.IsActive); + Assert.Equal(status, task.Status); + Assert.Equal("Done", task.Message); + Assert.Equal(UtcNow.AddMinutes(1), task.CompletedUtc); + Assert.Equal(expectedProgress, task.Progress); + } +} diff --git a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs new file mode 100644 index 0000000000..7996358f9e --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs @@ -0,0 +1,493 @@ +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Queues; +using Foundatio.Repositories.Utility; +using Foundatio.Serializer; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Services.BackgroundTasks; + +public class BackgroundTaskServiceTests : TestWithLoggingBase, IDisposable +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private const string OtherOrganizationId = "537650f3b77efe23a47914f5"; + private readonly FakeTimeProvider _time = new(new DateTimeOffset(2026, 11, 3, 4, 5, 6, TimeSpan.Zero)); + private readonly BackgroundTaskTestHarness _harness; + + public BackgroundTaskServiceTests(ITestOutputHelper output) : base(output) + { + _harness = new BackgroundTaskTestHarness(_time, Log); + } + + public void Dispose() + { + _harness.Dispose(); + } + + private BackgroundTaskService Tasks => _harness.Tasks; + + [Fact] + public async Task AddAsync_NewTask_StoresItQueuedAndIndexesIt() + { + // Arrange + var task = NewTask(); + + // Act + await Tasks.AddAsync(task); + var stored = await Tasks.GetAsync(task.Id); + var listed = await Tasks.GetByOrganizationAsync(OrganizationId); + var active = await Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.True(ObjectId.TryParse(task.Id, out _)); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Queued, stored.Status); + Assert.Equal(_time.GetUtcNow().UtcDateTime, stored.CreatedUtc); + Assert.Equal(stored.CreatedUtc, stored.UpdatedUtc); + Assert.Equal([task.Id], listed.Select(t => t.Id).ToArray()); + Assert.Equal([task.Id], active.ToArray()); + Assert.Empty(_harness.Publisher.Updates); + } + + [Fact] + public async Task GetByOrganizationAsync_ManyTasks_ReturnsTheNewestFirstUpToTheLimit() + { + // Arrange + var ids = new List(); + for (int index = 0; index < 4; index++) + { + var task = NewTask(); + await Tasks.AddAsync(task); + ids.Add(task.Id); + _time.Advance(TimeSpan.FromSeconds(5)); + } + + // Act + var newestTwo = await Tasks.GetByOrganizationAsync(OrganizationId, limit: 2); + + // Assert + Assert.Equal([ids[3], ids[2]], newestTwo.Select(t => t.Id).ToArray()); + } + + [Fact] + public async Task GetByOrganizationAsync_ExpiredAndForeignIds_AreSkippedAndExpiredIdsCleanedFromTheIndex() + { + // Arrange + var kept = NewTask(); + var expired = NewTask(); + var foreign = NewTask(OtherOrganizationId); + await Tasks.AddAsync(kept); + _time.Advance(TimeSpan.FromSeconds(1)); + await Tasks.AddAsync(expired); + _time.Advance(TimeSpan.FromSeconds(1)); + await Tasks.AddAsync(foreign); + await _harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(expired.Id)); + // A stray id from another organization must never be listed. + await _harness.Cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId), [foreign.Id]); + + // Act + var listed = await Tasks.GetByOrganizationAsync(OrganizationId); + var indexed = await _harness.Cache.GetListAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId)); + + // Assert + Assert.Equal([kept.Id], listed.Select(t => t.Id).ToArray()); + Assert.DoesNotContain(expired.Id, indexed.Value); + Assert.Contains(kept.Id, indexed.Value); + } + + [Fact] + public async Task UpdateAsync_UpdateReturnsFalse_SavesAndPublishesNothing() + { + // Arrange + var task = NewTask(); + await Tasks.AddAsync(task); + _time.Advance(TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.UpdateAsync(task.Id, current => + { + current.Progress = 50; + return false; + }); + var stored = await Tasks.GetAsync(task.Id); + + // Assert + Assert.NotNull(result); + Assert.Equal(0, stored!.Progress); + Assert.Equal(stored.CreatedUtc, stored.UpdatedUtc); + Assert.Empty(_harness.Publisher.Updates); + } + + [Fact] + public async Task UpdateAsync_ChangeThatLeavesTheTaskActive_SavesStampsAndPublishesIt() + { + // Arrange + var task = NewTask(); + await Tasks.AddAsync(task); + _time.Advance(TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.Progress = 40; + current.Message = "Working"; + return true; + }); + var stored = await Tasks.GetAsync(task.Id); + var active = await Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.Equal(40, stored!.Progress); + Assert.Equal(_time.GetUtcNow().UtcDateTime, result!.UpdatedUtc); + Assert.Equal(result.UpdatedUtc, stored.UpdatedUtc); + Assert.Equal([task.Id], active.ToArray()); + var update = Assert.Single(_harness.Publisher.Updates); + Assert.Equal(task.Id, update.Id); + Assert.Equal(OrganizationId, update.OrganizationId); + Assert.Equal(BackgroundTaskStatus.Running, update.Status); + Assert.Equal(40, update.Progress); + Assert.Equal("Working", update.Message); + Assert.Equal(task.CanCancel, update.CanCancel); + } + + [Fact] + public async Task UpdateAsync_TaskFinishes_RemovesItFromTheActiveIndexAndPublishesTheFinalState() + { + // Arrange + var task = NewTask(); + await Tasks.AddAsync(task); + + // Act + await Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); + var active = await Tasks.GetActiveIdsAsync(OrganizationId); + var listed = await Tasks.GetByOrganizationAsync(OrganizationId); + + // Assert + Assert.Empty(active); + Assert.Equal(BackgroundTaskStatus.Completed, Assert.Single(listed).Status); + var update = Assert.Single(_harness.Publisher.Updates); + Assert.Equal(BackgroundTaskStatus.Completed, update.Status); + Assert.Equal(100, update.Progress); + } + + [Fact] + public async Task UpdateAsync_UnknownTask_ReturnsNullWithoutRunningTheUpdate() + { + // Arrange + bool called = false; + + // Act + var result = await Tasks.UpdateAsync(ObjectId.GenerateNewId().ToString(), _ => + { + called = true; + return true; + }); + + // Assert + Assert.Null(result); + Assert.False(called); + Assert.Empty(_harness.Publisher.Updates); + } + + [Fact] + public async Task UpdateAsync_PublisherIsDown_StillSavesTheChange() + { + // Arrange + var task = NewTask(); + await Tasks.AddAsync(task); + _harness.Publisher.Failure = new TimeoutException("Message bus unavailable"); + + // Act + var result = await Tasks.UpdateAsync(task.Id, current => + { + current.Progress = 10; + return true; + }); + + // Assert + Assert.Equal(10, (await Tasks.GetAsync(task.Id))!.Progress); + Assert.Equal(10, result!.Progress); + } + + [Fact] + public async Task UpdateAsync_LockBusyAtFirst_RetriesWithThePolicyAndSucceeds() + { + // Arrange + var counting = new CountingLockProvider(); + using var harness = new BackgroundTaskTestHarness(TimeProvider.System, Log, decorateLockProvider: counting.Wrap); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + var held = await harness.LockProvider.AcquireAsync(BackgroundTaskService.GetUpdateLockResource(task.Id), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(1)); + counting.Reset(); + counting.OnAttempt = attempts => + { + // Release only after one attempt gave up, so the update can only succeed through a retry. + if (attempts == 2) + held.ReleaseAsync().GetAwaiter().GetResult(); + }; + + // Act + var result = await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Progress = 25; + return true; + }); + + // Assert + Assert.Equal(25, result!.Progress); + Assert.Equal(2, counting.Attempts); + } + + [Fact] + public async Task UpdateAsync_LockNeverReleased_GivesUpAfterTheConfiguredAttempts() + { + // Arrange + var counting = new CountingLockProvider(); + using var harness = new BackgroundTaskTestHarness(TimeProvider.System, Log, decorateLockProvider: counting.Wrap, lockAttempts: 3); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + await using var held = await harness.LockProvider.AcquireAsync(BackgroundTaskService.GetUpdateLockResource(task.Id), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(1)); + counting.Reset(); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.UpdateAsync(task.Id, _ => true)); + + // Assert + Assert.IsType(thrown); + Assert.Equal(3, counting.Attempts); + } + + [Fact] + public async Task CancelAsync_QueuedTask_CancelsImmediatelyAndRemembersTheRequest() + { + // Arrange + var task = NewTask(canCancel: true); + await Tasks.AddAsync(task); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.False(result.IsNotCancellable); + Assert.Equal(BackgroundTaskStatus.Cancelled, result.Task!.Status); + Assert.NotNull(result.Task.CompletedUtc); + Assert.True(await Tasks.IsCancelRequestedAsync(task.Id)); + Assert.Empty(await Tasks.GetActiveIdsAsync(OrganizationId)); + Assert.Equal(BackgroundTaskStatus.Cancelled, Assert.Single(_harness.Publisher.Updates).Status); + } + + [Fact] + public async Task CancelAsync_RunningTaskWithRecentProgress_OnlyRequestsCancellationForTheNextBatch() + { + // Arrange + var task = NewTask(canCancel: true); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + _time.Advance(BackgroundTaskService.StaleAfter - TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Running, result.Task!.Status); + Assert.True(await Tasks.IsCancelRequestedAsync(task.Id)); + Assert.Equal([task.Id], (await Tasks.GetActiveIdsAsync(OrganizationId)).ToArray()); + } + + [Fact] + public async Task CancelAsync_RunningTaskThatStoppedReportingProgress_CancelsImmediately() + { + // Arrange + var task = NewTask(canCancel: true); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, result.Task!.Status); + Assert.Empty(await Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Fact] + public async Task CancelAsync_TaskThatCannotBeCancelled_IsRejectedAndLeftRunning() + { + // Arrange + var task = NewTask(canCancel: false); + await Tasks.AddAsync(task); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.True(result.IsNotCancellable); + Assert.Equal(BackgroundTaskStatus.Queued, (await Tasks.GetAsync(task.Id))!.Status); + Assert.False(await Tasks.IsCancelRequestedAsync(task.Id)); + Assert.Empty(_harness.Publisher.Updates); + } + + [Fact] + public async Task CancelAsync_AbandonedTaskThatCannotBeCancelled_CanStillBeCleared() + { + // Arrange + var task = NewTask(canCancel: false); + await Tasks.AddAsync(task); + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.False(result.IsNotCancellable); + Assert.Equal(BackgroundTaskStatus.Cancelled, result.Task!.Status); + } + + [Fact] + public async Task CancelAsync_FinishedOrUnknownTask_ChangesNothing() + { + // Arrange + var task = NewTask(canCancel: true); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); + int publishedBefore = _harness.Publisher.Updates.Count; + + // Act + var finished = await Tasks.CancelAsync(task.Id); + var unknown = await Tasks.CancelAsync(ObjectId.GenerateNewId().ToString()); + + // Assert + Assert.Equal(BackgroundTaskStatus.Completed, finished.Task!.Status); + Assert.False(await Tasks.IsCancelRequestedAsync(task.Id)); + Assert.Null(unknown.Task); + Assert.Equal(publishedBefore, _harness.Publisher.Updates.Count); + } + + [Fact] + public async Task CreateAndEnqueueAsync_Success_QueuesTheWorkItemWithTheTaskIdAndPublishesQueued() + { + // Arrange + var task = NewTask(); + + // Act + var created = await Tasks.CreateAndEnqueueAsync(task, t => new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = "p1", TaskId = t.Id }); + var entry = await _harness.Queue.DequeueAsync(TestCancellationToken); + + // Assert + Assert.Same(task, created); + var workItem = Assert.IsType(_harness.Queue.Serializer.Deserialize(entry!.Value.Data, typeof(ResetProjectDataWorkItem))); + Assert.Equal(task.Id, workItem.TaskId); + var update = Assert.Single(_harness.Publisher.Updates); + Assert.Equal(BackgroundTaskStatus.Queued, update.Status); + Assert.Equal(task.Id, update.Id); + } + + [Fact] + public async Task EnqueueAsync_QueueRejectsTheWorkItem_FailsTheTaskSoItIsNotLeftWaiting() + { + // Arrange + var queue = new Mock>(); + queue.SetupGet(q => q.Serializer).Returns(_harness.Queue.Serializer); + queue.Setup(q => q.EnqueueAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new TimeoutException("Queue unavailable")); + using var harness = new BackgroundTaskTestHarness(_time, Log, queue: queue.Object); + var task = NewTask(); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CreateAndEnqueueAsync(task, t => new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = "p1", TaskId = t.Id })); + + // Assert + Assert.IsType(thrown); + var stored = await harness.Tasks.GetAsync(task.Id); + Assert.Equal(BackgroundTaskStatus.Failed, stored!.Status); + Assert.NotNull(stored.CompletedUtc); + Assert.Empty(await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + Assert.Equal(BackgroundTaskStatus.Failed, Assert.Single(harness.Publisher.Updates).Status); + } + + [Fact] + public void CacheKeysAndLockResources_ShareThePrefixAndAreNamedByKind() + { + // Arrange + string[] cacheKeys = + [ + BackgroundTaskService.GetTaskCacheKey("a"), + BackgroundTaskService.GetCancelCacheKey("a"), + BackgroundTaskService.GetOrganizationIndexCacheKey("o"), + BackgroundTaskService.GetActiveIndexCacheKey("o"), + BulkOperationService_GetPinnedSelectionCacheKey("a") + ]; + string[] lockResources = [BackgroundTaskService.GetUpdateLockResource("a"), BackgroundTaskService.GetCreateLockResource("o")]; + + // Act + var all = cacheKeys.Concat(lockResources).ToList(); + + // Assert + Assert.All(all, key => Assert.StartsWith(BackgroundTaskService.KeyPrefix + ":", key)); + Assert.Equal(all.Count, all.Distinct().Count()); + } + + private static string BulkOperationService_GetPinnedSelectionCacheKey(string id) => Exceptionless.Core.Services.BulkOperations.BulkOperationService.GetPinnedSelectionCacheKey(id); + + private static BackgroundTask NewTask(string organizationId = OrganizationId, bool canCancel = false) + { + return new BackgroundTask + { + OrganizationId = organizationId, + ProjectId = "537650f3b77efe23a47914f6", + Type = BackgroundTaskTypes.ResetProjectData, + Description = "Reset data for Test", + CreatedByUserId = "537650f3b77efe23a47914f8", + CanCancel = canCancel + }; + } + + private sealed class CountingLockProvider + { + private int _attempts; + + public int Attempts => Volatile.Read(ref _attempts); + + /// Called with the attempt number (since the last reset) as an attempt to acquire a lock starts. + public Action? OnAttempt { get; set; } + + public void Reset() => Volatile.Write(ref _attempts, 0); + + public ILockProvider Wrap(ILockProvider inner) + { + var mock = new Mock(); + mock.Setup(p => p.AcquireAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (resource, expires, releaseOnDispose, cancellationToken) => + { + int attempt = Interlocked.Increment(ref _attempts); + OnAttempt?.Invoke(attempt); + return await inner.AcquireAsync(resource, expires, releaseOnDispose, cancellationToken); + }); + return mock.Object; + } + } +} diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs new file mode 100644 index 0000000000..a85e7d596d --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs @@ -0,0 +1,266 @@ +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Serializer; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class BulkOperationServiceTests : TestWithLoggingBase, IDisposable +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private const string OtherOrganizationId = "537650f3b77efe23a47914f5"; + private const string ProjectId = "537650f3b77efe23a47914f6"; + private const string OtherProjectId = "537650f3b77efe23a47914f7"; + private readonly FakeTimeProvider _time = new(new DateTimeOffset(2026, 11, 3, 4, 5, 6, TimeSpan.Zero)); + private readonly BackgroundTaskTestHarness _harness; + private readonly BulkOperationService _service; + + public BulkOperationServiceTests(ITestOutputHelper output) : base(output) + { + _harness = new BackgroundTaskTestHarness(_time, Log); + _service = CreateService(_harness); + } + + public void Dispose() + { + _harness.Dispose(); + } + + [Fact] + public async Task CreateAsync_IdSelection_CountsDescribesAndQueuesAnUncancelledCancellableTask() + { + // Arrange + var task = NewTask("test.destructive", ProjectId, ids: 3); + + // Act + var result = await _service.CreateAsync(task); + var entry = await _harness.Queue.DequeueAsync(TestCancellationToken); + + // Assert + Assert.Same(task, result.Task); + Assert.Equal(3, task.Total); + Assert.Equal("Record 3", task.Description); + Assert.True(task.CanCancel); + Assert.Equal(BackgroundTaskStatus.Queued, task.Status); + var workItem = Assert.IsType(_harness.Queue.Serializer.Deserialize(entry!.Value.Data, typeof(BulkOperationWorkItem))); + Assert.Equal(task.Id, workItem.TaskId); + Assert.Equal(OrganizationId, workItem.OrganizationId); + Assert.Equal(BackgroundTaskStatus.Queued, Assert.Single(_harness.Publisher.Updates).Status); + } + + [Fact] + public async Task CreateAsync_UnknownTypeOrMissingBulkDetails_Throws() + { + // Arrange + var unknown = NewTask("test.unknown", ProjectId); + var withoutDetails = NewTask("test.destructive", ProjectId); + withoutDetails.Bulk = null; + + // Act + var unknownThrown = await Record.ExceptionAsync(async () => await _service.CreateAsync(unknown)); + var detailsThrown = await Record.ExceptionAsync(async () => await _service.CreateAsync(withoutDetails)); + + // Assert + Assert.IsType(unknownThrown); + Assert.IsType(detailsThrown); + Assert.Empty(await _harness.Tasks.GetByOrganizationAsync(OrganizationId)); + } + + [Fact] + public async Task CreateAsync_DestructiveTaskActiveOnTheSameProject_Conflicts() + { + // Arrange + var first = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Act + var second = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.Null(second.Task); + Assert.Equal(first.Task!.Id, second.Conflict?.Id); + Assert.Single(await _harness.Tasks.GetByOrganizationAsync(OrganizationId)); + Assert.Equal(1, (await _harness.Queue.GetQueueStatsAsync()).Enqueued); + } + + [Theory] + [InlineData(null, ProjectId)] + [InlineData(ProjectId, null)] + [InlineData(null, null)] + public async Task CreateAsync_OrganizationWideAndProjectScopedDestructiveTasks_Conflict(string? activeProjectId, string? candidateProjectId) + { + // Arrange + var active = await _service.CreateAsync(NewTask("test.destructive", activeProjectId)); + + // Act + var candidate = await _service.CreateAsync(NewTask("test.destructive", candidateProjectId)); + + // Assert + Assert.Null(candidate.Task); + Assert.Equal(active.Task!.Id, candidate.Conflict?.Id); + } + + [Fact] + public async Task CreateAsync_DestructiveTaskActiveOnAnotherProjectOrOrganization_DoesNotConflict() + { + // Arrange + await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Act + var otherProject = await _service.CreateAsync(NewTask("test.destructive", OtherProjectId)); + var otherOrganization = await _service.CreateAsync(NewTask("test.destructive", null, OtherOrganizationId)); + + // Assert + Assert.NotNull(otherProject.Task); + Assert.NotNull(otherOrganization.Task); + } + + [Fact] + public async Task CreateAsync_NonDestructiveTasks_NeitherConflictNorBlockDestructiveOnes() + { + // Arrange + var firstSafe = await _service.CreateAsync(NewTask("test.safe", ProjectId)); + + // Act + var secondSafe = await _service.CreateAsync(NewTask("test.safe", ProjectId)); + var destructive = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + var safeAfterDestructive = await _service.CreateAsync(NewTask("test.safe", ProjectId)); + + // Assert + Assert.NotNull(firstSafe.Task); + Assert.NotNull(secondSafe.Task); + Assert.NotNull(destructive.Task); + Assert.NotNull(safeAfterDestructive.Task); + } + + [Fact] + public async Task CreateAsync_ActiveDestructiveTaskStoppedReportingProgress_IsRetiredAndReplaced() + { + // Arrange + var stale = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var replacement = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + var retired = await _harness.Tasks.GetAsync(stale.Task!.Id); + + // Assert + Assert.NotNull(replacement.Task); + Assert.Equal(BackgroundTaskStatus.Failed, retired!.Status); + Assert.NotNull(retired.CompletedUtc); + Assert.Equal([replacement.Task.Id], (await _harness.Tasks.GetActiveIdsAsync(OrganizationId)).ToArray()); + Assert.Contains(_harness.Publisher.Updates, u => u.Id == stale.Task.Id && u.Status == BackgroundTaskStatus.Failed); + } + + [Fact] + public async Task CreateAsync_FinishedTasksStillInTheActiveIndex_AreDroppedFromIt() + { + // Arrange + var finished = NewTask("test.destructive", ProjectId); + var expired = NewTask("test.destructive", ProjectId); + await _harness.Tasks.AddAsync(finished); + await _harness.Tasks.AddAsync(expired); + // Simulate an index that was not cleaned: the task finished without being removed, and another record expired. + finished.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + await _harness.Tasks.SaveAsync(finished); + await _harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(expired.Id)); + + // Act + var created = await _service.CreateAsync(NewTask("test.destructive", ProjectId)); + var active = await _harness.Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.NotNull(created.Task); + Assert.Equal([created.Task.Id], active.ToArray()); + } + + [Fact] + public async Task CreateAsync_QueueRejectsTheWorkItem_FailsTheTaskSoItDoesNotBlockTheProject() + { + // Arrange + var queue = new Mock>(); + queue.SetupGet(q => q.Serializer).Returns(_harness.Queue.Serializer); + queue.SetupSequence(q => q.EnqueueAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new TimeoutException("Queue unavailable")) + .ReturnsAsync("queued"); + using var failingHarness = new BackgroundTaskTestHarness(_time, Log, queue: queue.Object); + var service = CreateService(failingHarness); + var task = NewTask("test.destructive", ProjectId); + + // Act + var thrown = await Record.ExceptionAsync(async () => await service.CreateAsync(task)); + var orphan = await failingHarness.Tasks.GetAsync(task.Id); + var activeAfterFailure = await failingHarness.Tasks.GetActiveIdsAsync(OrganizationId); + var retry = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.IsType(thrown); + Assert.Equal(BackgroundTaskStatus.Failed, orphan!.Status); + Assert.Empty(activeAfterFailure); + Assert.NotNull(retry.Task); + Assert.Equal(BackgroundTaskStatus.Queued, retry.Task.Status); + } + + [Fact] + public async Task PublishLostAsync_RecordIsGone_PublishesAFinalFailedUpdateFromTheWorkItem() + { + // Arrange + var workItem = new BulkOperationWorkItem { TaskId = Foundatio.Repositories.Utility.ObjectId.GenerateNewId().ToString(), OrganizationId = OrganizationId, Type = "test.destructive", Description = "Record 7" }; + + // Act + await _service.PublishLostAsync(workItem); + + // Assert + var update = Assert.Single(_harness.Publisher.Updates); + Assert.Equal(workItem.TaskId, update.Id); + Assert.Equal(OrganizationId, update.OrganizationId); + Assert.Equal("Record 7", update.Description); + Assert.Equal(BackgroundTaskStatus.Failed, update.Status); + Assert.False(String.IsNullOrEmpty(update.Message)); + } + + private BulkOperationService CreateService(BackgroundTaskTestHarness harness) + { + return new BulkOperationService(harness.Tasks, harness.Cache, harness.LockProvider, + [new StubHandler("test.destructive", isDestructive: true), new StubHandler("test.safe", isDestructive: false)], [], _time); + } + + private static BackgroundTask NewTask(string type, string? projectId, string organizationId = OrganizationId, int ids = 1) + { + return new BackgroundTask + { + OrganizationId = organizationId, + ProjectId = projectId, + Type = type, + CreatedByUserId = "537650f3b77efe23a47914f8", + Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Ids, + Ids = Enumerable.Range(0, ids).Select(_ => Foundatio.Repositories.Utility.ObjectId.GenerateNewId().ToString()).ToList() + } + } + }; + } + + private sealed class StubHandler(string type, bool isDestructive) : IBulkOperationHandler + { + public string Type => type; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => isDestructive; + + public Task AuthorizeAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task ValidateAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public string Describe(BackgroundTask task) => $"Record {task.Total}"; + public Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) => Task.FromResult(new BulkBatchResult(ids.Count, 0)); + } +} diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs new file mode 100644 index 0000000000..1097b95f80 --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs @@ -0,0 +1,233 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Core.Utility; +using Foundatio.Repositories; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class EventBulkSelectionResolverTests : IntegrationTestsBase +{ + private readonly EventBulkSelectionResolver _resolver; + private readonly IEventRepository _eventRepository; + + public EventBulkSelectionResolverTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) + { + _resolver = GetService(); + _eventRepository = GetService(); + } + + protected override async Task ResetDataAsync() + { + await base.ResetDataAsync(); + await GetService().CreateDataAsync(); + } + + [Fact] + public async Task GetPageAsync_MoreEventsThanOnePage_PagesWithTheCursorInDateOrderWithoutGapsOrRepeats() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var task = NewFilterTask(""); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 2); + + // Assert + Assert.Equal(5, preparation.Total); + Assert.Equal([2, 2, 1], pages.Select(p => p.Count).ToArray()); + Assert.Equal(events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), pages.SelectMany(p => p).ToArray()); + } + + [Fact] + public async Task GetPageAsync_EarlierPagesAreDeletedBetweenPages_ContinuesAfterTheCursorWithoutSkippingAnything() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + for (int i = 0; i < 5; i++) + d.Event().TestProject().Date(now.AddMinutes(-i - 1)); + }); + var task = NewFilterTask(""); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + var ordered = events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(); + + // Act + var first = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 2, TestCancellationToken); + await _eventRepository.RemoveAsync(events.Where(e => first.Ids.Contains(e.Id)).ToList(), o => o.ImmediateConsistency()); + var second = await _resolver.GetPageAsync(task, preparation.PinnedState, first.NextCursor, 2, TestCancellationToken); + var third = await _resolver.GetPageAsync(task, preparation.PinnedState, second.NextCursor, 2, TestCancellationToken); + + // Assert + Assert.Equal(ordered[..2], first.Ids.ToArray()); + Assert.Equal(ordered[2..4], second.Ids.ToArray()); + Assert.Equal(ordered[4..], third.Ids.ToArray()); + } + + [Fact] + public async Task GetPageAsync_EventsStoredAfterTheCutoff_AreNeverSelected() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, early) = await CreateDataAsync(d => d.Event().TestProject().Message("early").Date(now.AddHours(-1)).CreatedDate(now.UtcDateTime)); + var cutoff = now.UtcDateTime.AddMinutes(1); + TimeProvider.Advance(TimeSpan.FromMinutes(5)); + var (_, late) = await CreateDataAsync(d => d.Event().TestProject().Message("late").Date(now.AddHours(-1)).CreatedDate(TimeProvider.GetUtcNow().UtcDateTime)); + var task = NewFilterTask("", createdBefore: cutoff); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 10); + + // Assert + Assert.Equal(1, preparation.Total); + Assert.Equal(early.Select(e => e.Id).ToArray(), pages.SelectMany(p => p).ToArray()); + Assert.DoesNotContain(late.Single().Id, pages.SelectMany(p => p)); + } + + [Fact] + public async Task GetPageAsync_StacksChangeStatusAfterPrepare_StillPagesTheEventsThatWereConfirmed() + { + // Arrange + var (stacks, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("open").Status(StackStatus.Open); + d.Event().TestProject().Message("fixed").Status(StackStatus.Fixed); + }); + var task = NewFilterTask("status:open"); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + var openStack = stacks.Single(s => s.Status == StackStatus.Open); + openStack.Status = StackStatus.Fixed; + await GetService().SaveAsync(openStack, o => o.ImmediateConsistency()); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 10); + + // Assert + Assert.Equal(1, preparation.Total); + Assert.Equal([events.Single(e => e.Message == "open").Id], pages.SelectMany(p => p).ToArray()); + } + + [Fact] + public async Task GetPageAsync_NothingWasPrepared_ReturnsAnEmptyPageInsteadOfSelectingEverything() + { + // Arrange + await CreateDataAsync(d => d.Event().TestProject()); + var task = NewFilterTask(""); + + // Act + var page = await _resolver.GetPageAsync(task, null, null, 10, TestCancellationToken); + + // Assert + Assert.Empty(page.Ids); + Assert.Null(page.NextCursor); + } + + [Fact] + public async Task GetPageAsync_ProjectScopedTask_OnlySelectsThatProjectsEvents() + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(d => + { + d.Event().TestProject().Message("own").Date(now.AddHours(-1)); + d.Event().TestProject().Project(SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID).Message("other project").Date(now.AddHours(-1)); + d.Event().FreeProject().Message("other organization").Date(now.AddHours(-1)); + }); + var task = NewFilterTask(""); + task.ProjectId = SampleDataService.TEST_PROJECT_ID; + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 10); + + // Assert + Assert.Equal(1, preparation.Total); + Assert.Equal([events.Single(e => e.Message == "own").Id], pages.SelectMany(p => p).ToArray()); + } + + [Fact] + public async Task PrepareAsync_RelativeDates_ArePinnedToTheCutoff() + { + // Arrange + var task = NewFilterTask("date:[now-2h TO now]"); + + // Act + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Assert + Assert.True(preparation.Check.IsOk); + Assert.NotNull(preparation.PinnedState); + Assert.DoesNotContain("now", preparation.PinnedState, StringComparison.OrdinalIgnoreCase); + Assert.Contains("||-2h", preparation.PinnedState); + } + + [Theory] + [InlineData(SampleDataService.TEST_ORG_ID, "type:log", BulkOperationCheckStatus.Ok)] + [InlineData(SampleDataService.TEST_ORG_ID, "type:(log", BulkOperationCheckStatus.Invalid)] + [InlineData(SampleDataService.FREE_ORG_ID, "tag:premium-tag", BulkOperationCheckStatus.PlanLimit)] + public async Task ValidateSelectionAsync_Filter_IsCheckedForSyntaxAndPlan(string organizationId, string filter, BulkOperationCheckStatus expected) + { + // Arrange + var organization = await GetService().GetByIdAsync(organizationId); + var selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Filter = filter }; + + // Act + var result = await _resolver.ValidateSelectionAsync(organization!, selection); + + // Assert + Assert.Equal(expected, result.Status); + if (expected is BulkOperationCheckStatus.Invalid) + Assert.Equal("selection.filter", result.Field); + } + + private async Task>> GetAllPagesAsync(BackgroundTask task, string? pinnedState, int size) + { + var pages = new List>(); + string? cursor = null; + for (int guard = 0; guard < 20; guard++) + { + var page = await _resolver.GetPageAsync(task, pinnedState, cursor, size, TestCancellationToken); + if (page.Ids.Count == 0) + return pages; + + pages.Add(page.Ids.ToList()); + cursor = page.NextCursor; + } + + throw new InvalidOperationException("Paging did not finish."); + } + + private BackgroundTask NewFilterTask(string filter, DateTime? createdBefore = null) + { + var now = TimeProvider.GetUtcNow().UtcDateTime; + return new BackgroundTask + { + OrganizationId = SampleDataService.TEST_ORG_ID, + Type = "events.delete", + CreatedUtc = now, + CreatedByUserId = "537650f3b77efe23a47914f8", + Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Filter, + Filter = filter, + UtcStart = now.AddDays(-7), + UtcEnd = now, + UtcCreatedBefore = createdBefore ?? now + } + } + }; + } +} diff --git a/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs new file mode 100644 index 0000000000..553633c64f --- /dev/null +++ b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs @@ -0,0 +1,79 @@ +using Exceptionless.Core.Messaging.Models; +using Exceptionless.Core.Services.BackgroundTasks; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Tests.Utility; + +/// +/// A on in-memory infrastructure for isolated tests: no Elasticsearch, a recording +/// publisher instead of the message bus, and a quick resilience policy for lock retries. +/// +public sealed class BackgroundTaskTestHarness : IDisposable +{ + public BackgroundTaskTestHarness(TimeProvider timeProvider, ILoggerFactory loggerFactory, BackgroundTaskOptions? options = null, Func? decorateLockProvider = null, + IQueue? queue = null, int lockAttempts = 3) + { + TimeProvider = timeProvider; + Cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { CloneValues = true, TimeProvider = timeProvider, LoggerFactory = loggerFactory }); + MessageBus = new InMemoryMessageBus(new InMemoryMessageBusOptions { TimeProvider = timeProvider, LoggerFactory = loggerFactory }); + LockProvider = new CacheLockProvider(Cache, MessageBus, timeProvider, loggerFactory); + if (decorateLockProvider is not null) + LockProvider = decorateLockProvider(LockProvider); + Queue = queue ?? new InMemoryQueue(new InMemoryQueueOptions { TimeProvider = timeProvider, LoggerFactory = loggerFactory }); + Publisher = new RecordingPublisher(); + + var policies = new ResiliencePolicyProvider(timeProvider, loggerFactory) + .WithPolicy(b => b.WithMaxAttempts(lockAttempts).WithDelay(TimeSpan.FromMilliseconds(10))); + Tasks = new BackgroundTaskService(Cache, LockProvider, Publisher, Queue, policies, options ?? new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.FromMilliseconds(100) }, + timeProvider, loggerFactory.CreateLogger()); + } + + public TimeProvider TimeProvider { get; } + public InMemoryCacheClient Cache { get; } + public InMemoryMessageBus MessageBus { get; } + public ILockProvider LockProvider { get; } + public IQueue Queue { get; } + public RecordingPublisher Publisher { get; } + public BackgroundTaskService Tasks { get; } + + public void Dispose() + { + Queue.Dispose(); + MessageBus.Dispose(); + Cache.Dispose(); + } + + public sealed class RecordingPublisher : IMessagePublisher + { + private readonly List _messages = []; + + /// When set, publishing throws, like a message bus that is down. + public Exception? Failure { get; set; } + + public IReadOnlyList Updates + { + get + { + lock (_messages) + return _messages.OfType().ToList(); + } + } + + public Task PublishAsync(Type messageType, object message, MessageOptions? options = null, CancellationToken cancellationToken = default) + { + if (Failure is not null) + throw Failure; + + lock (_messages) + _messages.Add(message); + + return Task.CompletedTask; + } + } +} From bd86727831e3fa193845511d3b8d2c927e6a9703 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 3 Nov 2026 23:35:49 -0600 Subject: [PATCH 14/42] Hide internal errors from task messages and let abandoned tasks be cleared A failed run shows a generic message instead of the exception text. Cancelling an abandoned running task says it stopped responding, and the panel offers Cancel for any task that has not reported progress for two hours. --- .../BackgroundTaskTrackingWorkItemHandler.cs | 5 +++-- .../BackgroundTasks/BackgroundTaskService.cs | 2 +- .../tasks/components/task-item.svelte | 6 +++-- .../tasks/components/task-item.svelte.test.ts | 12 +++++----- .../tasks/components/task-panel.svelte | 6 ++--- .../src/lib/features/tasks/models.test.ts | 22 +++++++++++++++++++ .../src/lib/features/tasks/models.ts | 12 ++++++++++ ...kgroundTaskTrackingWorkItemHandlerTests.cs | 2 +- .../BackgroundTaskServiceTests.cs | 21 ++++++++++++++++++ 9 files changed, 73 insertions(+), 15 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs index bb5437979f..332e6fbde9 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs @@ -86,14 +86,15 @@ await TryUpdateAsync(taskId, task => { await _inner.HandleItemAsync(tracked); } - catch (Exception ex) + catch (Exception) { await TryUpdateAsync(taskId, task => { if (!task.IsActive) return false; - task.Finish(BackgroundTaskStatus.Failed, $"Failed: {ex.Message}", UtcNow); + // The exception is logged by the job; its message can contain internal details, so the user sees a generic one. + task.Finish(BackgroundTaskStatus.Failed, "An error occurred. It will be retried automatically if possible.", UtcNow); return true; }); diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs index 5af12c2c16..c8fe4e0fe0 100644 --- a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -242,7 +242,7 @@ public async Task CancelAsync(string id) if (isRunningNow) return false; - current.Finish(BackgroundTaskStatus.Cancelled, "Cancelled before it started.", utcNow); + current.Finish(BackgroundTaskStatus.Cancelled, current.Status is BackgroundTaskStatus.Queued ? "Cancelled before it started." : "Cancelled after it stopped responding.", utcNow); return true; }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte index 234872d1e4..aa74ad30f5 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte @@ -12,6 +12,8 @@ import { BackgroundTaskStatus, getProgressPercent, hasCounts, isTaskActive, type ViewBackgroundTask } from '../models'; interface Props { + /** Whether to offer Cancel while the task is active. */ + canCancel?: boolean; isCancelling?: boolean; onCancel: (task: ViewBackgroundTask) => void; onDismiss: (task: ViewBackgroundTask) => void; @@ -26,7 +28,7 @@ [BackgroundTaskStatus.Running]: 'in progress' }; - let { isCancelling = false, onCancel, onDismiss, task }: Props = $props(); + let { canCancel = false, isCancelling = false, onCancel, onDismiss, task }: Props = $props(); const isActive = $derived(isTaskActive(task.status)); const percent = $derived(getProgressPercent(task)); @@ -80,7 +82,7 @@

{#if isActive} - {#if task.can_cancel} + {#if canCancel} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts index cad36780f5..ed3a2e523a 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts @@ -36,7 +36,7 @@ describe('TaskItem', () => { // Arrange const onCancel = vi.fn(); const task = createTask(); - render(TaskItem, { props: { onCancel, onDismiss: vi.fn(), task } }); + render(TaskItem, { props: { canCancel: true, onCancel, onDismiss: vi.fn(), task } }); // Act await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); @@ -69,7 +69,7 @@ describe('TaskItem', () => { it('labels Cancel with the task it cancels', () => { // Arrange & Act - render(TaskItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() } }); + render(TaskItem, { props: { canCancel: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() } }); // Assert expect(screen.getByRole('button', { name: 'Cancel Delete 1,234 events matching filter' })).toBeTruthy(); @@ -78,7 +78,7 @@ describe('TaskItem', () => { it('shows the queued state without a count', () => { // Arrange & Act render(TaskItem, { - props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ processed: 0, status: BackgroundTaskStatus.Queued }) } + props: { canCancel: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ processed: 0, status: BackgroundTaskStatus.Queued }) } }); // Assert @@ -88,7 +88,7 @@ describe('TaskItem', () => { it('disables Cancel once cancellation was requested', () => { // Arrange & Act - render(TaskItem, { props: { isCancelling: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() } }); + render(TaskItem, { props: { canCancel: true, isCancelling: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() } }); // Assert expect((screen.getByRole('button', { name: /^Cancel / }) as HTMLButtonElement).disabled).toBe(true); @@ -147,9 +147,9 @@ describe('TaskItem', () => { expect(screen.getByRole('progressbar').getAttribute('aria-valuenow')).toBe('50'); }); - it('does not offer Cancel for a running task that cannot be cancelled', () => { + it('does not offer Cancel for a running task when cancelling is not allowed', () => { // Arrange & Act - render(TaskItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ can_cancel: false }) } }); + render(TaskItem, { props: { canCancel: false, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ can_cancel: false }) } }); // Assert expect(screen.queryByRole('button', { name: /^Cancel / })).toBeNull(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte index f1f7f071e5..c4c56fc9d8 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte @@ -9,7 +9,7 @@ import { organization } from '#features/organizations/context.svelte.ts'; import { cancelTask, getTasksQuery, invalidateTaskAffectedQueries, updateTaskFromMessage } from '../api.svelte'; - import { isTaskActive, selectVisibleTasks, type ViewBackgroundTask } from '../models'; + import { canCancelTask, isTaskActive, selectVisibleTasks, type ViewBackgroundTask } from '../models'; import TaskItem from './task-item.svelte'; const queryClient = useQueryClient(); @@ -39,7 +39,7 @@ return () => window.clearInterval(timer); }); - // An task that was seen running and is now finished changed events and stacks, whichever way the update arrived. + // A task that was seen running and is now finished changed events and stacks, whichever way the update arrived. $effect(() => { for (const task of tasksQuery.data ?? []) { if (isTaskActive(task.status)) { @@ -87,7 +87,7 @@ role="region" > {#each visibleTasks as task (task.id)} - + {/each} {/if} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts index 789f8a82d2..f368d93201 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts @@ -4,9 +4,11 @@ import { applyTaskUpdate, BackgroundTaskStatus, BulkOperationSelectionKind, + canCancelTask, FINISHED_TASK_VISIBLE_MS, getProgressPercent, selectVisibleTasks, + STALE_TASK_MS, type ViewBackgroundTask } from './models'; @@ -101,6 +103,26 @@ describe('getProgressPercent', () => { }); }); +describe('canCancelTask', () => { + const updated = new Date('2026-01-01T10:00:00Z').getTime(); + + it.each([ + ['a cancellable running task', { can_cancel: true }, updated, true], + ['a running task that cannot be cancelled', { can_cancel: false }, updated + STALE_TASK_MS - 1, false], + ['an abandoned task that cannot be cancelled', { can_cancel: false }, updated + STALE_TASK_MS, true], + ['a finished task', { can_cancel: true, status: BackgroundTaskStatus.Completed }, updated, false] + ])('%s: %j', (_name, overrides, now, expected) => { + // Arrange + const task = createTask(overrides); + + // Act + const result = canCancelTask(task, now); + + // Assert + expect(result).toBe(expected); + }); +}); + describe('selectVisibleTasks', () => { const now = new Date('2026-01-01T10:30:00Z').getTime(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts index 334ecdb53b..4982ccb53d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts @@ -56,6 +56,18 @@ export function hasCounts(task: Pick): boolean { return task.total > 0; } +/** An active task that has not reported progress for this long is abandoned, and the server lets anyone clear it. */ +export const STALE_TASK_MS = 2 * 60 * 60 * 1000; + +/** Whether the panel offers Cancel: the task can be stopped, or it is abandoned and only needs clearing. */ +export function canCancelTask(task: Pick, now: number): boolean { + if (!isTaskActive(task.status)) { + return false; + } + + return task.can_cancel || now - new Date(task.updated_utc).getTime() >= STALE_TASK_MS; +} + export function isTaskActive(status: BackgroundTaskStatus): boolean { return status === BackgroundTaskStatus.Queued || status === BackgroundTaskStatus.Running; } diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs index c900f94611..f88d6c4fde 100644 --- a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs @@ -110,7 +110,7 @@ public async Task HandleItemAsync_WorkThrows_FailsTheTaskAndRethrowsThenARetryBr // Assert Assert.IsType(thrown); Assert.Equal(BackgroundTaskStatus.Failed, failed!.Status); - Assert.Contains("Elasticsearch unavailable", failed.Message); + Assert.DoesNotContain("Elasticsearch unavailable", failed.Message); Assert.Empty(activeAfterFailure); Assert.Equal(BackgroundTaskStatus.Running, duringRetry!.Status); Assert.Null(duringRetry.CompletedUtc); diff --git a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs index 7996358f9e..e287892be0 100644 --- a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs @@ -364,6 +364,27 @@ public async Task CancelAsync_AbandonedTaskThatCannotBeCancelled_CanStillBeClear Assert.Equal(BackgroundTaskStatus.Cancelled, result.Task!.Status); } + [Fact] + public async Task CancelAsync_AbandonedRunningTask_SaysItStoppedResponding() + { + // Arrange + var task = NewTask(canCancel: false); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + + // Act + var result = await Tasks.CancelAsync(task.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, result.Task!.Status); + Assert.Equal("Cancelled after it stopped responding.", result.Task.Message); + } + [Fact] public async Task CancelAsync_FinishedOrUnknownTask_ChangesNothing() { From a7e836db552106b5e0ee1443672c4b42dddcd000 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 3 Nov 2026 23:41:22 -0600 Subject: [PATCH 15/42] Give the relative date freezer tests an Arrange section --- .../BulkOperations/RelativeDateFreezerTests.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs index bce2a78e2d..e3bc0c0c93 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs @@ -21,8 +21,12 @@ public RelativeDateFreezerTests(ITestOutputHelper output) : base(output) { } [InlineData("data.date:[now/d-4d TO now/d+1d}", "idx.date-d:[\"2026-11-03T04:05:06.789Z||/d-4d\" TO \"2026-11-03T04:05:06.789Z||/d+1d\"}")] public async Task FreezeAsync_RelativeDatesOnDateFields_AreAnchoredToTheGivenTime(string filter, string expected) { + // Arrange + var createdUtc = UtcNow; + Func> isDateFieldAsync = IsDateFieldAsync; + // Act - string? frozen = await RelativeDateFreezer.FreezeAsync(filter, UtcNow, IsDateFieldAsync); + string? frozen = await RelativeDateFreezer.FreezeAsync(filter, createdUtc, isDateFieldAsync); // Assert Assert.Equal(expected, frozen); @@ -43,8 +47,12 @@ public async Task FreezeAsync_RelativeDatesOnDateFields_AreAnchoredToTheGivenTim [InlineData("ref.parent:[now TO z]")] public async Task FreezeAsync_WithoutRelativeDatesOnDateFields_LeavesTheFilterAlone(string? filter) { + // Arrange + var createdUtc = UtcNow; + Func> isDateFieldAsync = IsDateFieldAsync; + // Act - string? frozen = await RelativeDateFreezer.FreezeAsync(filter, UtcNow, IsDateFieldAsync); + string? frozen = await RelativeDateFreezer.FreezeAsync(filter, createdUtc, isDateFieldAsync); // Assert Assert.DoesNotContain("||", frozen ?? String.Empty); From b81fe3a541ce2fff86b8e3b0a0df2695d466bafe Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Tue, 10 Nov 2026 06:50:24 -0600 Subject: [PATCH 16/42] Fix background task lifecycle and active task visibility --- .../BackgroundTaskTrackingWorkItemHandler.cs | 9 ++-- .../BackgroundTasks/BackgroundTaskService.cs | 20 ++++++--- .../Endpoints/BackgroundTaskEndpointTests.cs | 25 +++++++++++ .../BulkOperationEndpointTests.Reliability.cs | 2 +- ...kgroundTaskTrackingWorkItemHandlerTests.cs | 26 +++++++++--- .../BackgroundTaskServiceTests.cs | 42 ++++++++++++++++++- 6 files changed, 107 insertions(+), 17 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs index 332e6fbde9..2a2890900c 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs @@ -80,7 +80,8 @@ await TryUpdateAsync(taskId, task => await context.ReportProgressAsync(progress, message); }); - await StartAsync(taskId); + if (!await StartAsync(taskId)) + return; try { @@ -121,9 +122,9 @@ await TryUpdateAsync(taskId, task => /// /// Marks the task running. A queue retry runs the work item again after it failed, which brings the task back to running - /// and back into the active index. A cancelled or completed task stays as it is. + /// and back into the active index. A cancelled or completed task must not run again. /// - private async Task StartAsync(string taskId) + private async Task StartAsync(string taskId) { bool wasFinished = false; var task = await TryUpdateAsync(taskId, current => @@ -145,6 +146,8 @@ private async Task StartAsync(string taskId) if (wasFinished && task is not null) await TryAsync(taskId, () => _taskService.AddToActiveIndexAsync(task.OrganizationId, task.Id)); + + return task is not { Status: BackgroundTaskStatus.Cancelled or BackgroundTaskStatus.Completed }; } private async Task TryUpdateAsync(string taskId, Func update) diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs index c8fe4e0fe0..6275f80cb7 100644 --- a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -28,7 +28,7 @@ public class BackgroundTaskService /// An active task that has not reported progress for this long is considered abandoned. public static readonly TimeSpan StaleAfter = TimeSpan.FromHours(2); - /// How many tasks the list shows, newest first. + /// How many finished tasks the list shows, in addition to all active tasks. public const int ListLimit = 50; /// Every cache key and lock resource of background tasks starts with this prefix. @@ -77,26 +77,34 @@ public BackgroundTaskService(ICacheClient cache, ILockProvider lockProvider, IMe } /// - /// The newest tasks of the organization that are still stored, newest first. Every indexed id is read so a busy - /// organization cannot push recent tasks out of the list; only the history that is shown is limited. + /// All active tasks and the newest finished tasks of the organization, newest first. Only finished history is limited. /// public async Task> GetByOrganizationAsync(string organizationId, int limit = ListLimit) { + var activeIds = (await GetActiveIdsAsync(organizationId)).ToHashSet(StringComparer.Ordinal); // Ids are ObjectIds, so descending order is newest first no matter how the cache orders its list. var newestIds = (await GetIndexedIdsAsync(GetOrganizationIndexCacheKey(organizationId))) + .Concat(activeIds) .Distinct(StringComparer.Ordinal) - .OrderByDescending(id => id, StringComparer.Ordinal) - .Take(limit); + .OrderByDescending(id => id, StringComparer.Ordinal); var tasks = new List(); var expired = new List(); + int finishedCount = 0; foreach (string id in newestIds) { + if (finishedCount >= limit && !activeIds.Contains(id)) + continue; + var task = await GetAsync(id); if (task is null) expired.Add(id); - else if (String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal)) + else if (String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal) && (task.IsActive || finishedCount < limit)) + { tasks.Add(task); + if (!task.IsActive) + finishedCount++; + } } if (expired.Count > 0) diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs index f48042c6a8..823fc457b2 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs @@ -188,6 +188,31 @@ public async Task ResetData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() Assert.Empty(await GetService().GetByIdsAsync(events.Select(e => e.Id).ToArray(), o => o.Cache(false))); } + [Fact] + public async Task ResetData_CancelledBeforeDelayedDelivery_PreservesTheProjectEvents() + { + // Arrange + var (_, events) = await CreateDataAsync(d => d.Event().TestProject().Date(TimeProvider.GetUtcNow().AddHours(-1))); + var accepted = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() + .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "reset-data").StatusCodeShouldBeAccepted()); + string taskId = accepted!.Workers.Single(); + TimeProvider.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromSeconds(1)); + + // Act + var cancelled = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() + .AppendPaths("tasks", taskId, "cancel").StatusCodeShouldBeAccepted()); + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var stored = await GetTaskAsync(taskId); + await RefreshDataAsync(); + var remaining = await GetService().GetByIdsAsync(events.Select(e => e.Id).ToArray(), o => o.Cache(false)); + + // Assert + Assert.Equal(BackgroundTaskStatus.Cancelled, cancelled!.Status); + Assert.Equal(BackgroundTaskStatus.Cancelled, stored.Status); + Assert.Null(stored.StartedUtc); + Assert.Equal(events.Select(e => e.Id), remaining.Select(e => e.Id)); + } + [Fact] public async Task SampleData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() { diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs index 7f5f995cea..895dcd0255 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -116,7 +116,7 @@ public async Task Create_MoreThanFiveHundredOperations_StillSeesTheNewestActiveO Assert.NotNull(newest.Task); Assert.Null(blocked.Task); Assert.Equal(newest.Task.Id, blocked.Conflict?.Id); - Assert.Equal(BackgroundTaskService.ListLimit, list.Count); + Assert.Equal(501, list.Count); Assert.Equal(newest.Task.Id, list[0].Id); } diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs index f88d6c4fde..09201c2f8c 100644 --- a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs @@ -183,19 +183,33 @@ public async Task HandleItemAsync_TaskRecordIsGone_StillRunsTheWork() Assert.Empty(_harness.Publisher.Updates); } - [Fact] - public async Task HandleItemAsync_TaskWasCancelled_DoesNotRevive() + [Theory] + [InlineData(BackgroundTaskStatus.Cancelled)] + [InlineData(BackgroundTaskStatus.Completed)] + public async Task HandleItemAsync_TaskIsCancelledOrCompleted_DoesNotRunOrRevive(BackgroundTaskStatus status) { // Arrange var task = await AddTaskAsync(canCancel: true); - await _harness.Tasks.CancelAsync(task.Id); - var handler = CreateHandler(context => context.ReportProgressAsync(100, "Done")); + await _harness.Tasks.UpdateAsync(task.Id, current => + { + current.Finish(status, "Finished", _time.GetUtcNow().UtcDateTime); + return true; + }); + bool ran = false; + var handler = CreateHandler(_ => + { + ran = true; + return Task.CompletedTask; + }); + var context = CreateContext(task.Id); // Act - await handler.HandleItemAsync(CreateContext(task.Id)); + await handler.HandleItemAsync(context); // Assert - Assert.Equal(BackgroundTaskStatus.Cancelled, (await _harness.Tasks.GetAsync(task.Id))!.Status); + Assert.False(ran); + Assert.True(context.Result.IsSuccess); + Assert.Equal(status, (await _harness.Tasks.GetAsync(task.Id))!.Status); } [Fact] diff --git a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs index e287892be0..7c55c717e2 100644 --- a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs @@ -57,7 +57,7 @@ public async Task AddAsync_NewTask_StoresItQueuedAndIndexesIt() } [Fact] - public async Task GetByOrganizationAsync_ManyTasks_ReturnsTheNewestFirstUpToTheLimit() + public async Task GetByOrganizationAsync_FinishedTasks_ReturnsTheNewestFirstUpToTheLimit() { // Arrange var ids = new List(); @@ -65,6 +65,11 @@ public async Task GetByOrganizationAsync_ManyTasks_ReturnsTheNewestFirstUpToTheL { var task = NewTask(); await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); ids.Add(task.Id); _time.Advance(TimeSpan.FromSeconds(5)); } @@ -76,6 +81,41 @@ public async Task GetByOrganizationAsync_ManyTasks_ReturnsTheNewestFirstUpToTheL Assert.Equal([ids[3], ids[2]], newestTwo.Select(t => t.Id).ToArray()); } + [Fact] + public async Task GetByOrganizationAsync_ActiveTasks_DoNotConsumeTheFinishedHistoryLimit() + { + // Arrange + var oldest = NewTask(); + await Tasks.AddAsync(oldest); + var finishedIds = new List(); + for (int index = 0; index <= BackgroundTaskService.ListLimit; index++) + { + _time.Advance(TimeSpan.FromSeconds(1)); + var task = NewTask(); + await Tasks.AddAsync(task); + await Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); + finishedIds.Add(task.Id); + } + + _time.Advance(TimeSpan.FromSeconds(1)); + var newest = NewTask(); + await Tasks.AddAsync(newest); + + // Act + var listed = await Tasks.GetByOrganizationAsync(OrganizationId); + + // Assert + Assert.Equal(BackgroundTaskService.ListLimit + 2, listed.Count); + Assert.Equal(newest.Id, listed[0].Id); + Assert.Equal(oldest.Id, listed[^1].Id); + Assert.Equal(finishedIds.Skip(1).Reverse(), listed.Where(task => !task.IsActive).Select(task => task.Id)); + Assert.Equal(listed.Count, listed.Select(task => task.Id).Distinct().Count()); + } + [Fact] public async Task GetByOrganizationAsync_ExpiredAndForeignIds_AreSkippedAndExpiredIdsCleanedFromTheIndex() { From d14373fb7d33652592029d00239a672a40fe486e Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Nov 2026 19:54:22 -0600 Subject: [PATCH 17/42] fix: fail safely when background task startup cannot complete --- .../BackgroundTaskTrackingWorkItemHandler.cs | 5 ++- .../BackgroundTasks/BackgroundTaskService.cs | 9 ++-- .../BulkOperations/BulkOperationService.cs | 9 ++-- ...kgroundTaskTrackingWorkItemHandlerTests.cs | 37 +++++++++++++++- .../Models/BackgroundTaskTests.cs | 5 ++- .../BulkOperationServiceTests.cs | 43 +++++++++++++++++++ 6 files changed, 97 insertions(+), 11 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs index 2a2890900c..f6823d1e3d 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs @@ -122,12 +122,13 @@ await TryUpdateAsync(taskId, task => /// /// Marks the task running. A queue retry runs the work item again after it failed, which brings the task back to running - /// and back into the active index. A cancelled or completed task must not run again. + /// and back into the active index. A cancelled or completed task must not run again, so a failed start decision must + /// be retried by the queue rather than treated as a missing task record. /// private async Task StartAsync(string taskId) { bool wasFinished = false; - var task = await TryUpdateAsync(taskId, current => + var task = await _taskService.UpdateAsync(taskId, current => { if (current.Status is BackgroundTaskStatus.Cancelled or BackgroundTaskStatus.Completed) return false; diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs index 6275f80cb7..f3dda41b4c 100644 --- a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -158,13 +158,16 @@ public async Task CreateAndEnqueueAsync(BackgroundTask task, } /// - /// Queues the work item of a task that was added and publishes it. When queueing fails the task is failed, because - /// nothing would ever run it, and the error is rethrown. + /// Prepares and queues the work item of a task that was added and publishes it. When preparation or queueing fails + /// the task is failed, because nothing would ever run it, and the error is rethrown. /// - public async Task EnqueueAsync(BackgroundTask task, T workItem) where T : class + public async Task EnqueueAsync(BackgroundTask task, T workItem, Func? beforeEnqueue = null) where T : class { try { + if (beforeEnqueue is not null) + await beforeEnqueue(); + await _workItemQueue.EnqueueAsync(workItem); } catch (Exception ex) diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index 58745b696c..a0f0236994 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -104,15 +104,18 @@ public async Task CreateAsync(BackgroundTask task) } await _taskService.AddAsync(task); - if (pinnedSelection is not null) - await _cache.SetAsync(GetPinnedSelectionCacheKey(task.Id), pinnedSelection, BackgroundTaskService.TaskLifetime); - await _taskService.EnqueueAsync(task, new BulkOperationWorkItem { Description = task.Description, TaskId = task.Id, OrganizationId = task.OrganizationId, Type = task.Type + }, async () => + { + // A selection that cannot be saved must fail the task just like an enqueue failure, so an unqueued task + // cannot remain active and block the project's next operation. + if (pinnedSelection is not null && !await _cache.SetAsync(GetPinnedSelectionCacheKey(task.Id), pinnedSelection, BackgroundTaskService.TaskLifetime)) + throw new InvalidOperationException("The operation's selection could not be saved."); }); return BulkOperationCreateResult.Created(task); diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs index 09201c2f8c..805092d8a2 100644 --- a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs @@ -1,8 +1,10 @@ using Exceptionless.Core.Jobs.WorkItemHandlers; using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Tests.Utility; using Foundatio.Jobs; +using Foundatio.Lock; using Foundatio.Utility; using Foundatio.Xunit; using Microsoft.Extensions.Time.Testing; @@ -212,6 +214,37 @@ await _harness.Tasks.UpdateAsync(task.Id, current => Assert.Equal(status, (await _harness.Tasks.GetAsync(task.Id))!.Status); } + [Theory] + [InlineData(BackgroundTaskStatus.Cancelled)] + [InlineData(BackgroundTaskStatus.Completed)] + public async Task HandleItemAsync_TaskStatusCannotBeChecked_DoesNotRunTheWorker(BackgroundTaskStatus status) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.Zero }, lockAttempts: 1); + var task = await AddTaskAsync(harness: harness); + await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Finish(status, "Finished", _time.GetUtcNow().UtcDateTime); + return true; + }); + await using var held = await harness.LockProvider.AcquireAsync(BackgroundTaskService.GetUpdateLockResource(task.Id), TimeSpan.FromSeconds(30), TestCancellationToken); + bool ran = false; + var inner = new DelegateWorkItemHandler(_ => + { + ran = true; + return Task.CompletedTask; + }); + var handler = new BackgroundTaskTrackingWorkItemHandler(inner, harness.Tasks, _time); + + // Act + var exception = await Record.ExceptionAsync(() => handler.HandleItemAsync(CreateContext(task.Id))); + + // Assert + Assert.IsType(exception); + Assert.False(ran); + Assert.Equal(status, (await harness.Tasks.GetAsync(task.Id))!.Status); + } + [Fact] public void Handler_DelegatesTheJobSettingsToTheWrappedHandler() { @@ -227,7 +260,7 @@ public void Handler_DelegatesTheJobSettingsToTheWrappedHandler() Assert.Same(inner.Log, handler.Log); } - private async Task AddTaskAsync(bool canCancel = false) + private async Task AddTaskAsync(bool canCancel = false, BackgroundTaskTestHarness? harness = null) { var task = new BackgroundTask { @@ -238,7 +271,7 @@ private async Task AddTaskAsync(bool canCancel = false) CreatedByUserId = "537650f3b77efe23a47914f8", CanCancel = canCancel }; - await _harness.Tasks.AddAsync(task); + await (harness ?? _harness).Tasks.AddAsync(task); return task; } diff --git a/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs b/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs index dcec573cb3..85cc411d2b 100644 --- a/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs +++ b/tests/Exceptionless.Tests/Models/BackgroundTaskTests.cs @@ -1,12 +1,15 @@ using Exceptionless.Core.Models.BackgroundTasks; +using Foundatio.Xunit; using Xunit; namespace Exceptionless.Tests.Models; -public class BackgroundTaskTests +public class BackgroundTaskTests : TestWithLoggingBase { private static readonly DateTime UtcNow = new(2026, 11, 3, 4, 5, 6, DateTimeKind.Utc); + public BackgroundTaskTests(ITestOutputHelper output) : base(output) { } + [Fact] public void Start_CalledTwice_KeepsTheFirstStartTime() { diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs index a85e7d596d..a2fb844349 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs @@ -4,6 +4,7 @@ using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Core.Services.BulkOperations; using Exceptionless.Tests.Utility; +using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Queues; using Foundatio.Serializer; @@ -209,6 +210,48 @@ public async Task CreateAsync_QueueRejectsTheWorkItem_FailsTheTaskSoItDoesNotBlo Assert.Equal(BackgroundTaskStatus.Queued, retry.Task.Status); } + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task CreateAsync_SelectionCannotBeStored_FailsWithoutQueueingAndAllowsAnotherOperation(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var selectionCache = new InMemoryCacheClient(new InMemoryCacheClientOptions + { + TimeProvider = _time, + LoggerFactory = Log, + MaxEntrySize = 1, + SizeCalculator = _ => 2, + ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection + }); + var resolver = new Mock(); + resolver.SetupGet(r => r.Entity).Returns(BulkOperationEntity.Events); + resolver.Setup(r => r.PrepareAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new BulkSelectionPreparation(BulkOperationCheckResult.Ok, 1, "{}")); + var service = new BulkOperationService(_harness.Tasks, selectionCache, _harness.LockProvider, + [new StubHandler("test.destructive", isDestructive: true)], [resolver.Object], _time); + var task = NewTask("test.destructive", ProjectId); + task.Bulk!.Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter }; + + // Act + var thrown = await Record.ExceptionAsync(async () => await service.CreateAsync(task)); + var failed = await _harness.Tasks.GetAsync(task.Id); + var activeAfterFailure = await _harness.Tasks.GetActiveIdsAsync(OrganizationId); + long enqueuedAfterFailure = (await _harness.Queue.GetQueueStatsAsync()).Enqueued; + var retry = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.NotNull(failed); + Assert.Equal(BackgroundTaskStatus.Failed, failed.Status); + Assert.NotNull(failed.CompletedUtc); + Assert.Empty(activeAfterFailure); + Assert.Equal(0, enqueuedAfterFailure); + Assert.Contains(_harness.Publisher.Updates, update => update.Id == task.Id && update.Status == BackgroundTaskStatus.Failed); + Assert.NotNull(retry.Task); + Assert.Equal(BackgroundTaskStatus.Queued, retry.Task.Status); + } + [Fact] public async Task PublishLostAsync_RecordIsGone_PublishesAFinalFailedUpdateFromTheWorkItem() { From b5593a5f14e780338dda8e9e9bf7b0ac1a536abe Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Nov 2026 19:54:30 -0600 Subject: [PATCH 18/42] fix: preserve confirmed deletions and asynchronous task updates --- .../src/lib/features/events/api.svelte.ts | 28 ++--- .../src/lib/features/events/api.test.ts | 60 +++++++++- .../events-bulk-actions-dropdown-menu.svelte | 32 ++++-- ...-bulk-actions-dropdown-menu.svelte.test.ts | 57 ++++++++++ .../src/lib/features/tasks/api.svelte.ts | 54 ++++++--- .../src/lib/features/tasks/api.test.ts | 107 ++++++++++++++++-- .../tasks/components/task-panel.svelte | 9 +- .../components/task-panel.svelte.test.ts | 56 ++++++++- .../src/lib/features/tasks/models.test.ts | 15 ++- .../src/lib/features/tasks/models.ts | 6 +- 10 files changed, 370 insertions(+), 54 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts index f14e187745..b498e6c4cc 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts @@ -99,7 +99,7 @@ export async function invalidatePersistentEventQueries(queryClient: QueryClient, } export const queryKeys = { - deleteEvent: (ids: string[] | undefined) => [...queryKeys.type, 'delete', ...(ids ?? [])] as const, + deleteEvent: () => [...queryKeys.type, 'delete'] as const, eventsByReference: (referenceId: string | undefined, projectId?: string | undefined, params?: GetEventsByReferenceRequest['params']) => [...queryKeys.type, 'by-ref', referenceId, projectId, params] as const, id: (id: string | undefined) => [...queryKeys.type, id] as const, @@ -127,12 +127,6 @@ export const PERSISTENT_EVENT_DELETE_RECONCILE_RETRY_DELAY = 5000; export const ORGANIZATION_EVENT_QUERY_STALE_TIME_MS = 60 * 1000; export const ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS = 5 * 1000; -export interface DeleteEventsRequest { - route: { - ids: string[] | undefined; - }; -} - export interface EventNavigation { nextId: null | string; previousId: null | string; @@ -335,26 +329,26 @@ export function createEventWithNavigationQueryOptions(request: GetEventRequest, }; } -export function deleteEvent(request: DeleteEventsRequest) { +export function deleteEvent() { const queryClient = useQueryClient(); - return createMutation(() => ({ - enabled: () => !!accessToken.current && !!request.route.ids?.length, - mutationFn: async () => { + return createMutation(() => ({ + enabled: () => !!accessToken.current, + mutationFn: async (ids) => { const client = useFetchClient(); - const response = await client.deleteJSON(`events/${request.route.ids?.join(',')}`); + const response = await client.deleteJSON(`events/${ids.join(',')}`); return response.data!; }, - mutationKey: queryKeys.deleteEvent(request.route.ids), - onError: () => { - request.route.ids?.forEach((id) => + mutationKey: queryKeys.deleteEvent(), + onError: (_, ids) => { + ids.forEach((id) => queryClient.invalidateQueries({ queryKey: queryKeys.id(id) }) ); }, - onSuccess: () => { - request.route.ids?.forEach((id) => + onSuccess: (_, ids) => { + ids.forEach((id) => queryClient.invalidateQueries({ queryKey: queryKeys.id(id) }) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts index 26f88837ff..c692de219c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.test.ts @@ -1,6 +1,10 @@ -import { QueryClient } from '@tanstack/svelte-query'; +import type { ProblemDetails } from '@foundatiofx/fetchclient'; + +import { MutationObserver, type MutationObserverOptions, onlineManager, QueryClient } from '@tanstack/svelte-query'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { WorkInProgressResult } from '#shared/models/index.ts'; + import { ChangeType } from '#features/websockets/models.ts'; import type { PersistentEvent } from './models'; @@ -9,6 +13,7 @@ import { queryKeys as stackQueryKeys } from '../stacks/api.svelte'; import { createEventWithNavigationQueryOptions, createOrganizationEventNotificationRefresher, + deleteEvent, type GetEventRequest, invalidatePersistentEventQueries, ORGANIZATION_EVENT_NOTIFICATION_THROTTLE_MS, @@ -20,10 +25,19 @@ import { schedulePersistentEventDeleteReconciliation } from './api.svelte'; -const fetchClientMocks = vi.hoisted(() => ({ getJSON: vi.fn() })); +const fetchClientMocks = vi.hoisted(() => ({ deleteJSON: vi.fn(), getJSON: vi.fn() })); +const mutationMocks = vi.hoisted(() => ({ + createMutation: vi.fn<(options: () => MutationObserverOptions) => void>(), + useQueryClient: vi.fn() +})); vi.mock('@foundatiofx/fetchclient', () => ({ - useFetchClient: () => ({ getJSON: fetchClientMocks.getJSON }) + useFetchClient: () => fetchClientMocks +})); +vi.mock('@tanstack/svelte-query', async (importOriginal) => ({ + ...(await importOriginal()), + createMutation: mutationMocks.createMutation, + useQueryClient: mutationMocks.useQueryClient })); afterEach(() => { @@ -41,6 +55,46 @@ describe('retainPreviousOrganizationQueryData', () => { }); }); +describe('deleteEvent', () => { + it('keeps the confirmed event IDs when a paused deletion resumes after the selection changes', async () => { + // Arrange + vi.useFakeTimers(); + vi.stubGlobal('document', new EventTarget()); + onlineManager.setOnline(false); + const client = new QueryClient(); + const invalidate = vi.spyOn(client, 'invalidateQueries').mockResolvedValue(undefined); + mutationMocks.useQueryClient.mockReturnValue(client); + fetchClientMocks.deleteJSON.mockResolvedValue({ data: { workers: ['worker-id'] } }); + const selection = ['event-a']; + deleteEvent(); + const mutation = new MutationObserver(client, mutationMocks.createMutation.mock.calls.at(-1)![0]()); + + try { + // Act + const result = mutation.mutate([...selection]); + + // Assert + expect(mutation.getCurrentResult().isPaused).toBe(true); + expect(fetchClientMocks.deleteJSON).not.toHaveBeenCalled(); + + // Act + selection.splice(0, selection.length, 'event-b'); + onlineManager.setOnline(true); + await client.resumePausedMutations(); + await result; + + // Assert + expect(fetchClientMocks.deleteJSON).toHaveBeenCalledExactlyOnceWith('events/event-a'); + expect(invalidate).toHaveBeenCalledWith({ queryKey: queryKeys.id('event-a') }); + expect(invalidate).not.toHaveBeenCalledWith({ queryKey: queryKeys.id('event-b') }); + } finally { + onlineManager.setOnline(true); + client.clear(); + vi.unstubAllGlobals(); + } + }); +}); + describe('createOrganizationEventNotificationRefresher', () => { it('defers removal reconciliation without a leading refetch', async () => { // Arrange diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte index b439556ae7..73716c0984 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte @@ -32,28 +32,46 @@ const bulkActionsDescriptionId = `${componentId}-bulk-actions-description`; const ids = $derived(table.getSelectedRowModel().flatRows.map((row) => row.id)); const selectedCount = $derived(allMatching ? allMatching.count : ids.length); + const selectionIdentity = $derived( + JSON.stringify( + allMatching + ? ['filter', allMatching.organizationId, allMatching.filter ?? '', allMatching.time ?? null, allMatching.offset ?? null] + : ['ids', ...ids.toSorted()] + ) + ); let open = $state(false); let openRemoveEventDialog = $state(false); + let confirmedSelectionIdentity = $state(); - const removeEvents = deleteEvent({ - route: { - get ids() { - return ids; - } + $effect(() => { + if (openRemoveEventDialog && confirmedSelectionIdentity !== selectionIdentity) { + openRemoveEventDialog = false; } }); + const removeEvents = deleteEvent(); + const removeMatchingEvents = postBulkOperation(); + function confirmRemove(): void { + confirmedSelectionIdentity = selectionIdentity; + openRemoveEventDialog = true; + } + async function remove() { + // Navigation or a live update can change the selection while the confirmation is open. + if (confirmedSelectionIdentity !== selectionIdentity) { + return; + } + if (allMatching) { await removeAllMatching(allMatching); return; } const deletedCount = ids.length; - await removeEvents.mutateAsync(); + await removeEvents.mutateAsync([...ids]); if (deletedCount === 1) { toast.success('Successfully deleted event.'); @@ -110,7 +128,7 @@ - (openRemoveEventDialog = true)} class="text-destructive" title="Delete event">Delete + Delete diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts index b09c74fdd1..4b00edc33f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts @@ -76,6 +76,7 @@ describe('EventsBulkActionsDropdownMenu', () => { // Assert await waitFor(() => expect(mutateAsync).toHaveBeenCalledOnce()); + expect(mutateAsync).toHaveBeenCalledWith(['event-id']); expect(resetRowSelection).toHaveBeenCalledOnce(); expect(toast.success).toHaveBeenCalledWith('Successfully deleted event.'); expect(startBulkOperation).not.toHaveBeenCalled(); @@ -96,9 +97,38 @@ describe('EventsBulkActionsDropdownMenu', () => { // Assert await waitFor(() => expect(mutateAsync).toHaveBeenCalledOnce()); + expect(mutateAsync).toHaveBeenCalledWith(['event-id-1', 'event-id-2']); expect(toast.success).toHaveBeenCalledWith('Successfully deleted 2 events.'); }); + it('closes the confirmation when selected row IDs change even if the count stays the same', async () => { + // Arrange + const firstTable = { + getSelectedRowModel: () => ({ flatRows: [{ id: 'event-a' }] }), + resetRowSelection: vi.fn() + } as never; + const nextTable = { + getSelectedRowModel: () => ({ flatRows: [{ id: 'event-b' }] }), + resetRowSelection: vi.fn() + } as never; + const { rerender } = render(EventsBulkActionsDropdownMenu, { props: { table: firstTable } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Actions' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Delete' })); + + // Assert + expect(screen.getByRole('alertdialog')).toBeTruthy(); + + // Act + await rerender({ table: nextTable }); + + // Assert + await waitFor(() => expect(screen.queryByRole('alertdialog')).toBeNull()); + expect(mutateAsync).not.toHaveBeenCalled(); + expect(startBulkOperation).not.toHaveBeenCalled(); + }); + describe('when everything matching the filter is selected', () => { const allMatching = { count: 1234, filter: 'type:error', offset: '-300m', organizationId: 'organization-id', time: '[now-7d TO now]' }; @@ -143,6 +173,33 @@ describe('EventsBulkActionsDropdownMenu', () => { expect(toast.success).toHaveBeenCalledWith(expect.stringContaining('Deleting 1,234 events')); }); + it.each([ + ['filter', { ...allMatching, filter: 'type:log' }], + ['organization', { ...allMatching, organizationId: 'another-organization' }], + ['time range', { ...allMatching, time: '[now-30d TO now]' }], + ['time offset', { ...allMatching, offset: '-360m' }], + ['selection mode', undefined] + ])('closes the confirmation when the %s changes', async (_, target) => { + // Arrange + const table = { getSelectedRowModel: () => ({ flatRows: [{ id: 'event-id' }] }), resetRowSelection: vi.fn() } as never; + const { rerender } = render(EventsBulkActionsDropdownMenu, { props: { allMatching, table } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Actions' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Delete' })); + + // Assert + expect(screen.getByRole('alertdialog')).toBeTruthy(); + + // Act + await rerender({ allMatching: target, table }); + + // Assert + await waitFor(() => expect(screen.queryByRole('alertdialog')).toBeNull()); + expect(startBulkOperation).not.toHaveBeenCalled(); + expect(mutateAsync).not.toHaveBeenCalled(); + }); + it('keeps the selection when starting the operation fails', async () => { // Arrange startBulkOperation.mockRejectedValue({ title: 'Another destructive operation is already running for this project.' }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts index 05c5b64e79..597994149a 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts @@ -39,9 +39,7 @@ export function cancelTask() { return response.data!; }, mutationKey: queryKeys.cancel(undefined), - onSuccess: (task) => { - upsertTask(queryClient, task); - } + onSuccess: (task) => upsertTask(queryClient, task) })); } @@ -92,14 +90,12 @@ export function postBulkOperation() { return response.data!; }, mutationKey: queryKeys.post(), - onSuccess: (task) => { - upsertTask(queryClient, task); - } + onSuccess: (task) => upsertTask(queryClient, task) })); } /** Applies a pushed update to the cached list. Unknown tasks (started elsewhere) trigger a refetch instead. */ -export function updateTaskFromMessage(queryClient: QueryClient, update: BackgroundTaskUpdated) { +export async function updateTaskFromMessage(queryClient: QueryClient, update: BackgroundTaskUpdated) { const key = queryKeys.organization(update.organization_id); const current = queryClient.getQueryData(key); const next = current ? applyTaskUpdate(current, update) : undefined; @@ -110,12 +106,44 @@ export function updateTaskFromMessage(queryClient: QueryClient, update: Backgrou return; } - queryClient.setQueryData(key, next); + if (next === current) { + return; + } + + await queryClient.cancelQueries( + { + queryKey: key + }, + { + revert: false + } + ); + // Other messages can be applied while cancellation settles; merge with the latest cache, not the earlier snapshot. + const latest = queryClient.getQueryData(key); + const updated = latest ? applyTaskUpdate(latest, update) : undefined; + if (updated) { + queryClient.setQueryData(key, updated); + } } -function upsertTask(queryClient: QueryClient, task: ViewBackgroundTask) { - queryClient.setQueryData(queryKeys.organization(task.organization_id), (current) => [ - task, - ...(current ?? []).filter((existing) => existing.id !== task.id) - ]); +async function upsertTask(queryClient: QueryClient, task: ViewBackgroundTask) { + const key = queryKeys.organization(task.organization_id); + // An older list response must not remove newly queued work and stop polling. Keep any pushed updates received + // since that read started instead of reverting the cache to its pre-fetch snapshot. + await queryClient.cancelQueries( + { + queryKey: key + }, + { + revert: false + } + ); + queryClient.setQueryData(key, (current) => { + const existing = current?.find((item) => item.id === task.id); + if (existing && new Date(existing.updated_utc).getTime() > new Date(task.updated_utc).getTime()) { + return current; + } + + return [task, ...(current ?? []).filter((item) => item.id !== task.id)]; + }); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts index dc1d798c55..afea1b99e8 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts @@ -1,14 +1,25 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { QueryClient } from '@tanstack/svelte-query'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from './models'; +const mocks = vi.hoisted(() => ({ + createMutation: vi.fn<(options: () => { onSuccess: (task: ViewBackgroundTask) => Promise | void }) => void>(), + useQueryClient: vi.fn() +})); + vi.mock('#features/auth/index.svelte.ts', () => ({ accessToken: { current: 'access-token' } })); vi.mock('#features/events/api.svelte.ts', () => ({ queryKeys: { type: ['PersistentEvent'] } })); vi.mock('#features/stacks/api.svelte.ts', () => ({ queryKeys: { type: ['Stack'] } })); vi.mock('@foundatiofx/fetchclient', () => ({ useFetchClient: vi.fn() })); -vi.mock('@tanstack/svelte-query', () => ({ createMutation: vi.fn(), createQuery: vi.fn(), useQueryClient: vi.fn() })); +vi.mock('@tanstack/svelte-query', async (importOriginal) => ({ + ...(await importOriginal()), + createMutation: mocks.createMutation, + createQuery: vi.fn(), + useQueryClient: mocks.useQueryClient +})); -import { invalidateTaskAffectedQueries, queryKeys, updateTaskFromMessage } from './api.svelte'; +import { cancelTask, invalidateTaskAffectedQueries, postBulkOperation, queryKeys, updateTaskFromMessage } from './api.svelte'; function createTask(overrides: Partial = {}): ViewBackgroundTask { return { @@ -49,18 +60,18 @@ function createUpdate(id: string) { } describe('updateTaskFromMessage', () => { - const queryClient = { getQueryData: vi.fn(), invalidateQueries: vi.fn(), setQueryData: vi.fn() }; + const queryClient = { cancelQueries: vi.fn().mockResolvedValue(undefined), getQueryData: vi.fn(), invalidateQueries: vi.fn(), setQueryData: vi.fn() }; beforeEach(() => { vi.clearAllMocks(); }); - it('writes the progress of a known task into the cached list', () => { + it('writes the progress of a known task into the cached list', async () => { // Arrange queryClient.getQueryData.mockReturnValue([createTask()]); // Act - updateTaskFromMessage(queryClient as never, createUpdate('task-1')); + await updateTaskFromMessage(queryClient as never, createUpdate('task-1')); // Assert expect(queryClient.setQueryData).toHaveBeenCalledWith(queryKeys.organization('organization-id'), [ @@ -69,12 +80,12 @@ describe('updateTaskFromMessage', () => { expect(queryClient.invalidateQueries).not.toHaveBeenCalled(); }); - it('reloads the list when the task was started somewhere else', () => { + it('reloads the list when the task was started somewhere else', async () => { // Arrange queryClient.getQueryData.mockReturnValue([createTask()]); // Act - updateTaskFromMessage(queryClient as never, createUpdate('started-elsewhere')); + await updateTaskFromMessage(queryClient as never, createUpdate('started-elsewhere')); // Assert expect(queryClient.setQueryData).not.toHaveBeenCalled(); @@ -95,3 +106,83 @@ describe('invalidateTaskAffectedQueries', () => { expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: ['Stack'] }); }); }); + +describe('task cache lifecycle', () => { + let client: QueryClient; + const key = queryKeys.organization('organization-id'); + + beforeEach(() => { + vi.clearAllMocks(); + client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + mocks.useQueryClient.mockReturnValue(client); + }); + + afterEach(() => client.clear()); + + it('keeps a newly queued task when an earlier list request returns without it', async () => { + // Arrange + const pending = Promise.withResolvers(); + const read = client.fetchQuery({ queryFn: () => pending.promise, queryKey: key }).catch(() => undefined); + const task = createTask({ status: BackgroundTaskStatus.Queued }); + postBulkOperation(); + const options = mocks.createMutation.mock.calls[0]![0](); + + // Act + await options.onSuccess(task); + pending.resolve([]); + await read; + + // Assert + expect(client.getQueryData(key)).toEqual([task]); + }); + + it('preserves a newer pushed completion when cancellation returns an older running snapshot', async () => { + // Arrange + const running = createTask(); + client.setQueryData(key, [running]); + const pending = Promise.withResolvers(); + const read = client.fetchQuery({ queryFn: () => pending.promise, queryKey: key }).catch(() => undefined); + const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: '2026-01-01T10:01:00Z' }); + client.setQueryData(key, [completed]); + cancelTask(); + const options = mocks.createMutation.mock.calls[0]![0](); + + // Act + await options.onSuccess(running); + pending.resolve([running]); + await read; + + // Assert + expect(client.getQueryData(key)).toEqual([completed]); + }); + + it.each([ + [BackgroundTaskStatus.Failed, BackgroundTaskStatus.Running], + [BackgroundTaskStatus.Running, BackgroundTaskStatus.Completed] + ])('preserves a pushed %s -> %s transition when an earlier list request finishes', async (previousStatus, nextStatus) => { + // Arrange + const previous = createTask({ status: previousStatus }); + client.setQueryData(key, [previous]); + const pending = Promise.withResolvers(); + const read = client.fetchQuery({ queryFn: () => pending.promise, queryKey: key }).catch(() => undefined); + + // Act + await updateTaskFromMessage(client, { ...createUpdate(previous.id), status: nextStatus }); + pending.resolve([previous]); + await read; + + // Assert + expect(client.getQueryData(key)?.[0]).toMatchObject({ status: nextStatus, updated_utc: '2026-01-01T10:00:09Z' }); + }); + + it('retains concurrent pushed updates for different tasks', async () => { + // Arrange + client.setQueryData(key, [createTask(), createTask({ id: 'task-2' })]); + + // Act + await Promise.all([updateTaskFromMessage(client, createUpdate('task-1')), updateTaskFromMessage(client, createUpdate('task-2'))]); + + // Assert + expect(client.getQueryData(key)?.map((task) => task.processed)).toEqual([75, 75]); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte index c4c56fc9d8..8ddd90fb33 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte @@ -43,6 +43,8 @@ $effect(() => { for (const task of tasksQuery.data ?? []) { if (isTaskActive(task.status)) { + // A failed task can be retried with the same id; its new attempt and outcome must be visible. + dismissedIds.delete(task.id); activeSeenIds.add(task.id); continue; } @@ -62,7 +64,7 @@ return; } - updateTaskFromMessage(queryClient, update); + void updateTaskFromMessage(queryClient, update); }); async function onCancel(task: ViewBackgroundTask) { @@ -70,8 +72,11 @@ try { await cancelTaskMutation.mutateAsync(task.id); } catch { - cancellingIds.delete(task.id); toast.error('Unable to cancel the task.'); + } finally { + // The server may only acknowledge the request while a worker is running. Allow a later retry to clear + // an abandoned task if that worker never acknowledges cancellation. + cancellingIds.delete(task.id); } } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts index eb05a3e6a6..6738fa8a40 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts @@ -125,6 +125,33 @@ describe('TaskPanel', () => { await waitFor(() => expect(mocks.cancelMutateAsync).toHaveBeenCalledWith('task-1')); }); + it('allows cancellation to be retried after the request finishes while the worker is still running', async () => { + // Arrange + const request = Promise.withResolvers(); + mocks.cancelMutateAsync.mockReturnValueOnce(request.promise); + state.set('tasks', [createTask()]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); + + // Assert + expect(screen.getByRole('button', { name: /^Cancel / })).toBeDisabled(); + + // Act + request.resolve(createTask()); + await tick(); + + // Assert + await waitFor(() => expect(screen.getByRole('button', { name: /^Cancel / })).toBeEnabled()); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Cancel / })); + + // Assert + expect(mocks.cancelMutateAsync).toHaveBeenCalledTimes(2); + }); + it('removes a finished task when it is dismissed', async () => { // Arrange state.set('tasks', [createTask({ status: BackgroundTaskStatus.Completed })]); @@ -137,11 +164,38 @@ describe('TaskPanel', () => { expect(screen.queryByText('Delete 50 events matching filter')).toBeNull(); }); + it('shows a dismissed failed task again during and after an automatic retry', async () => { + // Arrange + state.set('tasks', [createTask({ status: BackgroundTaskStatus.Failed })]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + + // Assert + expect(screen.queryByText('Delete 50 events matching filter')).toBeNull(); + + // Act + state.set('tasks', [createTask()]); + await tick(); + + // Assert + expect(screen.getByRole('button', { name: /^Cancel / })).toBeEnabled(); + + // Act + state.set('tasks', [createTask({ processed: 50, status: BackgroundTaskStatus.Completed, succeeded: 50 })]); + await tick(); + + // Assert + expect(document.querySelector('[data-slot="task-progress"]')?.textContent).toContain('Completed'); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + }); + it('renders nothing when there is nothing to show', () => { // Arrange & Act render(TaskPanel); // Assert - expect(screen.queryByRole('region', { name: 'Bulk tasks' })).toBeNull(); + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts index f368d93201..b1aecc55ac 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts @@ -148,9 +148,9 @@ describe('selectVisibleTasks', () => { expect(result.map((task) => task.id)).toEqual(['recent', 'active']); }); - it('hides dismissed tasks', () => { + it('hides dismissed finished tasks', () => { // Arrange - const task = createTask({ id: 'dismissed' }); + const task = createTask({ id: 'dismissed', status: BackgroundTaskStatus.Failed }); // Act const result = selectVisibleTasks([task], new Set(['dismissed']), now); @@ -158,4 +158,15 @@ describe('selectVisibleTasks', () => { // Assert expect(result).toEqual([]); }); + + it.each([BackgroundTaskStatus.Queued, BackgroundTaskStatus.Running])('shows a dismissed failed task when it is retried as %s', (status) => { + // Arrange + const task = createTask({ id: 'retried', status }); + + // Act + const result = selectVisibleTasks([task], new Set(['retried']), now); + + // Assert + expect(result).toEqual([task]); + }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts index 4982ccb53d..8b923f88ac 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts @@ -76,11 +76,15 @@ export function isTaskActive(status: BackgroundTaskStatus): boolean { export function selectVisibleTasks(tasks: ViewBackgroundTask[], dismissedIds: ReadonlySet, now: number): ViewBackgroundTask[] { return tasks .filter((task) => { + if (isTaskActive(task.status)) { + return true; + } + if (dismissedIds.has(task.id)) { return false; } - return isTaskActive(task.status) || now - new Date(task.updated_utc).getTime() < FINISHED_TASK_VISIBLE_MS; + return now - new Date(task.updated_utc).getTime() < FINISHED_TASK_VISIBLE_MS; }) .sort((a, b) => new Date(b.created_utc).getTime() - new Date(a.created_utc).getTime()); } From a7235616601e5919d565b84375acc6925ede5c6e Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Nov 2026 20:01:01 -0600 Subject: [PATCH 19/42] fix: validate and pin bulk selection time ranges --- .../EventBulkSelectionResolver.cs | 9 +- .../BulkOperations/RelativeDateFreezer.cs | 59 ++++++--- .../Api/Handlers/BackgroundTaskHandler.cs | 118 +++++++++++++++++- .../Endpoints/BulkOperationEndpointTests.cs | 8 ++ .../Handlers/BackgroundTaskHandlerTests.cs | 92 ++++++++++++++ .../EventBulkSelectionResolverTests.cs | 31 +++++ .../RelativeDateFreezerTests.cs | 15 +++ 7 files changed, 304 insertions(+), 28 deletions(-) create mode 100644 tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs index e293d003db..c4de9674d5 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -67,10 +67,14 @@ public async Task PrepareAsync(BackgroundTask task, Ca if (appFilter is null) return new BulkSelectionPreparation(BulkOperationCheckResult.Ok); + // Counting and paging must use the same relative-date anchor, even when preparation runs after the selected day. + string? frozenFilter = await RelativeDateFreezer.FreezeAsync(task.Bulk!.Selection.Filter, + task.Bulk.Selection.UtcCreatedBefore ?? task.CreatedUtc, IsDateFieldAsync); + // The events list resolves stack level terms (for example status:open) and removes deleted stacks by looking up // matching stack ids. Do that once, here, and keep the result so every page uses the same stacks. var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection) - .FilterExpression(task.Bulk!.Selection.Filter) + .FilterExpression(frozenFilter) .EnforceEventStackFilter(); CountResult count; @@ -85,8 +89,7 @@ public async Task PrepareAsync(BackgroundTask task, Ca var pinned = new PinnedEventSelection { - // Relative dates in the filter are pinned to the creation time, like the outer time window. - EventFilter = await RelativeDateFreezer.FreezeAsync(query.GetFilterExpression(), task.Bulk!.Selection.UtcCreatedBefore ?? task.CreatedUtc, IsDateFieldAsync), + EventFilter = query.GetFilterExpression(), ExcludedStackIds = query.GetExcludedStacks().ToArray(), StackIds = query.GetStacks().ToArray() }; diff --git a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs index 18a2918a71..7235c6d1cb 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs @@ -50,46 +50,65 @@ public override async Task VisitAsync(TermRangeNode node, IQueryVisitorContext c if (!IsRelative(node.Min) && !IsRelative(node.Max)) return; - // A range inside a group such as date:(>now-2h AND <=now) takes its field from the group. + if (!await EnsureDateFieldAsync(node, [node.Min, node.Max])) + return; + + if (IsRelative(node.Min)) + { + node.Min = _anchor + node.Min![3..]; + node.IsMinQuotedTerm = true; + } + + if (IsRelative(node.Max)) + { + node.Max = _anchor + node.Max![3..]; + node.IsMaxQuotedTerm = true; + } + } + + public override async Task VisitAsync(TermNode node, IQueryVisitorContext context) + { + if (node.IsRegexTerm || node.Proximity is not null || !IsRelative(node.Term) || node.Term!.IndexOfAny(['*', '?']) >= 0 + || !await EnsureDateFieldAsync(node, [node.Term])) + return; + + // Elasticsearch treats a date term such as now/d as an inclusive date range, so it needs the same anchor. + node.Term = _anchor + node.Term![3..]; + node.IsQuotedTerm = true; + } + + private async ValueTask EnsureDateFieldAsync(IFieldQueryNode node, string?[] terms) + { + // An unfielded term or range inside a group takes its field from the group. string? field = node.Field ?? GetGroupField(node); if (String.IsNullOrEmpty(field)) - return; + return false; if (node.Field is null) { // The group already decided the stored field: a custom field left unchanged there is not a date. if (!IsStoredDateField(field) && (IsCustomField(field) || !await isDateFieldAsync(field))) - return; + return false; } else { - // Custom data fields are stored by type, and the events index picks a date field only when the bounds look like - // dates. Freezing the bounds would hide that, so name the stored date field directly. - string? storedField = EventFieldsQueryVisitor.GetCustomFieldName(field, [node.Min, node.Max]); + // Custom data fields are stored by type. Freezing their original terms would hide the date type, + // so name the stored date field directly. + string? storedField = EventFieldsQueryVisitor.GetCustomFieldName(field, terms); if (IsCustomField(field)) { if (!IsStoredDateField(storedField)) - return; + return false; node.Field = storedField; } else if (!IsStoredDateField(field) && !await isDateFieldAsync(field)) { - return; + return false; } } - if (IsRelative(node.Min)) - { - node.Min = _anchor + node.Min![3..]; - node.IsMinQuotedTerm = true; - } - - if (IsRelative(node.Max)) - { - node.Max = _anchor + node.Max![3..]; - node.IsMaxQuotedTerm = true; - } + return true; } private static bool IsCustomField(string field) @@ -115,7 +134,7 @@ private static bool IsStoredDateField(string? field) private static bool IsRelative(string? value) { - return value is not null && value.StartsWith("now", StringComparison.OrdinalIgnoreCase) && (value.Length == 3 || value[3] is '+' or '-' or '/'); + return value is not null && value.StartsWith("now", StringComparison.Ordinal) && (value.Length == 3 || value[3] is '+' or '-' or '/'); } } } diff --git a/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs index 1e59ecec4a..aae45dc272 100644 --- a/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs @@ -1,3 +1,4 @@ +using System.Text.RegularExpressions; using Exceptionless.Core; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; @@ -7,6 +8,9 @@ using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.DateTimeExtensions; +using Exceptionless.DateTimeExtensions.FormatParsers; +using Exceptionless.DateTimeExtensions.FormatParsers.PartParsers; using Exceptionless.Web.Api.Infrastructure; using Exceptionless.Web.Api.Messages; using Exceptionless.Web.Api.Results; @@ -30,7 +34,6 @@ public class BackgroundTaskHandler( ILogger logger) { private const int MaxIds = 1000; - private static readonly ICollection _allowedDateFields = new List { EventIndex.Alias.Date }; private HttpContext HttpContext => httpContextAccessor.HttpContext ?? throw new InvalidOperationException("HttpContext is unavailable."); @@ -192,10 +195,17 @@ public async Task>> Handle(CancelBac if (model.Ids is { Count: > 0 }) return (null, Result.Invalid(ValidationError.Create("selection", "A filter selection cannot include ids."))); - // Resolve the relative time range once, now. The end can never be later than this moment, so events that - // arrive after the operation is created are not part of it. - var timeInfo = TimeRangeParser.GetTimeInfo(model.Time, model.Offset, timeProvider, _allowedDateFields, EventIndex.Alias.Date, - organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); + if (!TryParseSelectionOffset(model.Offset, out var utcOffset)) + return (null, Result.Invalid(ValidationError.Create("selection.offset", "The UTC offset must be a valid whole-minute offset between -14h and 14h."))); + + // Validate the requested range before retention and open-end defaults can turn a parse failure into all history. + // Reuse the parsed range so validation and execution resolve relative dates at exactly the same instant. + if (!TryParseSelectionTime(model.Time, new DateTimeOffset(utcNow).ToOffset(utcOffset), out var range)) + return (null, Result.Invalid(ValidationError.Create("selection.time", "The time range is not valid."))); + + var timeInfo = new TimeInfo { Field = EventIndex.Alias.Date, Offset = utcOffset, Range = range! }; + timeInfo.ApplyMinimumUtcStartDate(organization.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); + timeInfo.AdjustEndTimeIfMaxValue(timeProvider); var utcStart = timeInfo.Range.UtcStart; var utcEnd = timeInfo.Range.UtcEnd > utcNow ? utcNow : timeInfo.Range.UtcEnd; if (utcStart > utcEnd) @@ -214,6 +224,104 @@ public async Task>> Handle(CancelBac return (selection, null); } + internal static bool TryParseSelectionTime(string? time, DateTimeOffset relativeBaseTime, out DateTimeRange? range) + { + if (String.IsNullOrWhiteSpace(time)) + { + range = DateTimeRange.Empty; + return true; + } + + string expression = time.Trim(); + if (expression.StartsWith($"{EventIndex.Alias.Date}|", StringComparison.Ordinal)) + expression = expression[(EventIndex.Alias.Date.Length + 1)..]; + + // DateTimeRange.Parse uses the same unbounded sentinel for invalid input and a valid [* TO *] range. + // Its two-part parser also supplies missing bounds, so require both to have been parsed explicitly. + range = null; + try + { + foreach (var parser in DateTimeRange.FormatParsers) + { + if (parser is TwoPartFormatParser twoPartParser) + { + var bounds = new List(2); + var partParsers = twoPartParser.Parsers.Select(partParser => partParser is WildcardPartParser + ? new SelectionTimeWildcardPartParser(bounds) + : new SelectionTimePartParser(partParser, bounds)); + range = new TwoPartFormatParser(partParsers).Parse(expression, relativeBaseTime); + if (range is not null && (bounds.Count != 2 || bounds[0] > bounds[1])) + { + range = null; + return false; + } + } + else + { + range = parser.Parse(expression, relativeBaseTime); + } + + if (range is not null) + return true; + } + } + catch (Exception ex) when (ex is ArgumentException or FormatException or OverflowException) + { + range = null; + } + + return false; + } + + internal static bool TryParseSelectionOffset(string? offset, out TimeSpan utcOffset) + { + utcOffset = TimeSpan.Zero; + if (String.IsNullOrWhiteSpace(offset)) + return true; + + try + { + if (!TimeUnit.TryParse(offset, out var parsedOffset) || !parsedOffset.HasValue + || parsedOffset.Value < TimeSpan.FromHours(-14) || parsedOffset.Value > TimeSpan.FromHours(14) + || parsedOffset.Value.Ticks % TimeSpan.TicksPerMinute != 0) + return false; + + utcOffset = parsedOffset.Value; + return true; + } + catch (Exception ex) when (ex is ArgumentException or FormatException or OverflowException) + { + return false; + } + } + + private sealed class SelectionTimePartParser(IPartParser inner, List bounds) : IPartParser + { + public Regex Regex => inner.Regex; + + public DateTimeOffset? Parse(Match match, DateTimeOffset relativeBaseTime, bool isUpperLimit) + { + var value = inner.Parse(match, relativeBaseTime, isUpperLimit); + if (value.HasValue) + bounds.Add(value.Value); + + return value; + } + } + + // The two-part parser checks this type to keep wildcard bounds open regardless of bracket inclusivity. + private sealed class SelectionTimeWildcardPartParser(List bounds) : WildcardPartParser, IPartParser + { + DateTimeOffset? IPartParser.Parse(Match match, DateTimeOffset relativeBaseTime, bool isUpperLimit) + { + var value = base.Parse(match, relativeBaseTime, isUpperLimit); + if (value.HasValue) + bounds.Add(value.Value); + + return value; + } + } + private static Result ToResult(BulkOperationCheckResult check) { return check.Status switch diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index 1ac49a18b6..ec6e2a13af 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -430,6 +430,14 @@ public async Task Create_InvalidRequests_AreRejectedWithoutStartingAnything() FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Parameters = new Dictionary { ["x"] = "y" } }, FilterRequest(SampleDataService.TEST_ORG_ID, null, "type:(log"), FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "[now+1d TO now+2d]"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "not a date"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "date|not a date"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "date|"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "-"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "[ - ]"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "", "[now/d TO now-1d/d]"), + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Time = "[now/d TO now/d]", Offset = "not an offset" } }, + FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Time = "[now/d TO now/d]", Offset = "2147483647d" } }, FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Filter, Ids = ["537650f3b77efe23a47914f3"] } }, FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = [] } }, FilterRequest(SampleDataService.TEST_ORG_ID, null, "") with { Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = ["not-an-id"] } }, diff --git a/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs b/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs new file mode 100644 index 0000000000..c995a13179 --- /dev/null +++ b/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs @@ -0,0 +1,92 @@ +using System.Globalization; +using Exceptionless.Web.Api.Handlers; +using Foundatio.Xunit; +using Xunit; + +namespace Exceptionless.Tests.Api.Handlers; + +public class BackgroundTaskHandlerTests : TestWithLoggingBase +{ + private static readonly DateTimeOffset UtcNow = new(2026, 11, 3, 12, 0, 0, TimeSpan.Zero); + + public BackgroundTaskHandlerTests(ITestOutputHelper output) : base(output) { } + + [Theory] + [InlineData("not a date")] + [InlineData("24h")] + [InlineData("date|not a date")] + [InlineData("date|")] + [InlineData("created_utc|[now-1d TO now]")] + [InlineData("-")] + [InlineData("[ - ]")] + [InlineData("- now")] + [InlineData("now -")] + [InlineData("[now-1d TO now")] + [InlineData("[now/d TO now-1d/d]")] + [InlineData("last 0 days")] + [InlineData("last 999999999999999999 days")] + public void TryParseSelectionTime_InvalidRange_IsRejectedBeforeDefaultsCanBroadenIt(string time) + { + // Arrange + var relativeBaseTime = UtcNow; + + // Act + bool parsed = BackgroundTaskHandler.TryParseSelectionTime(time, relativeBaseTime, out var range); + + // Assert + Assert.False(parsed); + Assert.Null(range); + } + + [Theory] + [InlineData(null, 0, "0001-01-01T00:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData("", 0, "0001-01-01T00:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData("[now-1d TO now]", 0, "2026-11-02T12:00:00Z", "2026-11-03T12:00:00Z")] + [InlineData("date|[now/d TO now/d]", -5, "2026-11-03T05:00:00Z", "2026-11-04T04:59:59.999Z")] + [InlineData("[now-1d TO *]", 0, "2026-11-02T12:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData("[* TO now]", 0, "0001-01-01T00:00:00Z", "2026-11-03T12:00:00Z")] + [InlineData("[* TO *]", 0, "0001-01-01T00:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData("{* TO *}", 0, "0001-01-01T00:00:00Z", "9999-12-31T23:59:59.9999999Z")] + [InlineData(" + { + d.Event().TestProject().Message("confirmed first").Date(cutoff.AddHours(-1)) + .Mutate(e => e.Data = new DataDictionary { ["scheduled"] = cutoff }) + .MutateStack(stack => stack.CreatedUtc = cutoff.AddHours(-2)); + d.Event().TestProject().Message("confirmed second").Date(cutoff.AddHours(-2)) + .Mutate(e => e.Data = new DataDictionary { ["scheduled"] = cutoff.Date.AddHours(12) }) + .MutateStack(stack => stack.CreatedUtc = cutoff.AddHours(-2)); + d.Event().TestProject().Message("different day").Date(cutoff.AddHours(-1)) + .Mutate(e => e.Data = new DataDictionary { ["scheduled"] = now }) + .MutateStack(stack => stack.CreatedUtc = cutoff.AddHours(-2)); + }); + // Use yesterday's selection anchor so the regression does not need to wait for Elasticsearch's clock to advance. + var task = NewFilterTask("data.scheduled:now/d", createdBefore: cutoff); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 1); + + // Assert + Assert.Equal(2, preparation.Total); + Assert.Equal(events.Where(e => e.Message!.StartsWith("confirmed", StringComparison.Ordinal)).OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(), + pages.SelectMany(page => page).ToArray()); + } + [Theory] [InlineData(SampleDataService.TEST_ORG_ID, "type:log", BulkOperationCheckStatus.Ok)] [InlineData(SampleDataService.TEST_ORG_ID, "type:(log", BulkOperationCheckStatus.Invalid)] diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs index e3bc0c0c93..c86d485edb 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs @@ -19,6 +19,11 @@ public RelativeDateFreezerTests(ITestOutputHelper output) : base(output) { } [InlineData("date:(>now-2h AND <=now)", "date:(>\"2026-11-03T04:05:06.789Z||-2h\" AND <=\"2026-11-03T04:05:06.789Z||\")")] [InlineData("data.date:(>=now/d AND =\"2026-11-03T04:05:06.789Z||/d\" AND <\"2026-11-03T04:05:06.789Z||/d+1d\")")] [InlineData("data.date:[now/d-4d TO now/d+1d}", "idx.date-d:[\"2026-11-03T04:05:06.789Z||/d-4d\" TO \"2026-11-03T04:05:06.789Z||/d+1d\"}")] + [InlineData("date:now/d", "date:\"2026-11-03T04:05:06.789Z||/d\"")] + [InlineData("NOT date:now/d^2", "NOT date:\"2026-11-03T04:05:06.789Z||/d\"^2")] + [InlineData("date:(now/d OR now/d-1d)", "date:(\"2026-11-03T04:05:06.789Z||/d\" OR \"2026-11-03T04:05:06.789Z||/d-1d\")")] + [InlineData("data.date:now/d", "idx.date-d:\"2026-11-03T04:05:06.789Z||/d\"")] + [InlineData("data.date:(now/d OR now/d-1d)", "idx.date-d:(\"2026-11-03T04:05:06.789Z||/d\" OR \"2026-11-03T04:05:06.789Z||/d-1d\")")] public async Task FreezeAsync_RelativeDatesOnDateFields_AreAnchoredToTheGivenTime(string filter, string expected) { // Arrange @@ -37,6 +42,16 @@ public async Task FreezeAsync_RelativeDatesOnDateFields_AreAnchoredToTheGivenTim [InlineData("")] [InlineData("type:error")] [InlineData("message:now")] + [InlineData("environment:now/d")] + [InlineData("environment:(now/d OR now/d-1d)")] + [InlineData("data.name:(hello OR now/d)")] + [InlineData("ref.parent:now/d")] + [InlineData("date:NOW/d")] + [InlineData("date:[NOW-1d TO NOW]")] + [InlineData("date:now/d*")] + [InlineData("date:now/d?")] + [InlineData("date:now/d~1")] + [InlineData("date:/now.*/")] [InlineData("date:[2026-01-01 TO 2026-02-01]")] [InlineData("tag:[nowhere TO zebra]")] [InlineData("environment:[now TO z]")] From 3d3cc99aa47de72a6329a06b9f0af7300a6861eb Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Nov 2026 20:09:11 -0600 Subject: [PATCH 20/42] test: separate background task endpoint journey phases --- .../Endpoints/BackgroundTaskEndpointTests.cs | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs index 823fc457b2..f51edceb9c 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BackgroundTaskEndpointTests.cs @@ -169,10 +169,12 @@ public async Task ResetData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() // Act var accepted = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "reset-data").StatusCodeShouldBeAccepted()); + + // Assert string taskId = Assert.Single(accepted!.Workers); + + // Act var queued = await GetTaskAsync(taskId); - await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); - var completed = await GetTaskAsync(taskId); // Assert Assert.Equal(BackgroundTaskTypes.ResetProjectData, queued.Type); @@ -180,12 +182,19 @@ public async Task ResetData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() Assert.False(queued.CanCancel); Assert.Equal(SampleDataService.TEST_PROJECT_ID, queued.ProjectId); Assert.StartsWith("Reset data for ", queued.Description); + + // Act + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var completed = await GetTaskAsync(taskId); + await RefreshDataAsync(); + var remaining = await GetService().GetByIdsAsync(events.Select(e => e.Id).ToArray(), o => o.Cache(false)); + + // Assert Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); Assert.Equal(100, completed.Progress); Assert.NotNull(completed.StartedUtc); Assert.NotNull(completed.CompletedUtc); - await RefreshDataAsync(); - Assert.Empty(await GetService().GetByIdsAsync(events.Select(e => e.Id).ToArray(), o => o.Cache(false))); + Assert.Empty(remaining); } [Fact] @@ -216,19 +225,30 @@ public async Task ResetData_CancelledBeforeDelayedDelivery_PreservesTheProjectEv [Fact] public async Task SampleData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() { + // Arrange + string projectId = SampleDataService.TEST_PROJECT_ID; + // Act var accepted = await SendRequestAsAsync(r => r.Post().AsTestOrganizationUser() - .AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "sample-data").StatusCodeShouldBeAccepted()); + .AppendPaths("projects", projectId, "sample-data").StatusCodeShouldBeAccepted()); + + // Assert string taskId = Assert.Single(accepted!.Workers); + + // Act var queued = await GetTaskAsync(taskId); - await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); - var completed = await GetTaskAsync(taskId); // Assert Assert.Equal(BackgroundTaskTypes.GenerateProjectSampleData, queued.Type); Assert.Equal(BackgroundTaskStatus.Queued, queued.Status); Assert.False(queued.CanCancel); Assert.StartsWith("Generate sample data for ", queued.Description); + + // Act + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var completed = await GetTaskAsync(taskId); + + // Assert Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); Assert.Equal(100, completed.Progress); } @@ -236,9 +256,12 @@ public async Task SampleData_ReturnsATaskIdThatTracksTheJobUntilItCompletes() [Fact] public async Task ResetAndSampleData_ProjectOfAnotherOrganization_IsNotFoundAndTracksNothing() { + // Arrange + string projectId = SampleDataService.TEST_PROJECT_ID; + // Act - await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "reset-data").StatusCodeShouldBeNotFound()); - await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("projects", SampleDataService.TEST_PROJECT_ID, "sample-data").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("projects", projectId, "reset-data").StatusCodeShouldBeNotFound()); + await SendRequestAsync(r => r.Post().AsFreeOrganizationUser().AppendPaths("projects", projectId, "sample-data").StatusCodeShouldBeNotFound()); // Assert Assert.Empty(await _taskService.GetByOrganizationAsync(SampleDataService.TEST_ORG_ID)); From 94b990a7542a09c089b101a9fc9844653b2378f7 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Thu, 12 Nov 2026 20:11:50 -0600 Subject: [PATCH 21/42] fix: retain newer task state and event selections --- .../events-bulk-actions-dropdown-menu.svelte | 13 ++++-- ...-bulk-actions-dropdown-menu.svelte.test.ts | 44 +++++++++++++++++++ .../src/lib/features/tasks/api.svelte.ts | 4 +- .../src/lib/features/tasks/api.test.ts | 9 ++-- .../src/lib/features/tasks/models.test.ts | 21 +++++++-- .../src/lib/features/tasks/models.ts | 15 ++++++- 6 files changed, 93 insertions(+), 13 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte index 73716c0984..046a9d8728 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte @@ -71,6 +71,7 @@ } const deletedCount = ids.length; + const deletedSelectionIdentity = selectionIdentity; await removeEvents.mutateAsync([...ids]); if (deletedCount === 1) { @@ -79,10 +80,14 @@ toast.success(`Successfully deleted ${Intl.NumberFormat().format(deletedCount)} events.`); } - table.resetRowSelection(); + // A slow or paused request must not clear a different selection made while it was in flight. + if (deletedSelectionIdentity === selectionIdentity) { + table.resetRowSelection(); + } } async function removeAllMatching(target: AllMatchingTarget) { + const startedSelectionIdentity = selectionIdentity; try { await removeMatchingEvents.mutateAsync({ organization_id: target.organizationId, @@ -101,8 +106,10 @@ } toast.success(`Deleting ${Intl.NumberFormat().format(target.count)} events. Progress is shown at the bottom of the page.`); - table.resetRowSelection(); - onAllMatchingStarted?.(); + if (startedSelectionIdentity === selectionIdentity) { + table.resetRowSelection(); + onAllMatchingStarted?.(); + } } function setOpen(isOpen: boolean): void { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts index 4b00edc33f..1ed1ade694 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-bulk-actions-dropdown-menu.svelte.test.ts @@ -101,6 +101,28 @@ describe('EventsBulkActionsDropdownMenu', () => { expect(toast.success).toHaveBeenCalledWith('Successfully deleted 2 events.'); }); + it('preserves a newer row selection when an earlier deletion finishes', async () => { + // Arrange + const pending = Promise.withResolvers(); + mutateAsync.mockReturnValueOnce(pending.promise); + const firstTable = { getSelectedRowModel: () => ({ flatRows: [{ id: 'event-a' }] }), resetRowSelection: vi.fn() } as never; + const resetRowSelection = vi.fn(); + const nextTable = { getSelectedRowModel: () => ({ flatRows: [{ id: 'event-b' }] }), resetRowSelection } as never; + const { rerender } = render(EventsBulkActionsDropdownMenu, { props: { table: firstTable } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Actions' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Delete' })); + await fireEvent.click(screen.getByRole('button', { name: 'Delete Event' })); + await rerender({ table: nextTable }); + pending.resolve(); + + // Assert + await waitFor(() => expect(toast.success).toHaveBeenCalledWith('Successfully deleted event.')); + expect(mutateAsync).toHaveBeenCalledWith(['event-a']); + expect(resetRowSelection).not.toHaveBeenCalled(); + }); + it('closes the confirmation when selected row IDs change even if the count stays the same', async () => { // Arrange const firstTable = { @@ -173,6 +195,28 @@ describe('EventsBulkActionsDropdownMenu', () => { expect(toast.success).toHaveBeenCalledWith(expect.stringContaining('Deleting 1,234 events')); }); + it('preserves a newer filter selection when an earlier bulk request finishes', async () => { + // Arrange + const pending = Promise.withResolvers(); + startBulkOperation.mockReturnValueOnce(pending.promise); + const resetRowSelection = vi.fn(); + const onAllMatchingStarted = vi.fn(); + const table = { getSelectedRowModel: () => ({ flatRows: [] }), resetRowSelection } as never; + const { rerender } = render(EventsBulkActionsDropdownMenu, { props: { allMatching, onAllMatchingStarted, table } }); + + // Act + await fireEvent.click(screen.getByRole('button', { name: 'Actions' })); + await fireEvent.click(screen.getByRole('menuitem', { name: 'Delete' })); + await fireEvent.click(screen.getByRole('button', { name: 'Delete 1,234 Events' })); + await rerender({ allMatching: { ...allMatching, filter: 'type:log' }, onAllMatchingStarted, table }); + pending.resolve(); + + // Assert + await waitFor(() => expect(toast.success).toHaveBeenCalledWith(expect.stringContaining('Deleting 1,234 events'))); + expect(resetRowSelection).not.toHaveBeenCalled(); + expect(onAllMatchingStarted).not.toHaveBeenCalled(); + }); + it.each([ ['filter', { ...allMatching, filter: 'type:log' }], ['organization', { ...allMatching, organizationId: 'another-organization' }], diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts index 597994149a..805bc2844a 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts @@ -9,7 +9,7 @@ import { accessToken } from '#features/auth/index.svelte.ts'; import { queryKeys as eventQueryKeys } from '#features/events/api.svelte.ts'; import { queryKeys as stackQueryKeys } from '#features/stacks/api.svelte.ts'; -import { applyTaskUpdate, isTaskActive, type NewBulkOperation, type ViewBackgroundTask } from './models'; +import { applyTaskUpdate, compareTaskUpdateTimes, isTaskActive, type NewBulkOperation, type ViewBackgroundTask } from './models'; /** While a task is active the list is polled as well, so progress still moves when push messages are unavailable. */ export const ACTIVE_TASK_POLL_INTERVAL_MS = 3000; @@ -140,7 +140,7 @@ async function upsertTask(queryClient: QueryClient, task: ViewBackgroundTask) { ); queryClient.setQueryData(key, (current) => { const existing = current?.find((item) => item.id === task.id); - if (existing && new Date(existing.updated_utc).getTime() > new Date(task.updated_utc).getTime()) { + if (existing && compareTaskUpdateTimes(existing.updated_utc, task.updated_utc) > 0) { return current; } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts index afea1b99e8..8ae658780b 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts @@ -136,13 +136,16 @@ describe('task cache lifecycle', () => { expect(client.getQueryData(key)).toEqual([task]); }); - it('preserves a newer pushed completion when cancellation returns an older running snapshot', async () => { + it.each([ + ['2026-01-01T10:00:00Z', '2026-01-01T10:01:00Z'], + ['2026-01-01T10:00:00.1234566Z', '2026-01-01T10:00:00.1234567Z'] + ])('ignores a cancellation snapshot at %s after a pushed completion at %s', async (older, newer) => { // Arrange - const running = createTask(); + const running = createTask({ updated_utc: older }); client.setQueryData(key, [running]); const pending = Promise.withResolvers(); const read = client.fetchQuery({ queryFn: () => pending.promise, queryKey: key }).catch(() => undefined); - const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: '2026-01-01T10:01:00Z' }); + const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: newer }); client.setQueryData(key, [completed]); cancelTask(); const options = mocks.createMutation.mock.calls[0]![0](); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts index b1aecc55ac..c2bdd9ed00 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts @@ -64,17 +64,32 @@ describe('applyTaskUpdate', () => { expect(result?.[1]).toBe(tasks[1]); }); - it('ignores a message that is older than what is already known', () => { + it.each([ + ['2026-01-01T10:00:05Z', '2026-01-01T10:00:10Z'], + ['2026-01-01T10:00:00.1234566Z', '2026-01-01T10:00:00.1234567Z'], + ['2026-01-01T10:00:00Z', '2026-01-01T10:00:00.0000001Z'] + ])('ignores a message at %s when a newer completion at %s is already known', (older, newer) => { // Arrange - const tasks = [createTask({ processed: 60, updated_utc: '2026-01-01T10:00:10Z' })]; + const tasks = [createTask({ processed: 100, status: BackgroundTaskStatus.Completed, updated_utc: newer })]; // Act - const result = applyTaskUpdate(tasks, createUpdate({ processed: 20 })); + const result = applyTaskUpdate(tasks, createUpdate({ processed: 20, updated_utc: older })); // Assert expect(result).toBe(tasks); }); + it('accepts a newer retry within the same millisecond', () => { + // Arrange + const tasks = [createTask({ status: BackgroundTaskStatus.Failed, updated_utc: '2026-01-01T10:00:00.1234566Z' })]; + + // Act + const result = applyTaskUpdate(tasks, createUpdate({ updated_utc: '2026-01-01T10:00:00.1234567Z' })); + + // Assert + expect(result?.[0]).toMatchObject({ status: BackgroundTaskStatus.Running, updated_utc: '2026-01-01T10:00:00.1234567Z' }); + }); + it('reports an unknown task so the list can be reloaded', () => { // Arrange const tasks = [createTask()]; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts index 8b923f88ac..6d0962104b 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts @@ -16,8 +16,8 @@ export function applyTaskUpdate(tasks: ViewBackgroundTask[], update: BackgroundT } const current = tasks[index]!; - if (new Date(update.updated_utc).getTime() < new Date(current.updated_utc).getTime()) { - // Messages can arrive out of order; never move an task backwards. + if (compareTaskUpdateTimes(update.updated_utc, current.updated_utc) < 0) { + // Messages can arrive out of order; never move a task backwards. return tasks; } @@ -39,6 +39,17 @@ export function applyTaskUpdate(tasks: ViewBackgroundTask[], update: BackgroundT return next; } +/** Date.parse truncates the four submillisecond digits of .NET timestamps; retain them when ordering updates. */ +export function compareTaskUpdateTimes(left: string, right: string): number { + const milliseconds = Date.parse(left) - Date.parse(right); + if (milliseconds !== 0) { + return milliseconds; + } + + const remainingTicks = (value: string) => Number((value.match(/\.(\d+)/)?.[1] ?? '').padEnd(7, '0').slice(3, 7)); + return remainingTicks(left) - remainingTicks(right); +} + /** Tasks that report item counts show those; tasks that only report a percent (total of zero) show the percent. */ export function getProgressPercent(task: Pick): number { if (task.status === BackgroundTaskStatus.Completed) { From affc7916c42f9838956c2a0fb52259d4be2607dc Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 00:00:20 -0600 Subject: [PATCH 22/42] fix: reject unsuccessful background task cache writes Stop creation, state publication, and cancellation acknowledgment when the cache refuses a write. Cover throwing and nonthrowing cache rejection with real in-memory infrastructure. --- .../BackgroundTasks/BackgroundTaskService.cs | 9 +- .../BackgroundTaskServiceTests.cs | 95 +++++++++++++++++++ .../Utility/BackgroundTaskTestHarness.cs | 6 +- 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs index f3dda41b4c..588dc5a6e3 100644 --- a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -242,7 +242,8 @@ public async Task CancelAsync(string id) if (!task.CanCancel && !isAbandoned) return BackgroundTaskCancelResult.NotCancellable(task); - await _cache.SetAsync(GetCancelCacheKey(id), true, TaskLifetime); + if (!await _cache.SetAsync(GetCancelCacheKey(id), true, TaskLifetime)) + throw new InvalidOperationException("The cancellation request could not be saved."); var updated = await UpdateAsync(id, current => { @@ -265,7 +266,11 @@ public async Task IsCancelRequestedAsync(string id) return (await _cache.GetAsync(GetCancelCacheKey(id))).HasValue; } - public Task SaveAsync(BackgroundTask task) => _cache.SetAsync(GetTaskCacheKey(task.Id), task, TaskLifetime); + public async Task SaveAsync(BackgroundTask task) + { + if (!await _cache.SetAsync(GetTaskCacheKey(task.Id), task, TaskLifetime)) + throw new InvalidOperationException("The background task could not be saved."); + } public Task PublishAsync(BackgroundTask task) { diff --git a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs index 7c55c717e2..61fe1a441d 100644 --- a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs @@ -2,6 +2,7 @@ using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Tests.Utility; +using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Queues; @@ -56,6 +57,35 @@ public async Task AddAsync_NewTask_StoresItQueuedAndIndexesIt() Assert.Empty(_harness.Publisher.Updates); } + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task CreateAndEnqueueAsync_TaskCannotBeStored_DoesNotQueueOrPublish(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, configureCache: options => + { + options.MaxEntrySize = 1; + options.SizeCalculator = value => value is BackgroundTask ? 2 : 1; + options.ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection; + }); + var task = NewTask(); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CreateAndEnqueueAsync(task, + current => new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = task.ProjectId!, TaskId = current.Id })); + var stored = await harness.Tasks.GetAsync(task.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var queued = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.Null(stored); + Assert.Empty(active); + Assert.Equal(0, queued.Enqueued); + Assert.Empty(harness.Publisher.Updates); + } + [Fact] public async Task GetByOrganizationAsync_FinishedTasks_ReturnsTheNewestFirstUpToTheLimit() { @@ -222,6 +252,38 @@ await Tasks.UpdateAsync(task.Id, current => Assert.Equal(100, update.Progress); } + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task UpdateAsync_TaskCannotBeStored_PreservesTheStoredStateAndActiveIndex(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, configureCache: options => + { + options.MaxEntrySize = 1; + options.SizeCalculator = value => value is BackgroundTask { Status: BackgroundTaskStatus.Completed } ? 2 : 1; + options.ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection; + }); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + })); + var stored = await harness.Tasks.GetAsync(task.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.Equal(BackgroundTaskStatus.Queued, stored!.Status); + Assert.Null(stored.CompletedUtc); + Assert.Equal([task.Id], active.ToArray()); + Assert.Empty(harness.Publisher.Updates); + } + [Fact] public async Task UpdateAsync_UnknownTask_ReturnsNullWithoutRunningTheUpdate() { @@ -350,6 +412,39 @@ await Tasks.UpdateAsync(task.Id, current => Assert.Equal([task.Id], (await Tasks.GetActiveIdsAsync(OrganizationId)).ToArray()); } + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task CancelAsync_RequestCannotBeStored_DoesNotAcknowledgeCancellation(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, configureCache: options => + { + options.MaxEntrySize = 1; + options.SizeCalculator = value => value is bool ? 2 : 1; + options.ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection; + }); + var task = NewTask(canCancel: true); + await harness.Tasks.AddAsync(task); + await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + int publishedBefore = harness.Publisher.Updates.Count; + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CancelAsync(task.Id)); + var stored = await harness.Tasks.GetAsync(task.Id); + bool cancellationRequested = await harness.Tasks.IsCancelRequestedAsync(task.Id); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.Equal(BackgroundTaskStatus.Running, stored!.Status); + Assert.False(cancellationRequested); + Assert.Equal(publishedBefore, harness.Publisher.Updates.Count); + } + [Fact] public async Task CancelAsync_RunningTaskThatStoppedReportingProgress_CancelsImmediately() { diff --git a/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs index 553633c64f..43a4b385ed 100644 --- a/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs +++ b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs @@ -17,10 +17,12 @@ namespace Exceptionless.Tests.Utility; public sealed class BackgroundTaskTestHarness : IDisposable { public BackgroundTaskTestHarness(TimeProvider timeProvider, ILoggerFactory loggerFactory, BackgroundTaskOptions? options = null, Func? decorateLockProvider = null, - IQueue? queue = null, int lockAttempts = 3) + IQueue? queue = null, int lockAttempts = 3, Action? configureCache = null) { TimeProvider = timeProvider; - Cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { CloneValues = true, TimeProvider = timeProvider, LoggerFactory = loggerFactory }); + var cacheOptions = new InMemoryCacheClientOptions { CloneValues = true, TimeProvider = timeProvider, LoggerFactory = loggerFactory }; + configureCache?.Invoke(cacheOptions); + Cache = new InMemoryCacheClient(cacheOptions); MessageBus = new InMemoryMessageBus(new InMemoryMessageBusOptions { TimeProvider = timeProvider, LoggerFactory = loggerFactory }); LockProvider = new CacheLockProvider(Cache, MessageBus, timeProvider, loggerFactory); if (decorateLockProvider is not null) From 070ddde7011a354f9ba8385155dcb7aa4767c849 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 00:00:21 -0600 Subject: [PATCH 23/42] fix: preserve bulk progress and close worker failure paths Retry checkpoints using the known batch result and cursor guard. Stop after unrecoverable checkpoint or continuation failure, handle startup retries and lost records, and reject premium filters after plan access changes. Add isolated failure-injection tests and an endpoint regression. --- .../BulkOperationWorkItemHandler.cs | 217 ++++--- .../EventBulkSelectionResolver.cs | 26 +- .../BulkOperations/IBulkOperationHandler.cs | 10 + .../BulkOperationEndpointTests.Reliability.cs | 31 + .../BulkOperationWorkItemHandlerTests.cs | 561 ++++++++++++++++++ .../EventBulkSelectionResolverPlanTests.cs | 120 ++++ 6 files changed, 886 insertions(+), 79 deletions(-) create mode 100644 tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs create mode 100644 tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs index a1c12bf08f..9f36cbe73c 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -7,6 +7,7 @@ using Exceptionless.Core.Services.BulkOperations; using Foundatio.Jobs; using Foundatio.Lock; +using Foundatio.Resilience; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Jobs.WorkItemHandlers; @@ -34,6 +35,7 @@ public class BulkOperationWorkItemHandler : WorkItemHandlerBase private readonly BulkOperationOptions _options; private readonly ILockProvider _lockProvider; private readonly TimeProvider _timeProvider; + private readonly IResiliencePolicy _checkpointPolicy; public BulkOperationWorkItemHandler(BulkOperationService service, BackgroundTaskService taskService, IEnumerable resolvers, BulkOperationOptions options, ILockProvider lockProvider, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory) @@ -44,6 +46,11 @@ public BulkOperationWorkItemHandler(BulkOperationService service, BackgroundTask _options = options; _lockProvider = lockProvider; _timeProvider = timeProvider; + _checkpointPolicy = new ResiliencePolicyBuilder(Log, timeProvider) + .WithMaxAttempts(MaxTransientFailures) + .WithDelay(options.RetryDelay) + .WithShouldRetry((_, exception) => !IsPermanent(exception)) + .Build(); // Progress is reported after every batch. Renewing the queue entry and the task lock then keeps a long run from // timing out and being delivered to a second worker while this one is still deleting. @@ -61,63 +68,70 @@ public override async Task HandleItemAsync(WorkItemContext context) var workItem = context.GetData()!; ArgumentException.ThrowIfNullOrWhiteSpace(workItem.TaskId); - var task = await _taskService.GetAsync(workItem.TaskId); - if (task is null) - { - // The record expired or the cache was cleared. Say so, otherwise the progress panel waits forever. - Log.LogWarning("Bulk operation {TaskId} no longer exists", workItem.TaskId); - await _service.PublishLostAsync(workItem); - return; - } - - using var _ = Log.BeginScope(new ExceptionlessState().Organization(task.OrganizationId).Tag("BulkOperation")); - if (!task.IsActive) + BackgroundTask? task = null; + try { - Log.LogInformation("Skipping bulk operation {TaskId} because it is already {Status}", task.Id, task.Status); - return; - } + task = await _taskService.GetAsync(workItem.TaskId); + if (task is null) + { + // The record expired or the cache was cleared. Say so, otherwise the progress panel waits forever. + Log.LogWarning("Bulk operation {TaskId} no longer exists", workItem.TaskId); + await _service.PublishLostAsync(workItem); + return; + } - var handler = _service.GetHandler(task.Type); - if (handler is null || !_resolvers.TryGetValue(handler.Entity, out var resolver)) - { - await FinishAsync(task.Id, BackgroundTaskStatus.Failed, $"Unknown bulk operation type '{task.Type}'."); - return; - } + using var _ = Log.BeginScope(new ExceptionlessState().Organization(task.OrganizationId).Tag("BulkOperation")); + if (!task.IsActive) + { + Log.LogInformation("Skipping bulk operation {TaskId} because it is already {Status}", task.Id, task.Status); + return; + } - if (await _taskService.IsCancelRequestedAsync(task.Id)) - { - await FinishAsync(task.Id, BackgroundTaskStatus.Cancelled, "Cancelled by user."); - return; - } + var handler = _service.GetHandler(task.Type); + if (handler is null || !_resolvers.TryGetValue(handler.Entity, out var resolver)) + { + await FinishAsync(workItem, BackgroundTaskStatus.Failed, $"Unknown bulk operation type '{task.Type}'."); + return; + } - string? pinnedSelection = null; - if (task.Bulk!.Selection.Kind is BulkOperationSelectionKind.Filter) - { - pinnedSelection = await _service.GetPinnedSelectionAsync(task.Id); - if (pinnedSelection is null) + if (await _taskService.IsCancelRequestedAsync(task.Id)) { - await FinishAsync(task.Id, BackgroundTaskStatus.Failed, "The saved selection expired, so the operation could not continue."); + await FinishAsync(workItem, BackgroundTaskStatus.Cancelled, "Cancelled by user."); return; } - } - task = await _taskService.UpdateAsync(task.Id, current => - { - if (!current.IsActive) - return false; + string? pinnedSelection = null; + if (task.Bulk!.Selection.Kind is BulkOperationSelectionKind.Filter) + { + pinnedSelection = await _service.GetPinnedSelectionAsync(task.Id); + if (pinnedSelection is null) + { + await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The saved selection expired, so the operation could not continue."); + return; + } + } - current.Start(UtcNow); - return true; - }); + task = await _taskService.UpdateAsync(task.Id, current => + { + if (!current.IsActive) + return false; - if (task is null || task.Status is not BackgroundTaskStatus.Running) - return; + current.Start(UtcNow); + return true; + }); - Log.LogInformation("Running bulk operation {TaskId} {Type} from cursor {Cursor}", task.Id, task.Type, task.Bulk!.Cursor); - await context.ReportProgressAsync(GetPercent(task), task.Description); + if (task is null) + { + await _service.PublishLostAsync(workItem); + return; + } + + if (task.Status is not BackgroundTaskStatus.Running) + return; + + Log.LogInformation("Running bulk operation {TaskId} {Type} from cursor {Cursor}", task.Id, task.Type, task.Bulk!.Cursor); + await context.ReportProgressAsync(GetPercent(task), task.Description); - try - { while (true) { if (context.CancellationToken.IsCancellationRequested) @@ -129,7 +143,7 @@ public override async Task HandleItemAsync(WorkItemContext context) if (await _taskService.IsCancelRequestedAsync(task.Id)) { - await FinishAsync(task.Id, BackgroundTaskStatus.Cancelled, $"Cancelled by user after {task.Processed:N0} of {task.Total:N0}."); + await FinishAsync(workItem, BackgroundTaskStatus.Cancelled, $"Cancelled by user after {task.Processed:N0} of {task.Total:N0}."); return; } @@ -142,23 +156,15 @@ public override async Task HandleItemAsync(WorkItemContext context) // the deletion they legitimately started, and any member of the organization can cancel it. var result = await handler.ExecuteBatchAsync(task, page.Ids, context.CancellationToken); - task = await _taskService.UpdateAsync(task.Id, current => + task = await SaveBatchAsync(task, page, result); + + if (task is null) { - if (!current.IsActive) - return false; - - current.Processed += page.Ids.Count; - current.Succeeded += result.Succeeded; - current.Failed += result.Failed; - current.Bulk!.Cursor = page.NextCursor ?? ExhaustedCursor; - current.Bulk!.RetryCount = 0; - current.Total = Math.Max(current.Total, current.Processed); - current.Message = result.Message; - current.Progress = GetPercent(current); - return true; - }); - - if (task is null || !task.IsActive) + await _service.PublishLostAsync(workItem); + return; + } + + if (!task.IsActive) return; await context.ReportProgressAsync(GetPercent(task), $"{task.Processed:N0} of {task.Total:N0}"); @@ -167,7 +173,7 @@ public override async Task HandleItemAsync(WorkItemContext context) break; } - await _taskService.UpdateAsync(task.Id, current => + task = await _taskService.UpdateAsync(task.Id, current => { if (!current.IsActive) return false; @@ -176,15 +182,28 @@ await _taskService.UpdateAsync(task.Id, current => current.Finish(BackgroundTaskStatus.Completed, current.Failed > 0 ? $"Completed with {current.Failed:N0} failures." : "Completed.", UtcNow); return true; }); + + if (task is null) + await _service.PublishLostAsync(workItem); } catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested) { - context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {task?.Id} for shutdown"); + context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {workItem.TaskId} for shutdown"); + } + catch (BatchCheckpointException ex) + { + Log.LogError(ex, "Bulk operation {TaskId} could not save completed batch progress: {Message}", workItem.TaskId, ex.Message); + await FinishAsync(workItem, BackgroundTaskStatus.Failed, ex.Message); + } + catch (BulkSelectionRejectedException ex) + { + Log.LogWarning(ex, "Bulk operation {TaskId} selection is no longer available: {Message}", workItem.TaskId, ex.Message); + await FinishAsync(workItem, BackgroundTaskStatus.Failed, ex.Message); } catch (Exception ex) when (!IsPermanent(ex)) { // Elasticsearch, cache and network hiccups: continue in a new work item from the saved counts and cursor. Rethrowing - // would use the queue's retries, which cover the whole work item, so an task that makes progress between errors + // would use the queue's retries, which cover the whole work item, so a task that makes progress between errors // would run out of deliveries while still marked running. After MaxTransientFailures in a row, give up. var failed = await _taskService.UpdateAsync(workItem.TaskId, current => { @@ -205,19 +224,67 @@ await _taskService.UpdateAsync(task.Id, current => return; } - Log.LogWarning(ex, "Bulk operation {TaskId} hit a temporary error and will continue in a new run: {Message}", workItem.TaskId, ex.Message); + Log.LogWarning(ex, "Bulk operation {TaskId} hit a temporary error: {Message}", workItem.TaskId, ex.Message); if (failed is null) + { + await _service.PublishLostAsync(workItem); return; + } - if (_options.RetryDelay > TimeSpan.Zero) - await Task.Delay(_options.RetryDelay * failed.Bulk!.RetryCount, _timeProvider, context.CancellationToken); + try + { + if (_options.RetryDelay > TimeSpan.Zero) + await Task.Delay(_options.RetryDelay * failed.Bulk!.RetryCount, _timeProvider, context.CancellationToken); - await _service.ContinueAsync(workItem); + await _service.ContinueAsync(workItem); + } + catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested) + { + context.Result = JobResult.CancelledWithMessage($"Stopping bulk operation {workItem.TaskId} for shutdown"); + } + catch (Exception continuationException) + { + Log.LogError(continuationException, "Bulk operation {TaskId} could not queue another run: {Message}", workItem.TaskId, continuationException.Message); + await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The operation stopped because another run could not be queued."); + } } catch (Exception ex) { Log.LogError(ex, "Bulk operation {TaskId} failed: {Message}", workItem.TaskId, ex.Message); - await FinishAsync(workItem.TaskId, BackgroundTaskStatus.Failed, "The operation stopped because of an unexpected error."); + await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The operation stopped because of an unexpected error."); + } + } + + private async Task SaveBatchAsync(BackgroundTask task, BulkSelectionPage page, BulkBatchResult result) + { + // Keep the known result while retrying its checkpoint, including during shutdown. Replaying a successful delete + // would classify its now-missing ids as failures or omit them from a filter selection's counts. A process stopping + // before any checkpoint is saved can still lose that result; the deletion and task record are separate writes. + try + { + return await _checkpointPolicy.ExecuteAsync(async _ => await _taskService.UpdateAsync(task.Id, current => + { + // A save can succeed before releasing its update lock throws. The prior cursor makes another attempt harmless + // if that checkpoint was committed already, or if the task has since advanced or finished. + if (!current.IsActive || !String.Equals(current.Bulk!.Cursor, task.Bulk!.Cursor, StringComparison.Ordinal)) + return false; + + current.Processed += page.Ids.Count; + current.Succeeded += result.Succeeded; + current.Failed += result.Failed; + current.Bulk.Cursor = page.NextCursor ?? ExhaustedCursor; + current.Bulk.RetryCount = 0; + current.Total = Math.Max(current.Total, current.Processed); + current.Message = result.Message; + current.Progress = GetPercent(current); + return true; + }), CancellationToken.None); + } + catch (Exception ex) + { + // Replaying a batch known to have succeeded would lose its result again. Stop if its bounded checkpoint + // retries are exhausted, keeping the last durable counts and making the uncertain outcome explicit. + throw new BatchCheckpointException(ex); } } @@ -255,9 +322,9 @@ private async Task GetPageAsync(BackgroundTask task, IBulkSel private DateTime UtcNow => _timeProvider.GetUtcNow().UtcDateTime; - private Task FinishAsync(string taskId, BackgroundTaskStatus status, string message) + private async Task FinishAsync(BulkOperationWorkItem workItem, BackgroundTaskStatus status, string message) { - return _taskService.UpdateAsync(taskId, current => + var task = await _taskService.UpdateAsync(workItem.TaskId, current => { if (!current.IsActive) return false; @@ -265,10 +332,16 @@ private Task FinishAsync(string taskId, BackgroundTaskStatus status, string mess current.Finish(status, message, UtcNow); return true; }); + + if (task is null) + await _service.PublishLostAsync(workItem); } private static int GetPercent(BackgroundTask task) { return task.Total <= 0 ? 0 : (int)Math.Min(99, task.Processed * 100 / task.Total); } + + private sealed class BatchCheckpointException(Exception innerException) + : InvalidOperationException("The operation stopped because its progress could not be saved. Some items may have been processed.", innerException); } diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs index c4de9674d5..c88bae8695 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -24,6 +24,7 @@ namespace Exceptionless.Core.Services.BulkOperations; public class EventBulkSelectionResolver : IBulkSelectionResolver { private const string CreatedUtcField = "created_utc"; + private const string PremiumSearchUpgradeMessage = "Please upgrade your plan to use premium search features."; private readonly IEventRepository _eventRepository; private readonly IOrganizationRepository _organizationRepository; private readonly IProjectRepository _projectRepository; @@ -56,14 +57,17 @@ public async Task ValidateSelectionAsync(Organization return BulkOperationCheckResult.Invalid("selection.filter", result.Message ?? "Invalid filter."); if (result.UsesPremiumFeatures && !organization.HasPremiumFeatures) - return BulkOperationCheckResult.PlanLimit("Please upgrade your plan to use premium search features."); + return BulkOperationCheckResult.PlanLimit(PremiumSearchUpgradeMessage); return BulkOperationCheckResult.Ok; } public async Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken) { - var appFilter = await GetAppFilterAsync(task); + var (appFilter, check) = await GetAppFilterAsync(task); + if (!check.IsOk) + return BulkSelectionPreparation.Rejected(check); + if (appFilter is null) return new BulkSelectionPreparation(BulkOperationCheckResult.Ok); @@ -99,7 +103,10 @@ public async Task PrepareAsync(BackgroundTask task, Ca public async Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) { - var appFilter = await GetAppFilterAsync(task); + var (appFilter, check) = await GetAppFilterAsync(task); + if (!check.IsOk) + throw new BulkSelectionRejectedException(check); + if (appFilter is null || pinnedState is null) return new BulkSelectionPage([], null); @@ -141,11 +148,11 @@ private static RepositoryQuery ApplyScope(RepositoryQuery GetAppFilterAsync(BackgroundTask task) + private async Task<(AppFilter? Filter, BulkOperationCheckResult Check)> GetAppFilterAsync(BackgroundTask task) { var organization = await _organizationRepository.GetByIdAsync(task.OrganizationId, o => o.Cache()); if (organization is null) - return null; + return (null, BulkOperationCheckResult.Ok); var validation = await _validator.ValidateQueryAsync(task.Bulk!.Selection.Filter); AppFilter appFilter; @@ -153,7 +160,7 @@ private static RepositoryQuery ApplyScope(RepositoryQuery o.Cache()); if (project is null || !String.Equals(project.OrganizationId, organization.Id, StringComparison.Ordinal)) - return null; + return (null, BulkOperationCheckResult.Ok); appFilter = new AppFilter(project, organization); } @@ -162,8 +169,13 @@ private static RepositoryQuery ApplyScope(RepositoryQueryWhat the stack level filter resolved to when the task was created. diff --git a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs index eb783ab7e4..92f19151df 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs @@ -32,6 +32,10 @@ public interface IBulkOperationHandler /// Executes one batch of ids through the same code path as the single-item action, so side effects stay identical. /// Ids that cannot be processed (missing, or outside the task's organization and project) must be reported as failed. /// + /// + /// Handlers must tolerate at-least-once execution: a process can stop after the action but before its checkpoint is + /// saved, so the same ids may be delivered again. Missing ids must still be reported as failed. + /// Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken); } @@ -64,6 +68,12 @@ public record BulkSelectionPreparation(BulkOperationCheckResult Check, long Tota public record BulkSelectionPage(IReadOnlyList Ids, string? NextCursor); +/// A previously accepted selection can no longer run under the organization's current plan. +public sealed class BulkSelectionRejectedException(BulkOperationCheckResult check) : InvalidOperationException(check.Message) +{ + public BulkOperationCheckResult Check { get; } = check; +} + public record BulkBatchResult(int Succeeded, int Failed, string? Message = null); /// The authenticated caller, independent of the transport the request arrived on. diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs index 895dcd0255..cc862adfb6 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -99,6 +99,37 @@ public async Task Execute_StackStatusChangesAfterCreation_StillDeletesTheEventsT Assert.Equal(["fixed"], (await GetRemainingMessagesAsync(events)).ToArray()); } + [Fact] + public async Task Execute_PremiumFeaturesRemovedAfterCreation_FailsWithoutDeletingOrLosingTheSelectionCount() + { + // Arrange + var (_, events) = await CreateDataAsync(data => data.Event().TestProject().Message("selected")); + var organizations = GetService(); + + // Act + var created = await CreateOperationAsync(FilterRequest(SampleDataService.TEST_ORG_ID, null, "message:selected")); + + // Assert + Assert.Equal(1, created.Total); + + // Arrange + var organization = await organizations.GetByIdAsync(SampleDataService.TEST_ORG_ID); + organization!.HasPremiumFeatures = false; + await organizations.SaveAsync(organization, options => options.ImmediateConsistency()); + + // Act + await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var finished = await GetOperationAsync(created.Id); + + // Assert + Assert.Equal(BackgroundTaskStatus.Failed, finished.Status); + Assert.Equal(1, finished.Total); + Assert.Equal(0, finished.Processed); + Assert.Equal(0, finished.Succeeded); + Assert.Equal("Please upgrade your plan to use premium search features.", finished.Message); + Assert.Equal(["selected"], (await GetRemainingMessagesAsync(events)).ToArray()); + } + [Fact] public async Task Create_MoreThanFiveHundredOperations_StillSeesTheNewestActiveOneForTheConflictCheck() { diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs new file mode 100644 index 0000000000..8ce68d537f --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs @@ -0,0 +1,561 @@ +using Exceptionless.Core.Jobs.WorkItemHandlers; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Queues; +using Foundatio.Utility; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Exceptionless.Tests.Jobs.WorkItemHandlers; + +public class BulkOperationWorkItemHandlerTests(ITestOutputHelper output) : TestWithLoggingBase(output) +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private const string ProjectId = "537650f3b77efe23a47914f6"; + private readonly FakeTimeProvider _time = new(); + + [Theory] + [InlineData(1, BackgroundTaskStatus.Completed, 1)] + [InlineData(3, BackgroundTaskStatus.Failed, 0)] + public async Task HandleItemAsync_StartupTemporarilyFails_UsesBoundedContinuationWithoutAbandoningAnActiveTask(int startupFailures, BackgroundTaskStatus expectedStatus, int expectedBatches) + { + // Arrange + var queue = new InMemoryQueue(new InMemoryQueueOptions + { + TimeProvider = _time, + LoggerFactory = Log, + Retries = 2, + RetryDelay = TimeSpan.Zero + }); + var locks = new FaultingLockProvider(); + using var harness = new BackgroundTaskTestHarness(_time, Log, decorateLockProvider: locks.Wrap, queue: queue, lockAttempts: 1); + using var subscription = queue.Dequeued.AddSyncHandler((_, _) => + { + if (startupFailures-- <= 0) + return; + + locks.AcquireFailuresRemaining = 1; + locks.NextAcquireFailure = new TimeoutException("Task startup temporarily unavailable"); + }); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + batches++; + return Task.FromResult(new BulkBatchResult(batch.Count, 0)); + }); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first"])); + var job = CreateJob(harness, service, resolver, batchSize: 1); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + var stats = await queue.GetQueueStatsAsync(); + + // Assert + Assert.NotNull(stored); + Assert.Equal(expectedStatus, stored.Status); + Assert.Equal(expectedBatches, batches); + Assert.Equal(expectedBatches, stored.Processed); + Assert.Equal(expectedBatches, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.Equal(0, stats.Abandoned); + Assert.Equal(0, stats.Deadletter); + Assert.Empty(await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Theory] + [InlineData(BulkOperationSelectionKind.Ids, false)] + [InlineData(BulkOperationSelectionKind.Filter, false)] + [InlineData(BulkOperationSelectionKind.Ids, true)] + [InlineData(BulkOperationSelectionKind.Filter, true)] + public async Task HandleItemAsync_CompletedBatchCheckpointTemporarilyFails_PreservesItsResult(BulkOperationSelectionKind kind, bool checkpointCommitted) + { + // Arrange + string[] ids = ["first", "second", "third"]; + var remaining = ids.ToHashSet(StringComparer.Ordinal); + var locks = new FaultingLockProvider(); + using var harness = new BackgroundTaskTestHarness(_time, Log, decorateLockProvider: locks.Wrap, lockAttempts: 1); + var executedIds = new List(); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + executedIds.AddRange(batch); + int succeeded = batch.Count(remaining.Remove); + if (++batches == 1) + { + if (checkpointCommitted) + locks.NextReleaseFailure = new TimeoutException("Checkpoint saved but lock release failed"); + else + locks.NextAcquireFailure = new TimeoutException("Checkpoint temporarily unavailable"); + } + + return Task.FromResult(new BulkBatchResult(succeeded, batch.Count - succeeded)); + }); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids, kind)); + var job = CreateJob(harness, service, resolver, batchSize: 2); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + + // Assert + Assert.Empty(remaining); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Completed, stored.Status); + Assert.Equal(ids.Length, stored.Processed); + Assert.Equal(ids.Length, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.Equal(ids, executedIds); + } + + [Fact] + public async Task HandleItemAsync_HostStopsAfterSuccessfulBatch_PersistsItsResultBeforeStopping() + { + // Arrange + string[] ids = ["first", "second", "third"]; + var remaining = ids.ToHashSet(StringComparer.Ordinal); + using var shutdown = new CancellationTokenSource(); + var locks = new FaultingLockProvider(); + using var harness = new BackgroundTaskTestHarness(_time, Log, decorateLockProvider: locks.Wrap, lockAttempts: 1); + int batches = 0; + var executedIds = new List(); + var operationHandler = new DelegateBulkHandler(batch => + { + executedIds.AddRange(batch); + int succeeded = batch.Count(remaining.Remove); + if (++batches == 1) + { + locks.NextAcquireFailure = new TimeoutException("Checkpoint temporarily unavailable"); + shutdown.Cancel(); + } + + return Task.FromResult(new BulkBatchResult(succeeded, batch.Count - succeeded)); + }); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids)); + var handler = CreateHandler(harness, service, resolver, batchSize: 2); + var interruptedContext = CreateContext(created.Task!.Id, shutdown.Token); + + // Act + await handler.HandleItemAsync(interruptedContext); + var interrupted = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.False(interruptedContext.Result.IsSuccess); + Assert.NotNull(interrupted); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); + Assert.Equal(2, interrupted.Processed); + Assert.Equal(2, interrupted.Succeeded); + + // Act + await handler.HandleItemAsync(CreateContext(created.Task.Id, TestCancellationToken)); + var completed = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(completed); + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(ids.Length, completed.Succeeded); + Assert.Equal(0, completed.Failed); + Assert.Equal(ids, executedIds); + Assert.Empty(remaining); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task HandleItemAsync_CompletedBatchCheckpointKeepsFailing_StopsWithoutReplayingOrDiscardingPriorProgress(bool dependencyCancelled) + { + // Arrange + string[] ids = ["first", "second", "third", "fourth", "fifth"]; + var remaining = ids.ToHashSet(StringComparer.Ordinal); + var locks = new FaultingLockProvider(); + using var harness = new BackgroundTaskTestHarness(_time, Log, decorateLockProvider: locks.Wrap, lockAttempts: 1); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + int succeeded = batch.Count(remaining.Remove); + if (++batches == 2) + { + locks.AcquireFailuresRemaining = BulkOperationWorkItemHandler.MaxTransientFailures; + locks.NextAcquireFailure = dependencyCancelled + ? new TaskCanceledException("Checkpoint cancelled by dependency") + : new TimeoutException("Checkpoint unavailable for every attempt"); + } + + return Task.FromResult(new BulkBatchResult(succeeded, batch.Count - succeeded)); + }); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids)); + var job = CreateJob(harness, service, resolver, batchSize: 2); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + var stats = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Equal(ids.Length, stored.Total); + Assert.Equal(2, stored.Processed); + Assert.Equal(2, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.Contains("Some items may have been processed", stored.Message); + Assert.Equal(["fifth"], remaining); + Assert.Equal(2, batches); + Assert.Equal(1, stats.Enqueued); + Assert.Equal(1, stats.Completed); + Assert.Empty(await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Theory] + [InlineData(BatchFailure.None)] + [InlineData(BatchFailure.Temporary)] + [InlineData(BatchFailure.Permanent)] + public async Task HandleItemAsync_TaskRecordDisappearsDuringBatch_PublishesLostAndStops(BatchFailure failure) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var task = NewTask(["first", "second"]); + int batches = 0; + var operationHandler = new DelegateBulkHandler(async batch => + { + batches++; + await harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(task.Id)); + if (failure is BatchFailure.Temporary) + throw new TimeoutException("Batch interrupted after the task record was lost"); + if (failure is BatchFailure.Permanent) + throw new InvalidOperationException("Batch rejected after the task record was lost"); + + return new BulkBatchResult(batch.Count, 0); + }); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + await service.CreateAsync(task); + var job = CreateJob(harness, service, resolver, batchSize: 1); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Null(stored); + Assert.Equal(1, batches); + var lost = Assert.Single(harness.Publisher.Updates, update => update.Status is BackgroundTaskStatus.Failed); + Assert.Equal(task.Id, lost.Id); + Assert.Equal(task.OrganizationId, lost.OrganizationId); + Assert.Equal(task.Type, lost.Type); + Assert.Equal(task.Description, lost.Description); + Assert.Contains("progress was lost", lost.Message); + Assert.Equal(1, (await harness.Queue.GetQueueStatsAsync()).Completed); + } + + [Fact] + public async Task HandleItemAsync_TaskRecordDisappearsAfterFinalCheckpoint_PublishesLostInsteadOfCompletion() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var task = NewTask(["first"]); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + batches++; + return Task.FromResult(new BulkBatchResult(batch.Count, 0)); + }); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + await service.CreateAsync(task); + using var subscription = harness.Queue.LockRenewed.AddHandler(async (_, _) => + { + var checkpoint = await harness.Tasks.GetAsync(task.Id); + if (checkpoint is { Processed: 1 }) + await harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(task.Id)); + }); + var job = CreateJob(harness, service, resolver, batchSize: 1); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Null(stored); + Assert.Equal(1, batches); + Assert.Single(harness.Publisher.Updates, update => update.Status is BackgroundTaskStatus.Failed); + Assert.DoesNotContain(harness.Publisher.Updates, update => update.Status is BackgroundTaskStatus.Completed); + Assert.Equal(1, (await harness.Queue.GetQueueStatsAsync()).Completed); + } + + [Fact] + public async Task HandleItemAsync_ContinuationCannotBeQueued_DoesNotLeaveAnActiveTaskWithoutWork() + { + // Arrange + var queue = new FaultingEnqueueQueue(new InMemoryQueueOptions + { + TimeProvider = _time, + LoggerFactory = Log, + Retries = 2, + RetryDelay = TimeSpan.Zero + }); + using var harness = new BackgroundTaskTestHarness(_time, Log, queue: queue); + int calls = 0; + var operationHandler = new DelegateBulkHandler(batch => ++calls % 2 == 0 + ? Task.FromException(new TimeoutException("Search temporarily unavailable")) + : Task.FromResult(new BulkBatchResult(batch.Count, 0))); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first", "second", "third", "fourth"])); + var job = CreateJob(harness, service, resolver, batchSize: 1); + queue.FailEnqueue = true; + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + var stats = await queue.GetQueueStatsAsync(); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.Equal(0, stats.Queued); + Assert.Equal(0, stats.Working); + Assert.Equal(0, stats.Deadletter); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Equal(4, stored.Total); + Assert.Equal(1, stored.Processed); + Assert.Equal(1, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.DoesNotContain(stored.Id, active); + } + + [Fact] + public async Task HandleItemAsync_HostStopsBeforeContinuation_KeepsTheTaskAvailableForQueueRetry() + { + // Arrange + using var shutdown = new CancellationTokenSource(); + using var harness = new BackgroundTaskTestHarness(_time, Log); + var operationHandler = new DelegateBulkHandler(_ => + { + shutdown.Cancel(); + return Task.FromException(new TimeoutException("Search interrupted during shutdown")); + }); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first"])); + var handler = CreateHandler(harness, service, resolver, batchSize: 1, retryDelay: TimeSpan.FromSeconds(5)); + var context = CreateContext(created.Task!.Id, shutdown.Token); + + // Act + var thrown = await Record.ExceptionAsync(() => handler.HandleItemAsync(context)); + var stored = await harness.Tasks.GetAsync(created.Task.Id); + var stats = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.Null(thrown); + Assert.False(context.Result.IsSuccess); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Running, stored.Status); + Assert.Null(stored.CompletedUtc); + Assert.Equal(1, stored.Bulk!.RetryCount); + Assert.Contains(stored.Id, await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + Assert.Equal(1, stats.Enqueued); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + public async Task HandleItemAsync_SelectionIsRejected_FailsWithoutLosingCompletedCountsOrRetrying(int successfulBatches) + { + // Arrange + const string message = "Please upgrade your plan to use premium search features."; + string[] ids = ["first", "second", "third"]; + var remaining = ids.ToHashSet(StringComparer.Ordinal); + using var harness = new BackgroundTaskTestHarness(_time, Log); + var operationHandler = new DelegateBulkHandler(batch => Task.FromResult(new BulkBatchResult(batch.Count(remaining.Remove), 0))); + var resolver = new RemainingIdsResolver(remaining) + { + Failure = call => call == successfulBatches + 1 ? new BulkSelectionRejectedException(BulkOperationCheckResult.PlanLimit(message)) : null + }; + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids, BulkOperationSelectionKind.Filter)); + var job = CreateJob(harness, service, resolver, batchSize: 1); + + // Act + await job.RunUntilEmptyAsync(TestCancellationToken); + var stored = await harness.Tasks.GetAsync(created.Task!.Id); + var stats = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Equal(message, stored.Message); + Assert.Equal(ids.Length, stored.Total); + Assert.Equal(successfulBatches, stored.Processed); + Assert.Equal(successfulBatches, stored.Succeeded); + Assert.Equal(0, stored.Failed); + Assert.Equal(0, stored.Bulk!.RetryCount); + Assert.Equal(ids.Length - successfulBatches, remaining.Count); + Assert.Equal(1, stats.Enqueued); + Assert.Equal(1, stats.Completed); + Assert.Empty(await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + } + + private BulkOperationService CreateService(BackgroundTaskTestHarness harness, IBulkOperationHandler handler, IBulkSelectionResolver resolver) + { + return new BulkOperationService(harness.Tasks, harness.Cache, harness.LockProvider, [handler], [resolver], _time); + } + + private WorkItemJob CreateJob(BackgroundTaskTestHarness harness, BulkOperationService service, IBulkSelectionResolver resolver, int batchSize) + { + var handler = CreateHandler(harness, service, resolver, batchSize); + var handlers = new Foundatio.Jobs.WorkItemHandlers(); + handlers.Register(() => handler); + return new WorkItemJob(harness.Queue, harness.Publisher, handlers, Log); + } + + private BulkOperationWorkItemHandler CreateHandler(BackgroundTaskTestHarness harness, BulkOperationService service, IBulkSelectionResolver resolver, int batchSize, TimeSpan? retryDelay = null) + { + return new BulkOperationWorkItemHandler(service, harness.Tasks, [resolver], + new BulkOperationOptions { BatchSize = batchSize, RetryDelay = retryDelay ?? TimeSpan.Zero }, harness.LockProvider, _time, Log); + } + + private static WorkItemContext CreateContext(string taskId, CancellationToken cancellationToken) + { + var workItem = new BulkOperationWorkItem { TaskId = taskId, OrganizationId = OrganizationId }; + return new WorkItemContext(workItem, "test-job", EmptyLock.Empty, cancellationToken, static (_, _) => Task.CompletedTask); + } + + private static BackgroundTask NewTask(IReadOnlyCollection ids, BulkOperationSelectionKind kind = BulkOperationSelectionKind.Ids) + { + return new BackgroundTask + { + OrganizationId = OrganizationId, + ProjectId = ProjectId, + Type = "test.delete", + Description = "Delete selected items", + CreatedByUserId = "537650f3b77efe23a47914f8", + Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection { Kind = kind, Ids = kind is BulkOperationSelectionKind.Ids ? ids.ToList() : null } + } + }; + } + + private sealed class DelegateBulkHandler(Func, Task> execute) : IBulkOperationHandler + { + public string Type => "test.delete"; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => true; + public Task AuthorizeAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task ValidateAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public string Describe(BackgroundTask task) => "Delete selected items"; + + public Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) => execute(ids); + } + + private sealed class RemainingIdsResolver(HashSet remaining) : IBulkSelectionResolver + { + private int _calls; + + public Func? Failure { get; init; } + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken) => Task.FromResult(new BulkSelectionPreparation(BulkOperationCheckResult.Ok, remaining.Count, "{}")); + + public Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) + { + if (Failure?.Invoke(++_calls) is { } failure) + return Task.FromException(failure); + + string[] ids = remaining.Order(StringComparer.Ordinal).Take(size).ToArray(); + return Task.FromResult(new BulkSelectionPage(ids, ids.LastOrDefault())); + } + } + + private sealed class FaultingEnqueueQueue(InMemoryQueueOptions options) : InMemoryQueue(options) + { + public bool FailEnqueue { get; set; } + + protected override Task EnqueueImplAsync(WorkItemData data, QueueEntryOptions options) + { + return FailEnqueue + ? Task.FromException(new TimeoutException("Continuation queue unavailable")) + : base.EnqueueImplAsync(data, options); + } + } + + private sealed class FaultingLockProvider : ILockProvider + { + private ILockProvider _inner = null!; + + public Exception? NextAcquireFailure { get; set; } + public Exception? NextReleaseFailure { get; set; } + public int AcquireFailuresRemaining { get; set; } = 1; + + public ILockProvider Wrap(ILockProvider inner) + { + _inner = inner; + return this; + } + + public async Task AcquireAsync(string resource, TimeSpan? timeUntilExpires = null, bool releaseOnDispose = true, CancellationToken cancellationToken = default) + { + if (NextAcquireFailure is { } failure) + { + if (--AcquireFailuresRemaining == 0) + NextAcquireFailure = null; + + throw failure; + } + + var acquired = await _inner.AcquireAsync(resource, timeUntilExpires, releaseOnDispose, cancellationToken); + if (NextReleaseFailure is not { } releaseFailure) + return acquired; + + NextReleaseFailure = null; + return new FaultingReleaseLock(acquired, releaseFailure); + } + + public Task TryAcquireAsync(string resource, TimeSpan? timeUntilExpires = null, bool releaseOnDispose = true, CancellationToken cancellationToken = default) => _inner.TryAcquireAsync(resource, timeUntilExpires, releaseOnDispose, cancellationToken); + public Task IsLockedAsync(string resource) => _inner.IsLockedAsync(resource); + public Task ReleaseAsync(string resource, string lockId) => _inner.ReleaseAsync(resource, lockId); + public Task ReleaseAsync(string resource) => _inner.ReleaseAsync(resource); + public Task RenewAsync(string resource, string lockId, TimeSpan? timeUntilExpires = null) => _inner.RenewAsync(resource, lockId, timeUntilExpires); + } + + private sealed class FaultingReleaseLock(ILock inner, Exception failure) : ILock + { + public string LockId => inner.LockId; + public string Resource => inner.Resource; + public DateTime AcquiredTimeUtc => inner.AcquiredTimeUtc; + public TimeSpan TimeWaitedForLock => inner.TimeWaitedForLock; + public int RenewalCount => inner.RenewalCount; + public Task RenewAsync(TimeSpan? timeUntilExpires = null) => inner.RenewAsync(timeUntilExpires); + public Task ReleaseAsync() => inner.ReleaseAsync(); + + public async ValueTask DisposeAsync() + { + await inner.DisposeAsync(); + throw failure; + } + } + + public enum BatchFailure + { + None, + Temporary, + Permanent + } +} diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs new file mode 100644 index 0000000000..f44a94b6fd --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs @@ -0,0 +1,120 @@ +using Elastic.Clients.Elasticsearch.Mapping; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Queries.Validation; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Services.BulkOperations; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; +using Foundatio.Serializer; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class EventBulkSelectionResolverPlanTests : TestWithServices +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private readonly Organization _organization = new() { Id = OrganizationId, HasPremiumFeatures = true }; + private readonly Mock _events = new(); + private readonly EventBulkSelectionResolver _resolver; + + public EventBulkSelectionResolverPlanTests(ITestOutputHelper output) : base(output) + { + var organizations = new Mock(); + organizations.Setup(repository => repository.GetByIdAsync(OrganizationId, It.IsAny>())).ReturnsAsync(_organization); + _events.Setup(repository => repository.CountAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new CountResult(1)); + _events.Setup(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new FindResults()); + var validator = GetService(); + var serializer = GetService(); + var configuration = GetService(); + // These filters use only the configured event fields, so no server mapping is needed. + var mapping = new TypeMappingDescriptor(); + configuration.Events.ConfigureIndexMapping(mapping); + configuration.Events.QueryParser.Configuration!.UseMappings(() => mapping, configuration.Client.Infer); + _resolver = new EventBulkSelectionResolver(_events.Object, organizations.Object, Mock.Of(), validator, serializer, configuration); + } + + [Fact] + public async Task PrepareAsync_PremiumFeaturesRemovedAfterValidation_RejectsBeforeCounting() + { + // Arrange + var task = NewTask("message:selected"); + + // Act + var accepted = await _resolver.ValidateSelectionAsync(_organization, task.Bulk!.Selection); + _organization.HasPremiumFeatures = false; + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Assert + Assert.True(accepted.IsOk); + Assert.Equal(BulkOperationCheckStatus.PlanLimit, preparation.Check.Status); + Assert.Equal("Please upgrade your plan to use premium search features.", preparation.Check.Message); + Assert.Null(preparation.PinnedState); + _events.VerifyNoOtherCalls(); + } + + [Fact] + public async Task GetPageAsync_PremiumFeaturesRemovedAfterPreparation_RejectsBeforeSearching() + { + // Arrange + var task = NewTask("message:selected"); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + _organization.HasPremiumFeatures = false; + + // Act + var thrown = await Record.ExceptionAsync(async () => await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken)); + + // Assert + Assert.True(preparation.Check.IsOk); + Assert.Equal(1, preparation.Total); + var rejection = Assert.IsType(thrown); + Assert.Equal(BulkOperationCheckStatus.PlanLimit, rejection.Check.Status); + Assert.Equal("Please upgrade your plan to use premium search features.", rejection.Message); + _events.Verify(repository => repository.CountAsync(It.IsAny(), It.IsAny()), Times.Once); + _events.VerifyNoOtherCalls(); + } + + [Fact] + public async Task GetPageAsync_PremiumFeaturesRemovedForFreeFilter_StillSearches() + { + // Arrange + var task = NewTask("type:log"); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + _organization.HasPremiumFeatures = false; + + // Act + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); + + // Assert + Assert.True(preparation.Check.IsOk); + Assert.Equal(1, preparation.Total); + Assert.Empty(page.Ids); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + private BackgroundTask NewTask(string filter) + { + var now = TimeProvider.GetUtcNow().UtcDateTime; + return new BackgroundTask + { + OrganizationId = OrganizationId, + CreatedUtc = now, + Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Filter, + Filter = filter, + UtcStart = now.AddDays(-1), + UtcEnd = now, + UtcCreatedBefore = now + } + } + }; + } +} From 4e1b317e331c5648aef06e5a10f4bbe6ff9277c7 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 00:00:23 -0600 Subject: [PATCH 24/42] fix: preserve task reconciliation and selection intent Keep unknown-task discovery alive when pushed updates cancel stale list reads, preserve task expiry timers during progress, and bind pending all-matching counts to the page selection. Add regression coverage for each asynchronous race. --- .../all-matching-selection.svelte.test.ts | 44 +++++++++- .../table/all-matching-selection.svelte.ts | 28 ++++-- .../src/lib/features/tasks/api.svelte.ts | 42 +++++---- .../src/lib/features/tasks/api.test.ts | 85 ++++++++++++++++--- .../tasks/components/task-panel.svelte | 3 +- .../components/task-panel.svelte.test.ts | 26 +++++- .../src/routes/(app)/event/+page.svelte | 24 +++--- 7 files changed, 202 insertions(+), 50 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts index cee0a284d6..7800dd442f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.test.ts @@ -1,3 +1,4 @@ +import { createTable, stockFeatures, type StockFeatures, type Table, tableFeatures } from '@tanstack/svelte-table'; import { describe, expect, it } from 'vitest'; import { createAllMatchingSelection, getEventsQueryIdentity } from './all-matching-selection.svelte'; @@ -38,11 +39,11 @@ describe('createAllMatchingSelection', () => { // Arrange let identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' }); const selection = createAllMatchingSelection(() => identity); - const identityWhenCountingStarted = selection.identity; + const selectionWhenCountingStarted = selection.snapshot; // Act identity = getEventsQueryIdentity({ filter: 'type:log', organizationId: 'organization-id' }); - const selected = selection.select(1234, identityWhenCountingStarted); + const selected = selection.select(1234, selectionWhenCountingStarted); // Assert expect(selected).toBe(false); @@ -64,4 +65,43 @@ describe('createAllMatchingSelection', () => { // Assert expect(selection.active).toBeUndefined(); }); + + it.each([ + ['unchanged', false], + ['cleared and selected again', true] + ])('handles a deferred count when the page selection is %s', async (_, changeSelection) => { + // Arrange + const identity = getEventsQueryIdentity({ filter: 'type:error', organizationId: 'organization-id' }); + let table: Table; + const dispose = $effect.root(() => { + table = createTable({ + columns: [{ accessorKey: 'id' }], + data: [{ id: 'event-1' }, { id: 'event-2' }], + features: tableFeatures(stockFeatures), + getRowId: (row) => row.id + }); + }); + table!.toggleAllRowsSelected(true); + const selection = createAllMatchingSelection( + () => identity, + () => table!.store.state.rowSelection + ); + const counting = Promise.withResolvers(); + const selectionWhenCountingStarted = selection.snapshot; + const selecting = counting.promise.then((count) => selection.select(count, selectionWhenCountingStarted)); + + // Act + if (changeSelection) { + table!.resetRowSelection(); + table!.toggleAllRowsSelected(true); + } + counting.resolve(1234); + const selected = await selecting; + dispose(); + + // Assert + expect(table!.getIsAllRowsSelected()).toBe(true); + expect(selected).toBe(!changeSelection); + expect(selection.active).toEqual(changeSelection ? undefined : { count: 1234, identity }); + }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts index cdb891d0ff..3c41f18933 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/all-matching-selection.svelte.ts @@ -1,3 +1,5 @@ +import type { RowSelectionState } from '@tanstack/svelte-table'; + export interface AllMatchingSelection { /** How many events matched when the user chose "select all matching". */ count: number; @@ -5,6 +7,11 @@ export interface AllMatchingSelection { identity: string; } +export interface AllMatchingSelectionSnapshot { + identity: string; + rowSelection: RowSelectionState | undefined; +} + /** Everything matching the current filter, as passed to an action menu. */ export interface AllMatchingTarget { count: number; @@ -19,7 +26,7 @@ export interface AllMatchingTarget { * for: as soon as the query identity (organization, filter, time range, saved view) changes it is no longer active, so * an action can never run against a different set of events than the user confirmed. */ -export function createAllMatchingSelection(getIdentity: () => string) { +export function createAllMatchingSelection(getIdentity: () => string, getRowSelection?: () => RowSelectionState) { let selection = $state(); return { @@ -29,17 +36,13 @@ export function createAllMatchingSelection(getIdentity: () => string) { clear() { selection = undefined; }, - /** The query the list is showing right now. Capture it before awaiting a count, then pass it to select. */ - get identity(): string { - return getIdentity(); - }, /** - * Selects everything matching the current query. When an expected identity is given and the query changed in - * the meantime (the count was fetched for a different filter), nothing is selected and false is returned. + * Selects everything matching the current query. A pending count must not replace a newer filter or page + * selection, including clearing and then selecting the same rows again. */ - select(count: number, expectedIdentity?: string): boolean { + select(count: number, expected?: AllMatchingSelectionSnapshot): boolean { const identity = getIdentity(); - if (expectedIdentity !== undefined && expectedIdentity !== identity) { + if (expected && (expected.identity !== identity || expected.rowSelection !== getRowSelection?.())) { return false; } @@ -48,6 +51,13 @@ export function createAllMatchingSelection(getIdentity: () => string) { identity }; return true; + }, + /** Capture the query and page selection before awaiting a count, then pass the snapshot to select. */ + get snapshot(): AllMatchingSelectionSnapshot { + return { + identity: getIdentity(), + rowSelection: getRowSelection?.() + }; } }; } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts index 805bc2844a..b0f8aa24d9 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts @@ -110,6 +110,17 @@ export async function updateTaskFromMessage(queryClient: QueryClient, update: Ba return; } + await updateCachedTasks(queryClient, key, (latest) => (latest ? applyTaskUpdate(latest, update) : undefined)); +} + +async function updateCachedTasks( + queryClient: QueryClient, + key: ReturnType, + update: (current: undefined | ViewBackgroundTask[]) => undefined | ViewBackgroundTask[] +) { + const wasInvalidated = queryClient.getQueryState(key)?.isInvalidated; + // An older list response must not remove newly queued work and stop polling. Keep any pushed updates received + // since that read started instead of reverting the cache to its pre-fetch snapshot. await queryClient.cancelQueries( { queryKey: key @@ -118,27 +129,26 @@ export async function updateTaskFromMessage(queryClient: QueryClient, update: Ba revert: false } ); - // Other messages can be applied while cancellation settles; merge with the latest cache, not the earlier snapshot. - const latest = queryClient.getQueryData(key); - const updated = latest ? applyTaskUpdate(latest, update) : undefined; - if (updated) { - queryClient.setQueryData(key, updated); + + // An unknown-task message can invalidate the list while cancellation settles. A cache write clears that flag, + // so preserve the required reconciliation even if this update finishes the last task we already know about. + const needsReconciliation = wasInvalidated || queryClient.getQueryState(key)?.isInvalidated; + queryClient.setQueryData(key, update); + if (needsReconciliation) { + void queryClient.invalidateQueries( + { + queryKey: key + }, + { + cancelRefetch: false + } + ); } } async function upsertTask(queryClient: QueryClient, task: ViewBackgroundTask) { const key = queryKeys.organization(task.organization_id); - // An older list response must not remove newly queued work and stop polling. Keep any pushed updates received - // since that read started instead of reverting the cache to its pre-fetch snapshot. - await queryClient.cancelQueries( - { - queryKey: key - }, - { - revert: false - } - ); - queryClient.setQueryData(key, (current) => { + await updateCachedTasks(queryClient, key, (current) => { const existing = current?.find((item) => item.id === task.id); if (existing && compareTaskUpdateTimes(existing.updated_utc, task.updated_utc) > 0) { return current; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts index 8ae658780b..20600f5ea2 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts @@ -1,4 +1,4 @@ -import { QueryClient } from '@tanstack/svelte-query'; +import { QueryClient, QueryObserver } from '@tanstack/svelte-query'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from './models'; @@ -60,35 +60,38 @@ function createUpdate(id: string) { } describe('updateTaskFromMessage', () => { - const queryClient = { cancelQueries: vi.fn().mockResolvedValue(undefined), getQueryData: vi.fn(), invalidateQueries: vi.fn(), setQueryData: vi.fn() }; + let queryClient: QueryClient; beforeEach(() => { vi.clearAllMocks(); + queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + vi.spyOn(queryClient, 'invalidateQueries'); }); + afterEach(() => queryClient.clear()); + it('writes the progress of a known task into the cached list', async () => { // Arrange - queryClient.getQueryData.mockReturnValue([createTask()]); + queryClient.setQueryData(queryKeys.organization('organization-id'), [createTask()]); // Act - await updateTaskFromMessage(queryClient as never, createUpdate('task-1')); + await updateTaskFromMessage(queryClient, createUpdate('task-1')); // Assert - expect(queryClient.setQueryData).toHaveBeenCalledWith(queryKeys.organization('organization-id'), [ - expect.objectContaining({ id: 'task-1', processed: 75 }) - ]); + expect(queryClient.getQueryData(queryKeys.organization('organization-id'))).toMatchObject([{ id: 'task-1', processed: 75 }]); expect(queryClient.invalidateQueries).not.toHaveBeenCalled(); }); it('reloads the list when the task was started somewhere else', async () => { // Arrange - queryClient.getQueryData.mockReturnValue([createTask()]); + const tasks = [createTask()]; + queryClient.setQueryData(queryKeys.organization('organization-id'), tasks); // Act - await updateTaskFromMessage(queryClient as never, createUpdate('started-elsewhere')); + await updateTaskFromMessage(queryClient, createUpdate('started-elsewhere')); // Assert - expect(queryClient.setQueryData).not.toHaveBeenCalled(); + expect(queryClient.getQueryData(queryKeys.organization('organization-id'))).toEqual(tasks); expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: queryKeys.organization('organization-id') }); }); }); @@ -188,4 +191,66 @@ describe('task cache lifecycle', () => { // Assert expect(client.getQueryData(key)?.map((task) => task.processed)).toEqual([75, 75]); }); + + it.each(['pushed completion', 'cancellation response'] as const)( + 'finishes discovering an unknown task after a %s cancels the list request', + async (source) => { + // Arrange + const running = createTask(); + const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: '2026-01-01T10:00:09Z' }); + const unknown = createTask({ id: 'started-elsewhere' }); + client.setQueryData(key, [running]); + const previousRead = Promise.withResolvers(); + const nextRead = Promise.withResolvers(); + const queryFn = vi.fn().mockReturnValueOnce(previousRead.promise).mockReturnValueOnce(nextRead.promise); + const observer = new QueryObserver(client, { queryFn, queryKey: key, staleTime: Infinity }); + const unsubscribe = observer.subscribe(() => {}); + + // Act + await updateTaskFromMessage(client, createUpdate(unknown.id)); + if (source === 'pushed completion') { + await updateTaskFromMessage(client, { ...completed, message: undefined, project_id: undefined }); + } else { + cancelTask(); + await mocks.createMutation.mock.calls[0]![0]().onSuccess(completed); + } + + // Assert + expect(client.getQueryData(key)).toMatchObject([completed]); + expect(queryFn).toHaveBeenCalledTimes(2); + + // Act + previousRead.resolve([running, unknown]); + nextRead.resolve([completed, unknown]); + await client.getQueryCache().find({ queryKey: key })?.promise; + unsubscribe(); + + // Assert + expect(client.getQueryData(key)).toEqual([completed, unknown]); + } + ); + + it('keeps unknown-task reconciliation pending when the panel unmounts before the list is merged', async () => { + // Arrange + const running = createTask(); + const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: '2026-01-01T10:00:09Z' }); + client.setQueryData(key, [running]); + const pending = Promise.withResolvers(); + const queryFn = vi.fn(({ signal }: { signal: AbortSignal }) => { + void signal; + return pending.promise; + }); + const observer = new QueryObserver(client, { queryFn, queryKey: key, staleTime: Infinity }); + const unsubscribe = observer.subscribe(() => {}); + + // Act + await updateTaskFromMessage(client, createUpdate('started-elsewhere')); + unsubscribe(); + await updateTaskFromMessage(client, { ...completed, message: undefined, project_id: undefined }); + + // Assert + expect(client.getQueryData(key)).toMatchObject([completed]); + expect(client.getQueryState(key)?.isInvalidated).toBe(true); + expect(queryFn).toHaveBeenCalledOnce(); + }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte index 8ddd90fb33..a76c99c505 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte @@ -28,10 +28,11 @@ let now = $state(Date.now()); const visibleTasks = $derived(selectVisibleTasks(tasksQuery.data ?? [], dismissedIds, now)); + const hasVisibleTasks = $derived(visibleTasks.length > 0); // Finished tasks fade out of the panel on their own, so re-evaluate time while any of them is shown. $effect(() => { - if (visibleTasks.length === 0) { + if (!hasVisibleTasks) { return; } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts index 6738fa8a40..fad835b260 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts @@ -64,7 +64,10 @@ describe('TaskPanel', () => { mocks.cancelMutateAsync.mockResolvedValue(undefined); }); - afterEach(cleanup); + afterEach(() => { + cleanup(); + vi.useRealTimers(); + }); it('hydrates running tasks from the list after a reload and ignores stale finished ones', async () => { // Arrange @@ -198,4 +201,25 @@ describe('TaskPanel', () => { // Assert expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); }); + + it('expires a finished task while another task keeps reporting progress', async () => { + // Arrange + vi.useFakeTimers(); + vi.setSystemTime('2026-01-01T10:00:00Z'); + const finished = createTask({ description: 'Finished task', id: 'finished', status: BackgroundTaskStatus.Completed }); + state.set('tasks', [createTask(), finished]); + render(TaskPanel); + await tick(); + + // Act + for (let progress = 1; progress <= 25; progress++) { + await vi.advanceTimersByTimeAsync(5000); + state.set('tasks', [createTask({ processed: progress }), finished]); + await tick(); + } + + // Assert + expect(screen.queryByText('Finished task')).toBeNull(); + expect(screen.getByText('Delete 50 events matching filter')).toBeVisible(); + }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte index 7f97ccf559..55543c30da 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/+page.svelte @@ -771,13 +771,15 @@ // Gmail-style "select everything matching the filter". It belongs to the query it was made for, so a different // filter, time range, project or saved view drops it. - const allMatching = createAllMatchingSelection(() => - getEventsQueryIdentity({ - filter: eventsQueryParameters.filter, - organizationId: organization.current, - savedViewId: savedViewsState.activeSavedView?.id, - time: eventsQueryParameters.time - }) + const allMatching = createAllMatchingSelection( + () => + getEventsQueryIdentity({ + filter: eventsQueryParameters.filter, + organizationId: organization.current, + savedViewId: savedViewsState.activeSavedView?.id, + time: eventsQueryParameters.time + }), + () => table.store.state.rowSelection ); const pageRowCount = $derived(table.getRowModel().rows.length); const isPageSelected = $derived(pageRowCount > 0 && table.getIsAllRowsSelected()); @@ -879,13 +881,13 @@ } async function selectAllMatching() { - // Count again right now, so the number the user confirms is current. If the filter changes while the count is - // loading, the count belongs to the old filter and must not be bound to the new one. - const identity = allMatching.identity; + // Count again right now, so the number the user confirms is current. The result may only expand the query + // and page selection that started this request, even if the same rows are cleared and selected again. + const selection = allMatching.snapshot; const result = await matchingTotalQuery.refetch(); const count = (result.data?.meta.total ?? totalMatching) as number | undefined; if (count !== undefined) { - allMatching.select(count, identity); + allMatching.select(count, selection); } } From 7231062ffed2adba9230deb7dd495bdb3aba5e73 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 07:31:43 -0600 Subject: [PATCH 25/42] Pin bulk selections and harden task failure recovery --- .../BulkOperationWorkItemHandler.cs | 39 ++- .../BackgroundTasks/BackgroundTaskService.cs | 44 +++- .../BulkOperations/BulkOperationService.cs | 136 +++++++--- .../EventBulkSelectionResolver.cs | 241 +++++++++++++++--- .../BulkOperations/IBulkOperationHandler.cs | 12 +- .../BulkOperations/RelativeDateFreezer.cs | 3 +- .../Api/Handlers/BackgroundTaskHandler.cs | 3 + .../BulkOperationEndpointTests.Reliability.cs | 18 +- .../Handlers/BackgroundTaskHandlerTests.cs | 80 ++++++ .../BulkOperationWorkItemHandlerTests.cs | 64 ++++- .../BackgroundTaskServiceTests.cs | 109 ++++++++ .../BulkOperationServiceTests.cs | 94 +++++++ .../BulkSelectionResourceTests.cs | 146 +++++++++++ .../EventBulkSelectionResolverPlanTests.cs | 182 ++++++++++++- .../EventBulkSelectionResolverTests.cs | 174 +++++++++++-- .../RelativeDateFreezerTests.cs | 21 ++ .../Utility/BackgroundTaskTestHarness.cs | 5 +- 17 files changed, 1242 insertions(+), 129 deletions(-) create mode 100644 tests/Exceptionless.Tests/Services/BulkOperations/BulkSelectionResourceTests.cs diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs index 9f36cbe73c..7b8e0b8283 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -69,6 +69,9 @@ public override async Task HandleItemAsync(WorkItemContext context) ArgumentException.ThrowIfNullOrWhiteSpace(workItem.TaskId); BackgroundTask? task = null; + BackgroundTask? selectionTask = null; + string? pinnedSelection = null; + string? latestSelectionCursor = null; try { task = await _taskService.GetAsync(workItem.TaskId); @@ -80,6 +83,8 @@ public override async Task HandleItemAsync(WorkItemContext context) return; } + selectionTask = task; + latestSelectionCursor = task.Bulk?.Cursor; using var _ = Log.BeginScope(new ExceptionlessState().Organization(task.OrganizationId).Tag("BulkOperation")); if (!task.IsActive) { @@ -100,7 +105,6 @@ public override async Task HandleItemAsync(WorkItemContext context) return; } - string? pinnedSelection = null; if (task.Bulk!.Selection.Kind is BulkOperationSelectionKind.Filter) { pinnedSelection = await _service.GetPinnedSelectionAsync(task.Id); @@ -148,6 +152,7 @@ public override async Task HandleItemAsync(WorkItemContext context) } var page = await GetPageAsync(task, resolver, pinnedSelection, context.CancellationToken); + latestSelectionCursor = page.NextCursor ?? latestSelectionCursor; if (page.Ids.Count == 0) break; @@ -253,6 +258,38 @@ public override async Task HandleItemAsync(WorkItemContext context) Log.LogError(ex, "Bulk operation {TaskId} failed: {Message}", workItem.TaskId, ex.Message); await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The operation stopped because of an unexpected error."); } + finally + { + try + { + // FinishAsync and enqueue acknowledgement races can change the stored outcome without changing the local + // task. Only terminal/lost work releases its selection; active continuation and shutdown must retain it. + var saved = await _taskService.GetAsync(workItem.TaskId); + if (saved?.IsActive != true) + { + var finished = saved ?? selectionTask; + if (finished is null && workItem.Type is not null) + { + // The selection may outlive a lost task record. The work item still identifies its resolver. + finished = new BackgroundTask + { + Id = workItem.TaskId, + OrganizationId = workItem.OrganizationId, + Type = workItem.Type, + Bulk = new BulkOperationDetails { Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter } } + }; + } + + if (finished is not null) + await _service.ReleaseSelectionAsync(finished, pinnedSelection, latestSelectionCursor); + } + } + catch (Exception ex) + { + // A bounded snapshot lease is the fallback when the task store or search backend is unavailable. + Log.LogWarning(ex, "Could not check bulk operation {TaskId} for selection cleanup: {Message}", workItem.TaskId, ex.Message); + } + } } private async Task SaveBatchAsync(BackgroundTask task, BulkSelectionPage page, BulkBatchResult result) diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs index 588dc5a6e3..451d92c865 100644 --- a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -116,7 +116,9 @@ public async Task> GetByOrganizationAsync(string o /// The ids of the organization's tasks that have not finished. public async Task> GetActiveIdsAsync(string organizationId) { - return (await GetIndexedIdsAsync(GetActiveIndexCacheKey(organizationId))).Distinct(StringComparer.Ordinal).ToList(); + // Completion removes ids concurrently. Read one snapshot so a removal cannot shift an unread id past a page offset. + var ids = await _cache.GetListAsync(GetActiveIndexCacheKey(organizationId)); + return ids.HasValue ? ids.Value.Distinct(StringComparer.Ordinal).ToList() : []; } /// Puts a task that is running again (for example after a queue retry) back into the active index. @@ -143,8 +145,20 @@ public async Task AddAsync(BackgroundTask task) task.UpdatedUtc = utcNow; await SaveAsync(task); - await _cache.ListAddAsync(GetOrganizationIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime); - await _cache.ListAddAsync(GetActiveIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime); + try + { + if (await _cache.ListAddAsync(GetOrganizationIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime) != 1) + throw new InvalidOperationException("The background task could not be added to the organization index."); + + if (await _cache.ListAddAsync(GetActiveIndexCacheKey(task.OrganizationId), [task.Id], TaskLifetime) != 1) + throw new InvalidOperationException("The background task could not be added to the active index."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to index background task {TaskId}: {Message}", task.Id, ex.Message); + await FailToStartAsync(task.Id); + throw; + } } /// @@ -158,8 +172,8 @@ public async Task CreateAndEnqueueAsync(BackgroundTask task, } /// - /// Prepares and queues the work item of a task that was added and publishes it. When preparation or queueing fails - /// the task is failed, because nothing would ever run it, and the error is rethrown. + /// Prepares and queues the work item of a task that was added and publishes it. When preparation or queueing fails, + /// a task that is still queued is failed and the error is rethrown. Worker progress and cancellation are preserved. /// public async Task EnqueueAsync(BackgroundTask task, T workItem, Func? beforeEnqueue = null) where T : class { @@ -173,12 +187,7 @@ public async Task EnqueueAsync(BackgroundTask task, T workItem, Func? b catch (Exception ex) { _logger.LogError(ex, "Unable to queue background task {TaskId}: {Message}", task.Id, ex.Message); - var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - task.Finish(BackgroundTaskStatus.Failed, "The task could not be started. Please try again.", utcNow); - task.UpdatedUtc = utcNow; - await SaveAsync(task); - await RemoveFromActiveIndexAsync(task.OrganizationId, [task.Id]); - await PublishAsync(task); + await FailToStartAsync(task.Id); throw; } @@ -309,6 +318,19 @@ public async Task PublishAsync(BackgroundTaskUpdated message) } } + private Task FailToStartAsync(string id) + { + return UpdateAsync(id, current => + { + // Queueing may have succeeded before its acknowledgement failed, allowing a worker or user to advance the task. + if (current.Status is not BackgroundTaskStatus.Queued) + return false; + + current.Finish(BackgroundTaskStatus.Failed, "The task could not be started. Please try again.", _timeProvider.GetUtcNow().UtcDateTime); + return true; + }); + } + private async Task> GetIndexedIdsAsync(string key) { var ids = new List(); diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index a0f0236994..9984a5505f 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -6,6 +6,7 @@ using Foundatio.Caching; using Foundatio.Lock; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; namespace Exceptionless.Core.Services.BulkOperations; @@ -31,9 +32,10 @@ public class BulkOperationService private readonly IReadOnlyDictionary _handlers; private readonly IReadOnlyDictionary _resolvers; private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; public BulkOperationService(BackgroundTaskService taskService, ICacheClient cache, ILockProvider lockProvider, - IEnumerable handlers, IEnumerable resolvers, TimeProvider timeProvider) + IEnumerable handlers, IEnumerable resolvers, TimeProvider timeProvider, ILogger? logger = null) { _taskService = taskService; _cache = cache; @@ -41,6 +43,7 @@ public BulkOperationService(BackgroundTaskService taskService, ICacheClient cach _handlers = handlers.ToDictionary(h => h.Type, StringComparer.Ordinal); _resolvers = resolvers.ToDictionary(r => r.Entity); _timeProvider = timeProvider; + _logger = logger ?? NullLogger.Instance; } public static string GetPinnedSelectionCacheKey(string id) => $"{BackgroundTaskService.KeyPrefix}:{id}:selection"; @@ -59,6 +62,24 @@ public BulkOperationService(BackgroundTaskService taskService, ICacheClient cach return cached.HasValue ? cached.Value : null; } + /// Releases a rejected or terminal task's selection without changing its durable outcome. + public async Task ReleaseSelectionAsync(BackgroundTask task, string? pinnedSelection = null, string? cursor = null) + { + if (task.Bulk?.Selection.Kind is not BulkOperationSelectionKind.Filter + || GetHandler(task.Type) is not { } handler || !_resolvers.TryGetValue(handler.Entity, out var resolver)) + return; + + try + { + pinnedSelection ??= await GetPinnedSelectionAsync(task.Id); + await resolver.ReleaseAsync(task, pinnedSelection, cursor ?? task.Bulk.Cursor); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not release the selection for bulk operation {TaskId}: {Message}", task.Id, ex.Message); + } + } + /// /// Stores and enqueues the task unless it conflicts with an active destructive task on the same project. /// The check and the write happen under a per-organization lock so two requests cannot both pass it. @@ -70,55 +91,88 @@ public async Task CreateAsync(BackgroundTask task) task.CanCancel = true; - // Count and pin the selection before taking the lock, so a slow search never holds up other requests. string? pinnedSelection = null; - if (bulk.Selection.Kind is BulkOperationSelectionKind.Filter) + bool enqueueAttempted = false; + bool retainSelection = false; + try { - if (!_resolvers.TryGetValue(handler.Entity, out var resolver)) - throw new InvalidOperationException($"No selection resolver is registered for {handler.Entity}."); + // Count and pin before taking the lock, so a slow search never holds up other requests. + if (bulk.Selection.Kind is BulkOperationSelectionKind.Filter) + { + if (!_resolvers.TryGetValue(handler.Entity, out var resolver)) + throw new InvalidOperationException($"No selection resolver is registered for {handler.Entity}."); - task.CreatedUtc = _timeProvider.GetUtcNow().UtcDateTime; - var preparation = await resolver.PrepareAsync(task, CancellationToken.None); - if (!preparation.Check.IsOk) - return BulkOperationCreateResult.Rejected(preparation.Check); + task.CreatedUtc = _timeProvider.GetUtcNow().UtcDateTime; + var preparation = await resolver.PrepareAsync(task, CancellationToken.None); + pinnedSelection = preparation.PinnedState; + if (!preparation.Check.IsOk) + return BulkOperationCreateResult.Rejected(preparation.Check); - task.Total = preparation.Total; - pinnedSelection = preparation.PinnedState; - } - else - { - task.Total = bulk.Selection.Ids?.Count ?? 0; - } + task.Total = preparation.Total; + } + else + { + task.Total = bulk.Selection.Ids?.Count ?? 0; + } - task.Description = handler.Describe(task); + task.Description = handler.Describe(task); - await using var createLock = await _lockProvider.TryAcquireAsync(BackgroundTaskService.GetCreateLockResource(task.OrganizationId), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); - if (createLock is null) - return BulkOperationCreateResult.Busy(); + await using var createLock = await _lockProvider.TryAcquireAsync(BackgroundTaskService.GetCreateLockResource(task.OrganizationId), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); + if (createLock is null) + return BulkOperationCreateResult.Busy(); - if (handler.IsDestructive) - { - var conflict = await FindConflictingTaskAsync(task); - if (conflict is not null) - return BulkOperationCreateResult.Conflicting(conflict); - } + if (handler.IsDestructive) + { + var conflict = await FindConflictingTaskAsync(task); + if (conflict is not null) + return BulkOperationCreateResult.Conflicting(conflict); + } - await _taskService.AddAsync(task); - await _taskService.EnqueueAsync(task, new BulkOperationWorkItem - { - Description = task.Description, - TaskId = task.Id, - OrganizationId = task.OrganizationId, - Type = task.Type - }, async () => + await _taskService.AddAsync(task); + enqueueAttempted = true; + await _taskService.EnqueueAsync(task, new BulkOperationWorkItem + { + Description = task.Description, + TaskId = task.Id, + OrganizationId = task.OrganizationId, + Type = task.Type + }, async () => + { + // A selection that cannot be saved must fail the task just like an enqueue failure, so an unqueued task + // cannot remain active and block the project's next operation. + if (pinnedSelection is not null && !await _cache.SetAsync(GetPinnedSelectionCacheKey(task.Id), pinnedSelection, BackgroundTaskService.TaskLifetime)) + throw new InvalidOperationException("The operation's selection could not be saved."); + }); + + retainSelection = true; + return BulkOperationCreateResult.Created(task); + } + finally { - // A selection that cannot be saved must fail the task just like an enqueue failure, so an unqueued task - // cannot remain active and block the project's next operation. - if (pinnedSelection is not null && !await _cache.SetAsync(GetPinnedSelectionCacheKey(task.Id), pinnedSelection, BackgroundTaskService.TaskLifetime)) - throw new InvalidOperationException("The operation's selection could not be saved."); - }); - - return BulkOperationCreateResult.Created(task); + if (!retainSelection && pinnedSelection is not null) + { + var cleanupTask = task; + if (enqueueAttempted) + { + try + { + // An enqueue can succeed before its acknowledgement is lost. The worker's saved state wins; + // never close an accepted active task's snapshot just because the creating request failed. + var saved = await _taskService.GetAsync(task.Id); + retainSelection = saved?.IsActive == true; + cleanupTask = saved ?? task; + } + catch (Exception ex) + { + retainSelection = true; + _logger.LogWarning(ex, "Could not check bulk operation {TaskId} before releasing its selection: {Message}", task.Id, ex.Message); + } + } + + if (!retainSelection) + await ReleaseSelectionAsync(cleanupTask, pinnedSelection); + } + } } /// diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs index c88bae8695..aad44206f6 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -1,19 +1,26 @@ +using System.Globalization; using Elastic.Clients.Elasticsearch.Mapping; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Queries.Validation; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Base; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Core.Repositories.Options; using Exceptionless.Core.Repositories.Queries; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.DateTimeExtensions; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch; using Foundatio.Repositories.Elasticsearch.Extensions; -using Exceptionless.Core.Repositories.Base; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; using Foundatio.Serializer; +using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Services.BulkOperations; @@ -25,15 +32,19 @@ public class EventBulkSelectionResolver : IBulkSelectionResolver { private const string CreatedUtcField = "created_utc"; private const string PremiumSearchUpgradeMessage = "Please upgrade your plan to use premium search features."; + private const string SnapshotUnavailableMessage = "The saved event selection expired or is unavailable. Please start the operation again."; private readonly IEventRepository _eventRepository; private readonly IOrganizationRepository _organizationRepository; private readonly IProjectRepository _projectRepository; private readonly PersistentEventQueryValidator _validator; private readonly ITextSerializer _serializer; private readonly ExceptionlessElasticConfiguration _configuration; + private readonly AppOptions _appOptions; + private readonly BulkOperationOptions _options; + private readonly ILogger _logger; public EventBulkSelectionResolver(IEventRepository eventRepository, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, - PersistentEventQueryValidator validator, ITextSerializer serializer, ExceptionlessElasticConfiguration configuration) + PersistentEventQueryValidator validator, ITextSerializer serializer, ExceptionlessElasticConfiguration configuration, AppOptions appOptions, BulkOperationOptions options) { _eventRepository = eventRepository; _organizationRepository = organizationRepository; @@ -41,6 +52,9 @@ public EventBulkSelectionResolver(IEventRepository eventRepository, IOrganizatio _validator = validator; _serializer = serializer; _configuration = configuration; + _appOptions = appOptions; + _options = options; + _logger = configuration.LoggerFactory.CreateLogger(); } private ValueTask IsDateFieldAsync(string field) @@ -69,52 +83,105 @@ public async Task PrepareAsync(BackgroundTask task, Ca return BulkSelectionPreparation.Rejected(check); if (appFilter is null) - return new BulkSelectionPreparation(BulkOperationCheckResult.Ok); + return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, 0, + _serializer.SerializeToString(new PinnedEventSelection { Version = 1, Ids = [] })); // Counting and paging must use the same relative-date anchor, even when preparation runs after the selected day. string? frozenFilter = await RelativeDateFreezer.FreezeAsync(task.Bulk!.Selection.Filter, task.Bulk.Selection.UtcCreatedBefore ?? task.CreatedUtc, IsDateFieldAsync); - // The events list resolves stack level terms (for example status:open) and removes deleted stacks by looking up - // matching stack ids. Do that once, here, and keep the result so every page uses the same stacks. - var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection) + var pinned = CreatePinnedSelection(appFilter); + // Resolve stack terms once and open an event snapshot in the same search that counts the selection. A created-at + // cutoff alone cannot pin mutable event fields such as the value of an active session or its user description. + var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection, pinned) .FilterExpression(frozenFilter) + .Include(e => e.Id) + .SortExpression($"-{EventIndex.Alias.Date}") .EnforceEventStackFilter(); - CountResult count; + string? pointInTimeId = null; + bool retainSnapshot = false; try { - count = await _eventRepository.CountAsync(query); + cancellationToken.ThrowIfCancellationRequested(); + var results = await _eventRepository.FindAsync(query, SnapshotOptions(Math.Max(1, _options.BatchSize)).TrackTotalHits()); + pointInTimeId = results.GetPointInTimeId(); + cancellationToken.ThrowIfCancellationRequested(); + if (String.IsNullOrEmpty(pointInTimeId)) + return BulkSelectionPreparation.Rejected(SnapshotUnavailable()); + + pinned.EventFilter = query.GetFilterExpression(); + pinned.ExcludedStackIds = query.GetExcludedStacks().ToArray(); + pinned.StackIds = query.GetStacks().ToArray(); + pinned.Total = results.Total; + + string[] ids = GetIds(results); + if (results.Total <= ids.Length) + { + // The repository may already have closed a snapshot that fits on one page. Keep its complete, bounded + // membership instead of retaining a server resource for a small operation. + pinned.Ids = ids; + } + else + { + pinned.PointInTimeId = pointInTimeId; + } + + string state = _serializer.SerializeToString(pinned); + retainSnapshot = pinned.PointInTimeId is not null; + return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, results.Total, state); } catch (DocumentLimitExceededException ex) { return BulkSelectionPreparation.Rejected(BulkOperationCheckResult.Invalid("selection.filter", ex.Message)); } - - var pinned = new PinnedEventSelection + finally { - EventFilter = query.GetFilterExpression(), - ExcludedStackIds = query.GetExcludedStacks().ToArray(), - StackIds = query.GetStacks().ToArray() - }; - - return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, count.Total, _serializer.SerializeToString(pinned)); + if (!retainSnapshot) + await CloseSnapshotAsync(pointInTimeId, task.Id); + } } public async Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) { + var pinned = ReadPinnedSelection(pinnedState); + var position = String.IsNullOrEmpty(cursor) ? new SelectionCursor() : _serializer.Deserialize(cursor) + ?? throw new BulkSelectionRejectedException(SnapshotUnavailable()); + if (position.Position < 0) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + // A final checkpoint can survive a host restart before completion is recorded. There is no remaining action to + // authorize in that case, and a later plan change must not turn already completed work into a failed task. + if (position.Exhausted || pinned.Ids is { Length: 0 }) + return new BulkSelectionPage([], null); + var (appFilter, check) = await GetAppFilterAsync(task); if (!check.IsOk) throw new BulkSelectionRejectedException(check); - if (appFilter is null || pinnedState is null) + if (appFilter is null) return new BulkSelectionPage([], null); - var pinned = _serializer.Deserialize(pinnedState) ?? new PinnedEventSelection(); - var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection) - .FilterExpression(pinned.EventFilter) - .Include(e => e.Id) - .SortExpression($"-{EventIndex.Alias.Date}"); + cancellationToken.ThrowIfCancellationRequested(); + size = Math.Max(1, size); + var query = ApplyScope(new RepositoryQuery(), appFilter, task.Bulk!.Selection, pinned).Include(e => e.Id); + + if (pinned.Ids is not null) + { + // Recheck current access policy, but never re-evaluate mutable user predicates. Search every remaining + // bounded id so a missing or newly excluded prefix does not hide eligible ids on a later page. + string[] remaining = pinned.Ids.Skip((int)Math.Min(position.Position, pinned.Ids.Length)).ToArray(); + if (remaining.Length == 0) + return new BulkSelectionPage([], null); + + var available = await _eventRepository.FindAsync(query.Id(remaining), new CommandOptions().PageLimit(remaining.Length).TrackTotalHits(false)); + var allowed = GetIds(available).ToHashSet(StringComparer.Ordinal); + string[] ids = remaining.Where(allowed.Contains).Take(size).ToArray(); + int next = ids.Length == 0 ? pinned.Ids.Length : Array.IndexOf(pinned.Ids, ids[^1]) + 1; + return new BulkSelectionPage(ids, _serializer.SerializeToString(new SelectionCursor { Position = next, Exhausted = next >= pinned.Ids.Length })); + } + + query.FilterExpression(pinned.EventFilter).SortExpression($"-{EventIndex.Alias.Date}"); if (pinned.StackIds.Length > 0) query.Stack(pinned.StackIds); @@ -122,18 +189,120 @@ public async Task GetPageAsync(BackgroundTask task, string? p if (pinned.ExcludedStackIds.Length > 0) query.ExcludeStack(pinned.ExcludedStackIds); - // Paging only needs the next ids; the total was counted once when the task was created. - var options = new CommandOptions().SearchAfterToken(cursor, _serializer).PageLimit(size).TrackTotalHits(false); + // Always supply the saved snapshot id. Missing or expired state must never open a new live selection. + string pointInTimeId = position.PointInTimeId ?? pinned.PointInTimeId!; + if (String.IsNullOrEmpty(pointInTimeId) || (position.Position > 0 && String.IsNullOrEmpty(position.After))) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + var options = SnapshotOptions(size).PointInTimeId(pointInTimeId).SearchAfterToken(position.After, _serializer).TrackTotalHits(false); var results = await _eventRepository.FindAsync(query, options); + // Foundatio represents a missing PIT (404) as an empty result. A valid PIT response includes its id even when + // current retention policy has narrowed the selection to no matches. + string? nextPointInTimeId = results.GetPointInTimeId(); + if (String.IsNullOrEmpty(nextPointInTimeId)) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + string[] pageIds = GetIds(results); + long nextPosition = position.Position + pageIds.Length; + string? after = results.Hits.LastOrDefault()?.GetSortToken(_serializer); + bool exhausted = !results.HasMore || nextPosition >= pinned.Total; + if (!exhausted && String.IsNullOrEmpty(after)) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + // Retain the latest PIT id even on the last page, so terminal cleanup and a restart after the final checkpoint + // can close the correct resource without replaying the selection. + return new BulkSelectionPage(pageIds, _serializer.SerializeToString(new SelectionCursor + { + After = after, + PointInTimeId = nextPointInTimeId, + Position = nextPosition, + Exhausted = exhausted + })); + } + + public async Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor) + { + if (pinnedState is null) + return; + + try + { + var pinned = _serializer.Deserialize(pinnedState); + if (String.IsNullOrEmpty(pinned?.PointInTimeId)) + return; + + var position = String.IsNullOrEmpty(cursor) ? null : _serializer.Deserialize(cursor); + await CloseSnapshotAsync(position?.PointInTimeId ?? pinned.PointInTimeId, task.Id); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Could not release the event selection for task {TaskId}: {Message}", task.Id, ex.Message); + } + } - string[] ids = results.Hits.Select(h => h.Id).Where(id => !String.IsNullOrEmpty(id)).Select(id => id!).ToArray(); - string? nextCursor = results.Hits.LastOrDefault()?.GetSortToken(_serializer); - return new BulkSelectionPage(ids, ids.Length > 0 ? nextCursor : null); + private async Task CloseSnapshotAsync(string? pointInTimeId, string taskId) + { + if (String.IsNullOrEmpty(pointInTimeId) || _eventRepository is not ISupportPointInTime repository) + return; + + try + { + await repository.ClosePointInTimeAsync(pointInTimeId); + } + catch (Exception ex) + { + // Expiry is the bounded fallback if best-effort cleanup cannot reach Elasticsearch. + _logger.LogWarning(ex, "Could not close the event snapshot for task {TaskId}: {Message}", taskId, ex.Message); + } } - private static RepositoryQuery ApplyScope(RepositoryQuery query, AppFilter appFilter, BulkOperationSelection selection) + private static CommandOptions SnapshotOptions(int size) => new CommandOptions() + .SearchAfterPaging(SearchAfterPagingMode.PointInTime) + // A queued task is stale after this idle period. Each page renews the same lease for active work and retries. + .SnapshotPagingLifetime(BackgroundTaskService.StaleAfter) + .PageLimit(size); + + private static string[] GetIds(FindResults results) => results.Hits + .Select(hit => hit.Id ?? throw new BulkSelectionRejectedException(SnapshotUnavailable())).ToArray(); + + private static BulkOperationCheckResult SnapshotUnavailable() => BulkOperationCheckResult.Invalid("selection", SnapshotUnavailableMessage); + + private PinnedEventSelection ReadPinnedSelection(string? state) + { + var pinned = state is null ? null : _serializer.Deserialize(state); + // Records created before snapshots were introduced contain only filters and stack ids. Replaying them would + // broaden the confirmed membership, so those tasks must be restarted instead of silently upgrading in place. + if (pinned is null || pinned.Version != 1 || (pinned.Ids is null && String.IsNullOrEmpty(pinned.PointInTimeId))) + throw new BulkSelectionRejectedException(SnapshotUnavailable()); + + return pinned; + } + + private PinnedEventSelection CreatePinnedSelection(AppFilter appFilter) + { + var organization = appFilter.Organizations.Single(); + var project = appFilter.Projects?.SingleOrDefault(); + var timeProvider = _configuration.TimeProvider; + var utcNow = timeProvider.GetUtcNow().UtcDateTime; + var retentionDate = organization.GetRetentionUtcCutoff(_appOptions.MaximumRetentionDays, + project?.CreatedUtc.SafeSubtract(TimeSpan.FromDays(3)), timeProvider); + // Keep AppFilterQueryBuilder's date-math rounding, including the inclusive upper day's rounding. Pinning only + // the submitted time range could admit older events after a retention upgrade or project-scope change. + double retentionDays = Math.Max(Math.Round(Math.Abs(utcNow.Subtract(retentionDate).TotalDays), MidpointRounding.AwayFromZero), 1); + string anchor = utcNow.ToString("yyyy-MM-ddTHH:mm:ss.fff'Z'", CultureInfo.InvariantCulture) + "||"; + return new PinnedEventSelection + { + Version = 1, + RetentionStart = FormattableString.Invariant($"{anchor}/d-{(int)retentionDays}d"), + RetentionEnd = $"{anchor}/d+1d" + }; + } + + private static RepositoryQuery ApplyScope(RepositoryQuery query, AppFilter appFilter, BulkOperationSelection selection, PinnedEventSelection pinned) { query.AppFilter(appFilter); + if (pinned.RetentionStart is not null && pinned.RetentionEnd is not null) + query.ElasticFilter(new DateRangeQuery { Field = EventIndex.Alias.Date, Gte = pinned.RetentionStart, Lte = pinned.RetentionEnd }); if (selection.UtcStart.HasValue || selection.UtcEnd.HasValue) { @@ -178,11 +347,25 @@ private static RepositoryQuery ApplyScope(RepositoryQueryWhat the stack level filter resolved to when the task was created. + /// A bounded id set or the query and snapshot that fix the originally confirmed membership. private sealed class PinnedEventSelection { + public int Version { get; set; } + public string[]? Ids { get; set; } + public string? PointInTimeId { get; set; } + public long Total { get; set; } + public string? RetentionStart { get; set; } + public string? RetentionEnd { get; set; } public string? EventFilter { get; set; } public string[] ExcludedStackIds { get; set; } = []; public string[] StackIds { get; set; } = []; } + + private sealed class SelectionCursor + { + public string? After { get; set; } + public string? PointInTimeId { get; set; } + public long Position { get; set; } + public bool Exhausted { get; set; } + } } diff --git a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs index 92f19151df..0cfd1f64eb 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs @@ -49,8 +49,8 @@ public interface IBulkSelectionResolver /// /// Runs once when the task is created. Counts what matches and pins every part of the selection that could - /// otherwise change while the task runs (for events, the stack level filter), so the selection stays exactly - /// what the user confirmed and pages stay cheap. The pinned state is stored with the task for its lifetime. + /// otherwise change while the task runs, so pages never expand beyond what the user confirmed. Current access + /// policy may still narrow that membership. The pinned state is stored with the task for its lifetime. /// Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken); @@ -59,6 +59,12 @@ public interface IBulkSelectionResolver /// paging stays correct while earlier items are being deleted or changed. Paging must not count totals. /// Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken); + + /// + /// Best-effort release of selection resources when creation is rejected or a task is terminal. Active retries and + /// host shutdown keep their resources so they can resume. Implementations must tolerate duplicate cleanup. + /// + Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor) => Task.CompletedTask; } public record BulkSelectionPreparation(BulkOperationCheckResult Check, long Total = 0, string? PinnedState = null) @@ -68,7 +74,7 @@ public record BulkSelectionPreparation(BulkOperationCheckResult Check, long Tota public record BulkSelectionPage(IReadOnlyList Ids, string? NextCursor); -/// A previously accepted selection can no longer run under the organization's current plan. +/// A previously accepted selection can no longer run with its saved membership or current access policy. public sealed class BulkSelectionRejectedException(BulkOperationCheckResult check) : InvalidOperationException(check.Message) { public BulkOperationCheckResult Check { get; } = check; diff --git a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs index 7235c6d1cb..efe88962de 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/RelativeDateFreezer.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Exceptionless.Core.Repositories.Queries; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Nodes; @@ -31,7 +32,7 @@ public static class RelativeDateFreezer private sealed class FreezeRelativeDatesVisitor(DateTime utcNow, Func> isDateFieldAsync) : ChainableQueryVisitor { // Elasticsearch date math anchors an expression on a date with ||, for example 2026-11-03T04:00:00.000Z||/d-4d. - private readonly string _anchor = $"{utcNow.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.fffZ}||"; + private readonly string _anchor = utcNow.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ss.fffZ", CultureInfo.InvariantCulture) + "||"; public override async Task VisitAsync(GroupNode node, IQueryVisitorContext context) { diff --git a/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs index aae45dc272..e08c7d7b79 100644 --- a/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs @@ -153,6 +153,9 @@ public async Task>> Handle(CancelBac if (cancelled.IsNotCancellable) return Result.Invalid(ValidationError.Create("task", "This task cannot be cancelled.")); + if (!cancelled.Task.IsActive) + await bulkOperationService.ReleaseSelectionAsync(cancelled.Task); + logger.LogInformation("User {User} requested cancellation of task {TaskId}", HttpContext.Request.GetUser().Id, cancelled.Task.Id); return new AcceptedResponse(MapToViewModel(cancelled.Task)); } diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs index cc862adfb6..93ea1a0782 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.Reliability.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using Exceptionless.Core; using Exceptionless.Core.Messaging.Models; using Exceptionless.Core.Models; using Exceptionless.Core.Models.BackgroundTasks; @@ -16,6 +17,7 @@ using Foundatio.Messaging; using Foundatio.Queues; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; using Foundatio.Repositories.Utility; @@ -39,11 +41,11 @@ public async Task GetPageAsync_Paging_DoesNotCountTotalHits() var repository = new Mock(); repository.Setup(r => r.FindAsync(It.IsAny(), It.IsAny())) .Callback((_, options) => capturedOptions = options) - .ReturnsAsync(new FindResults()); + .ReturnsAsync(new FindResults(data: new Dictionary { [ElasticDataKeys.PointInTimeId] = "snapshot" })); var resolver = CreateResolver(repository.Object); // Act - await resolver.GetPageAsync(NewRecordingOperation("test.record"), "{}", null, 10, TestCancellationToken); + await resolver.GetPageAsync(NewRecordingOperation("test.record"), "{\"version\":1,\"point_in_time_id\":\"snapshot\",\"total\":1}", null, 10, TestCancellationToken); // Assert Assert.NotNull(capturedOptions); @@ -56,7 +58,7 @@ public async Task Create_FilterMatchingTooManyStacks_IsRejectedWithTheRealMessag // Arrange const string message = "Event filter resulted in too many matching stacks (30000 of 20000 max). Please limit your search criteria."; var repository = new Mock(); - repository.Setup(r => r.CountAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new DocumentLimitExceededException(message)); + repository.Setup(r => r.FindAsync(It.IsAny(), It.IsAny())).ThrowsAsync(new DocumentLimitExceededException(message)); var service = CreateService([GetService()], [CreateResolver(repository.Object)]); // Act @@ -451,8 +453,14 @@ public async Task Cancel_FinishedOperation_ChangesNothing() private EventBulkSelectionResolver CreateResolver(IEventRepository eventRepository) { - return new EventBulkSelectionResolver(eventRepository, GetService(), GetService(), - GetService(), GetService(), GetService()); + var organizations = GetService(); + var projects = GetService(); + var validator = GetService(); + var serializer = GetService(); + var configuration = GetService(); + var appOptions = GetService(); + var options = GetService(); + return new EventBulkSelectionResolver(eventRepository, organizations, projects, validator, serializer, configuration, appOptions, options); } private BackgroundTask NewIdsOperation(string type) diff --git a/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs b/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs index c995a13179..6e8ee4997c 100644 --- a/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs +++ b/tests/Exceptionless.Tests/Api/Handlers/BackgroundTaskHandlerTests.cs @@ -1,6 +1,22 @@ using System.Globalization; +using System.Security.Claims; +using Exceptionless.Core; +using Exceptionless.Core.Extensions; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Tests.Utility; using Exceptionless.Web.Api.Handlers; +using Exceptionless.Web.Api.Messages; +using Exceptionless.Web.Extensions; using Foundatio.Xunit; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using Moq; using Xunit; namespace Exceptionless.Tests.Api.Handlers; @@ -11,6 +27,70 @@ public class BackgroundTaskHandlerTests : TestWithLoggingBase public BackgroundTaskHandlerTests(ITestOutputHelper output) : base(output) { } + [Theory] + [InlineData(false, BackgroundTaskStatus.Cancelled, 1)] + [InlineData(true, BackgroundTaskStatus.Running, 0)] + public async Task Handle_CancelBulkTask_ReleasesSelectionOnlyAfterItBecomesTerminal(bool running, BackgroundTaskStatus expectedStatus, int expectedReleases) + { + // Arrange + const string organizationId = "537650f3b77efe23a47914f4"; + const string pinnedSelection = "saved-event-selection"; + var time = new FakeTimeProvider(); + using var harness = new BackgroundTaskTestHarness(time, Log); + var user = new User + { + Id = "537650f3b77efe23a47914f8", + EmailAddress = "user@example.test", + OrganizationIds = new HashSet { organizationId } + }; + var task = new BackgroundTask + { + OrganizationId = organizationId, + Type = DeleteEventsBulkOperationHandler.OperationType, + Description = "Delete matching events", + CreatedByUserId = user.Id, + CanCancel = true, + Bulk = new BulkOperationDetails { Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter } } + }; + await harness.Tasks.AddAsync(task); + await harness.Cache.SetAsync(BulkOperationService.GetPinnedSelectionCacheKey(task.Id), pinnedSelection, BackgroundTaskService.TaskLifetime); + if (running) + { + await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Start(time.GetUtcNow().UtcDateTime); + return true; + }); + } + + var operation = new Mock(MockBehavior.Strict); + operation.SetupGet(handler => handler.Type).Returns(task.Type); + operation.SetupGet(handler => handler.Entity).Returns(BulkOperationEntity.Events); + var resolver = new Mock(MockBehavior.Strict); + resolver.SetupGet(selection => selection.Entity).Returns(BulkOperationEntity.Events); + resolver.Setup(selection => selection.ReleaseAsync(It.IsAny(), pinnedSelection, null)).Returns(Task.CompletedTask); + var bulkService = new BulkOperationService(harness.Tasks, harness.Cache, harness.LockProvider, [operation.Object], [resolver.Object], time); + var context = new DefaultHttpContext { User = new ClaimsPrincipal(user.ToIdentity()) }; + context.Request.SetUser(user); + var organizations = new Mock(MockBehavior.Strict); + var projects = new Mock(MockBehavior.Strict); + var handler = new BackgroundTaskHandler(harness.Tasks, bulkService, [resolver.Object], organizations.Object, projects.Object, + new AppOptions(), time, new HttpContextAccessor { HttpContext = context }, Log.CreateLogger()); + + // Act + var result = await handler.Handle(new CancelBackgroundTask(task.Id)); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.True(result.IsSuccess); + Assert.Equal(expectedStatus, result.Value.Value.Status); + Assert.Equal(expectedStatus, stored!.Status); + Assert.True(await harness.Tasks.IsCancelRequestedAsync(task.Id)); + resolver.Verify(selection => selection.ReleaseAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(expectedReleases)); + resolver.Verify(selection => selection.ReleaseAsync( + It.Is(current => current.Id == task.Id && current.Status == BackgroundTaskStatus.Cancelled), pinnedSelection, null), Times.Exactly(expectedReleases)); + } + [Theory] [InlineData("not a date")] [InlineData("24h")] diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs index 8ce68d537f..6c83237130 100644 --- a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs @@ -118,10 +118,14 @@ public async Task HandleItemAsync_CompletedBatchCheckpointTemporarilyFails_Prese Assert.Equal(ids.Length, stored.Succeeded); Assert.Equal(0, stored.Failed); Assert.Equal(ids, executedIds); + Assert.Equal(kind is BulkOperationSelectionKind.Filter ? 1 : 0, resolver.Releases.Count); + Assert.All(resolver.Releases, release => Assert.Equal(stored.Bulk!.Cursor, release.Cursor)); } - [Fact] - public async Task HandleItemAsync_HostStopsAfterSuccessfulBatch_PersistsItsResultBeforeStopping() + [Theory] + [InlineData(BulkOperationSelectionKind.Ids)] + [InlineData(BulkOperationSelectionKind.Filter)] + public async Task HandleItemAsync_HostStopsAfterSuccessfulBatch_PersistsItsResultBeforeStopping(BulkOperationSelectionKind kind) { // Arrange string[] ids = ["first", "second", "third"]; @@ -145,7 +149,7 @@ public async Task HandleItemAsync_HostStopsAfterSuccessfulBatch_PersistsItsResul }); var resolver = new RemainingIdsResolver(remaining); var service = CreateService(harness, operationHandler, resolver); - var created = await service.CreateAsync(NewTask(ids)); + var created = await service.CreateAsync(NewTask(ids, kind)); var handler = CreateHandler(harness, service, resolver, batchSize: 2); var interruptedContext = CreateContext(created.Task!.Id, shutdown.Token); @@ -159,6 +163,7 @@ public async Task HandleItemAsync_HostStopsAfterSuccessfulBatch_PersistsItsResul Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); Assert.Equal(2, interrupted.Processed); Assert.Equal(2, interrupted.Succeeded); + Assert.Empty(resolver.Releases); // Act await handler.HandleItemAsync(CreateContext(created.Task.Id, TestCancellationToken)); @@ -171,6 +176,7 @@ public async Task HandleItemAsync_HostStopsAfterSuccessfulBatch_PersistsItsResul Assert.Equal(0, completed.Failed); Assert.Equal(ids, executedIds); Assert.Empty(remaining); + Assert.Equal(kind is BulkOperationSelectionKind.Filter ? 1 : 0, resolver.Releases.Count); } [Theory] @@ -351,9 +357,9 @@ public async Task HandleItemAsync_HostStopsBeforeContinuation_KeepsTheTaskAvaila shutdown.Cancel(); return Task.FromException(new TimeoutException("Search interrupted during shutdown")); }); - var resolver = new RemainingIdsResolver([]); + var resolver = new RemainingIdsResolver(["first"]); var service = CreateService(harness, operationHandler, resolver); - var created = await service.CreateAsync(NewTask(["first"])); + var created = await service.CreateAsync(NewTask(["first"], BulkOperationSelectionKind.Filter)); var handler = CreateHandler(harness, service, resolver, batchSize: 1, retryDelay: TimeSpan.FromSeconds(5)); var context = CreateContext(created.Task!.Id, shutdown.Token); @@ -371,6 +377,7 @@ public async Task HandleItemAsync_HostStopsBeforeContinuation_KeepsTheTaskAvaila Assert.Equal(1, stored.Bulk!.RetryCount); Assert.Contains(stored.Id, await harness.Tasks.GetActiveIdsAsync(OrganizationId)); Assert.Equal(1, stats.Enqueued); + Assert.Empty(resolver.Releases); } [Theory] @@ -410,6 +417,46 @@ public async Task HandleItemAsync_SelectionIsRejected_FailsWithoutLosingComplete Assert.Equal(1, stats.Enqueued); Assert.Equal(1, stats.Completed); Assert.Empty(await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + var released = Assert.Single(resolver.Releases); + Assert.Equal("{}", released.PinnedState); + Assert.Equal(stored.Bulk.Cursor, released.Cursor); + } + + [Fact] + public async Task HandleItemAsync_TransientFailure_KeepsTheSelectionUntilContinuationCompletes() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var remaining = new HashSet(["first"], StringComparer.Ordinal); + int attempts = 0; + var operationHandler = new DelegateBulkHandler(batch => ++attempts == 1 + ? Task.FromException(new TimeoutException("Search temporarily unavailable")) + : Task.FromResult(new BulkBatchResult(batch.Count(remaining.Remove), 0))); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first"], BulkOperationSelectionKind.Filter)); + var handler = CreateHandler(harness, service, resolver, batchSize: 1); + + // Act + await handler.HandleItemAsync(CreateContext(created.Task!.Id, TestCancellationToken)); + var interrupted = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(interrupted); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); + Assert.Equal(1, interrupted.Bulk!.RetryCount); + Assert.Empty(resolver.Releases); + Assert.Equal(2, (await harness.Queue.GetQueueStatsAsync()).Enqueued); + + // Act + await handler.HandleItemAsync(CreateContext(created.Task.Id, TestCancellationToken)); + var completed = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(completed); + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(1, completed.Succeeded); + Assert.Single(resolver.Releases); } private BulkOperationService CreateService(BackgroundTaskTestHarness harness, IBulkOperationHandler handler, IBulkSelectionResolver resolver) @@ -470,6 +517,7 @@ private sealed class RemainingIdsResolver(HashSet remaining) : IBulkSele private int _calls; public Func? Failure { get; init; } + public List<(string? PinnedState, string? Cursor)> Releases { get; } = []; public BulkOperationEntity Entity => BulkOperationEntity.Events; public Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection) => Task.FromResult(BulkOperationCheckResult.Ok); public Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken) => Task.FromResult(new BulkSelectionPreparation(BulkOperationCheckResult.Ok, remaining.Count, "{}")); @@ -482,6 +530,12 @@ public Task GetPageAsync(BackgroundTask task, string? pinnedS string[] ids = remaining.Order(StringComparer.Ordinal).Take(size).ToArray(); return Task.FromResult(new BulkSelectionPage(ids, ids.LastOrDefault())); } + + public Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor) + { + Releases.Add((pinnedState, cursor)); + return Task.CompletedTask; + } } private sealed class FaultingEnqueueQueue(InMemoryQueueOptions options) : InMemoryQueue(options) diff --git a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs index 61fe1a441d..d12f582c86 100644 --- a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs @@ -1,6 +1,8 @@ using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Services.BackgroundTasks; +using Exceptionless.Core.Services.BulkOperations; using Exceptionless.Tests.Utility; using Foundatio.Caching; using Foundatio.Jobs; @@ -86,6 +88,35 @@ public async Task CreateAndEnqueueAsync_TaskCannotBeStored_DoesNotQueueOrPublish Assert.Empty(harness.Publisher.Updates); } + [Theory] + [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] + [InlineData(false, typeof(InvalidOperationException))] + public async Task CreateAndEnqueueAsync_IndexCannotBeStored_DoesNotQueueOrLeaveAnActiveTask(bool throwOnRejection, Type expectedExceptionType) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, configureCache: options => + { + options.MaxEntrySize = 1; + options.SizeCalculator = value => value is IDictionary ? 2 : 1; + options.ShouldThrowOnMaxEntrySizeExceeded = throwOnRejection; + }); + var task = NewTask(); + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CreateAndEnqueueAsync(task, + current => new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = task.ProjectId!, TaskId = current.Id })); + var stored = await harness.Tasks.GetAsync(task.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var queued = await harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.IsType(expectedExceptionType, thrown); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Empty(active); + Assert.Equal(0, queued.Enqueued); + } + [Fact] public async Task GetByOrganizationAsync_FinishedTasks_ReturnsTheNewestFirstUpToTheLimit() { @@ -585,6 +616,70 @@ public async Task EnqueueAsync_QueueRejectsTheWorkItem_FailsTheTaskSoItIsNotLeft Assert.Equal(BackgroundTaskStatus.Failed, Assert.Single(harness.Publisher.Updates).Status); } + [Theory] + [InlineData(BackgroundTaskStatus.Running, 40, 10)] + [InlineData(BackgroundTaskStatus.Completed, 100, 4)] + [InlineData(BackgroundTaskStatus.Cancelled, 40, 10)] + public async Task EnqueueAsync_AcknowledgementFailsAfterTaskAdvances_PreservesTheStoredState(BackgroundTaskStatus status, int expectedProgress, int expectedTotal) + { + // Arrange + var queue = new LostAcknowledgementQueue(new InMemoryQueueOptions { TimeProvider = _time, LoggerFactory = Log }); + using var harness = new BackgroundTaskTestHarness(_time, Log, queue: queue); + var task = NewTask(canCancel: true); + task.Type = DeleteEventsBulkOperationHandler.OperationType; + task.Total = 10; + task.Bulk = new BulkOperationDetails + { + Selection = new BulkOperationSelection + { + Kind = BulkOperationSelectionKind.Ids, + Ids = Enumerable.Range(0, 10).Select(_ => ObjectId.GenerateNewId().ToString()).ToList() + } + }; + queue.AfterEnqueue = async () => + { + await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.Progress = 40; + current.Processed = 4; + current.Succeeded = 3; + current.Failed = 1; + current.Bulk!.Cursor = "4"; + current.Message = "Processed four events"; + if (status is BackgroundTaskStatus.Completed) + current.Total = current.Processed; + + if (status is not BackgroundTaskStatus.Running) + current.Finish(status, current.Message, _time.GetUtcNow().UtcDateTime); + + return true; + }); + }; + + // Act + var thrown = await Record.ExceptionAsync(async () => await harness.Tasks.CreateAndEnqueueAsync(task, + current => new BulkOperationWorkItem { OrganizationId = OrganizationId, TaskId = current.Id, Type = current.Type })); + var stored = await harness.Tasks.GetAsync(task.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var queued = await queue.GetQueueStatsAsync(); + + // Assert + Assert.IsType(thrown); + Assert.NotNull(stored); + Assert.Equal(status, stored.Status); + Assert.Equal(expectedProgress, stored.Progress); + Assert.Equal(expectedTotal, stored.Total); + Assert.Equal(4, stored.Processed); + Assert.Equal(3, stored.Succeeded); + Assert.Equal(1, stored.Failed); + Assert.Equal("4", stored.Bulk!.Cursor); + Assert.Equal("Processed four events", stored.Message); + Assert.Equal(status is BackgroundTaskStatus.Running, active.Contains(task.Id)); + Assert.Equal(1, queued.Enqueued); + Assert.Equal(status, Assert.Single(harness.Publisher.Updates).Status); + } + [Fact] public void CacheKeysAndLockResources_ShareThePrefixAndAreNamedByKind() { @@ -622,6 +717,20 @@ private static BackgroundTask NewTask(string organizationId = OrganizationId, bo }; } + private sealed class LostAcknowledgementQueue(InMemoryQueueOptions options) : InMemoryQueue(options) + { + public Func? AfterEnqueue { get; set; } + + protected override async Task EnqueueImplAsync(WorkItemData data, QueueEntryOptions options) + { + await base.EnqueueImplAsync(data, options); + if (AfterEnqueue is { } afterEnqueue) + await afterEnqueue(); + + throw new TimeoutException("The work item was queued but its acknowledgement was lost"); + } + } + private sealed class CountingLockProvider { private int _attempts; diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs index a2fb844349..14ec5dfddc 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs @@ -92,6 +92,40 @@ public async Task CreateAsync_DestructiveTaskActiveOnTheSameProject_Conflicts() Assert.Equal(1, (await _harness.Queue.GetQueueStatsAsync()).Enqueued); } + [Fact] + public async Task CreateAsync_TaskCompletesDuringActiveIndexRead_StillFindsTheConflictingTask() + { + // Arrange + CompletingTaskOnReadCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new CompletingTaskOnReadCacheClient(inner)); + var service = CreateService(harness); + var completing = NewTask("test.safe", ProjectId); + await harness.Tasks.AddAsync(completing); + for (int index = 1; index < 500; index++) + await harness.Tasks.AddAsync(NewTask("test.safe", ProjectId)); + + var conflicting = NewTask("test.destructive", ProjectId); + await harness.Tasks.AddAsync(conflicting); + cache!.AfterRead = async () => + { + await harness.Tasks.UpdateAsync(completing.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Completed", _time.GetUtcNow().UtcDateTime); + return true; + }); + }; + + // Act + var result = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.Null(result.Task); + Assert.Equal(conflicting.Id, result.Conflict?.Id); + Assert.Equal(0, (await harness.Queue.GetQueueStatsAsync()).Enqueued); + Assert.Equal(BackgroundTaskStatus.Completed, (await harness.Tasks.GetAsync(completing.Id))!.Status); + } + [Theory] [InlineData(null, ProjectId)] [InlineData(ProjectId, null)] @@ -210,6 +244,31 @@ public async Task CreateAsync_QueueRejectsTheWorkItem_FailsTheTaskSoItDoesNotBlo Assert.Equal(BackgroundTaskStatus.Queued, retry.Task.Status); } + [Fact] + public async Task CreateAsync_ActiveIndexWriteFails_DoesNotLeaveAnUnqueuedTaskBlockingTheProject() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => new FailingActiveIndexCacheClient(inner)); + var service = CreateService(harness); + var task = NewTask("test.destructive", ProjectId); + + // Act + var thrown = await Record.ExceptionAsync(async () => await service.CreateAsync(task)); + var stored = await harness.Tasks.GetAsync(task.Id); + var activeAfterFailure = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + long enqueuedAfterFailure = (await harness.Queue.GetQueueStatsAsync()).Enqueued; + var retry = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.IsType(thrown); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.Empty(activeAfterFailure); + Assert.Equal(0, enqueuedAfterFailure); + Assert.NotNull(retry.Task); + } + [Theory] [InlineData(true, typeof(MaxEntrySizeExceededCacheException))] [InlineData(false, typeof(InvalidOperationException))] @@ -248,6 +307,7 @@ public async Task CreateAsync_SelectionCannotBeStored_FailsWithoutQueueingAndAll Assert.Empty(activeAfterFailure); Assert.Equal(0, enqueuedAfterFailure); Assert.Contains(_harness.Publisher.Updates, update => update.Id == task.Id && update.Status == BackgroundTaskStatus.Failed); + resolver.Verify(r => r.ReleaseAsync(It.Is(saved => saved.Id == task.Id && saved.Status == BackgroundTaskStatus.Failed), "{}", null), Times.Once); Assert.NotNull(retry.Task); Assert.Equal(BackgroundTaskStatus.Queued, retry.Task.Status); } @@ -306,4 +366,38 @@ private sealed class StubHandler(string type, bool isDestructive) : IBulkOperati public string Describe(BackgroundTask task) => $"Record {task.Total}"; public Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) => Task.FromResult(new BulkBatchResult(ids.Count, 0)); } + + private sealed class CompletingTaskOnReadCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + public Func? AfterRead { get; set; } + + public new async Task>> GetListAsync(string key, int? page = null, int pageSize = 100) where T : notnull + { + var result = await base.GetListAsync(key, page, pageSize); + if (key == BackgroundTaskService.GetActiveIndexCacheKey(OrganizationId) && AfterRead is { } afterRead) + { + AfterRead = null; + await afterRead(); + } + + return result; + } + } + + private sealed class FailingActiveIndexCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + private bool _failed; + + public new async Task ListAddAsync(string key, IEnumerable values, TimeSpan? expiresIn = null) where T : notnull + { + long added = await base.ListAddAsync(key, values, expiresIn); + if (!_failed && key == BackgroundTaskService.GetActiveIndexCacheKey(OrganizationId)) + { + _failed = true; + throw new TimeoutException("The active index write succeeded but its acknowledgement was lost"); + } + + return added; + } + } } diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/BulkSelectionResourceTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/BulkSelectionResourceTests.cs new file mode 100644 index 0000000000..32412088ed --- /dev/null +++ b/tests/Exceptionless.Tests/Services/BulkOperations/BulkSelectionResourceTests.cs @@ -0,0 +1,146 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; +using Exceptionless.Core.Services.BulkOperations; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Queues; +using Foundatio.Xunit; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace Exceptionless.Tests.Services.BulkOperations; + +public class BulkSelectionResourceTests(ITestOutputHelper output) : TestWithLoggingBase(output) +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private readonly FakeTimeProvider _time = new(); + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CreateAsync_ConflictingSelection_ReleasesOnlyTheRejectedSelectionWithoutChangingTheResult(bool cleanupFails) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var resolver = new RecordingResolver { CleanupFails = cleanupFails }; + var service = CreateService(harness, resolver); + var accepted = await service.CreateAsync(NewTask()); + + // Act + var rejected = await service.CreateAsync(NewTask()); + + // Assert + Assert.Null(rejected.Task); + Assert.Equal(accepted.Task!.Id, rejected.Conflict?.Id); + var release = Assert.Single(resolver.Releases); + Assert.Equal("selection-2", release.PinnedState); + Assert.Equal("selection-1", await service.GetPinnedSelectionAsync(accepted.Task.Id)); + Assert.True((await harness.Tasks.GetAsync(accepted.Task.Id))!.IsActive); + } + + [Theory] + [InlineData(BackgroundTaskStatus.Queued, BackgroundTaskStatus.Failed, true)] + [InlineData(BackgroundTaskStatus.Running, BackgroundTaskStatus.Running, false)] + [InlineData(BackgroundTaskStatus.Completed, BackgroundTaskStatus.Completed, true)] + [InlineData(BackgroundTaskStatus.Cancelled, BackgroundTaskStatus.Cancelled, true)] + public async Task CreateAsync_EnqueueAcknowledgementIsLost_UsesSavedStateToReleaseOrRetainTheSelection( + BackgroundTaskStatus acceptedStatus, BackgroundTaskStatus expectedStatus, bool shouldRelease) + { + // Arrange + var queue = new LostAcknowledgementQueue(new InMemoryQueueOptions { TimeProvider = _time, LoggerFactory = Log }); + using var harness = new BackgroundTaskTestHarness(_time, Log, queue: queue); + var resolver = new RecordingResolver(); + var service = CreateService(harness, resolver); + var task = NewTask(); + queue.AfterAccept = async () => + { + await harness.Tasks.UpdateAsync(task.Id, current => + { + if (acceptedStatus is BackgroundTaskStatus.Queued) + return false; + + current.Start(_time.GetUtcNow().UtcDateTime); + current.Bulk!.Cursor = "latest-snapshot-cursor"; + if (acceptedStatus is not BackgroundTaskStatus.Running) + current.Finish(acceptedStatus, "Finished before the enqueue acknowledgement.", _time.GetUtcNow().UtcDateTime); + + return true; + }); + }; + + // Act + var thrown = await Record.ExceptionAsync(async () => await service.CreateAsync(task)); + var saved = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.IsType(thrown); + Assert.NotNull(saved); + Assert.Equal(expectedStatus, saved.Status); + Assert.Equal(shouldRelease ? 1 : 0, resolver.Releases.Count); + Assert.All(resolver.Releases, release => + { + Assert.Equal("selection-1", release.PinnedState); + Assert.Equal(saved.Bulk!.Cursor, release.Cursor); + }); + Assert.Equal("selection-1", await service.GetPinnedSelectionAsync(task.Id)); + Assert.Equal(1, (await queue.GetQueueStatsAsync()).Enqueued); + } + + private BulkOperationService CreateService(BackgroundTaskTestHarness harness, RecordingResolver resolver) => + new(harness.Tasks, harness.Cache, harness.LockProvider, [new DeleteHandler()], [resolver], _time); + + private static BackgroundTask NewTask() => new() + { + OrganizationId = OrganizationId, + ProjectId = "537650f3b77efe23a47914f6", + Type = "test.delete", + Bulk = new BulkOperationDetails { Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Filter } } + }; + + private sealed class RecordingResolver : IBulkSelectionResolver + { + private int _prepared; + public bool CleanupFails { get; init; } + public List<(string? PinnedState, string? Cursor)> Releases { get; } = []; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public Task ValidateSelectionAsync(Organization organization, BulkOperationSelection selection) => Task.FromResult(BulkOperationCheckResult.Ok); + + public Task PrepareAsync(BackgroundTask task, CancellationToken cancellationToken) => + Task.FromResult(new BulkSelectionPreparation(BulkOperationCheckResult.Ok, 1, $"selection-{++_prepared}")); + + public Task GetPageAsync(BackgroundTask task, string? pinnedState, string? cursor, int size, CancellationToken cancellationToken) => + throw new NotSupportedException("These tests do not execute the queued selection."); + + public Task ReleaseAsync(BackgroundTask task, string? pinnedState, string? cursor) + { + Releases.Add((pinnedState, cursor)); + return CleanupFails ? Task.FromException(new TimeoutException("Snapshot cleanup unavailable")) : Task.CompletedTask; + } + } + + private sealed class DeleteHandler : IBulkOperationHandler + { + public string Type => "test.delete"; + public BulkOperationEntity Entity => BulkOperationEntity.Events; + public bool IsDestructive => true; + public Task AuthorizeAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public Task ValidateAsync(BulkOperationRequest request) => Task.FromResult(BulkOperationCheckResult.Ok); + public string Describe(BackgroundTask task) => "Delete selected events"; + public Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) => + throw new NotSupportedException("These tests do not execute the queued selection."); + } + + private sealed class LostAcknowledgementQueue(InMemoryQueueOptions options) : InMemoryQueue(options) + { + public Func? AfterAccept { get; set; } + + protected override async Task EnqueueImplAsync(WorkItemData data, QueueEntryOptions options) + { + await base.EnqueueImplAsync(data, options); + if (AfterAccept is not null) + await AfterAccept(); + throw new TimeoutException("The queue accepted the work but its acknowledgement was lost."); + } + } +} diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs index f44a94b6fd..e6ff8d1af1 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverPlanTests.cs @@ -1,13 +1,20 @@ +using System.Globalization; +using System.Text; using Elastic.Clients.Elasticsearch.Mapping; +using Exceptionless.Core; using Exceptionless.Core.Models; using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Queries.Validation; using Exceptionless.Core.Repositories; using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Repositories.Options; using Exceptionless.Core.Services.BulkOperations; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch; +using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; using Foundatio.Serializer; using Moq; using Xunit; @@ -19,16 +26,18 @@ public class EventBulkSelectionResolverPlanTests : TestWithServices private const string OrganizationId = "537650f3b77efe23a47914f4"; private readonly Organization _organization = new() { Id = OrganizationId, HasPremiumFeatures = true }; private readonly Mock _events = new(); + private readonly Mock _snapshots; + private readonly BulkOperationOptions _options = new() { BatchSize = 1 }; private readonly EventBulkSelectionResolver _resolver; public EventBulkSelectionResolverPlanTests(ITestOutputHelper output) : base(output) { var organizations = new Mock(); organizations.Setup(repository => repository.GetByIdAsync(OrganizationId, It.IsAny>())).ReturnsAsync(_organization); - _events.Setup(repository => repository.CountAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new CountResult(1)); + _snapshots = _events.As(); + _snapshots.Setup(repository => repository.ClosePointInTimeAsync(It.IsAny())).ReturnsAsync(true); _events.Setup(repository => repository.FindAsync(It.IsAny(), It.IsAny())) - .ReturnsAsync(new FindResults()); + .ReturnsAsync(Results("prepared", 1, false, "selected")); var validator = GetService(); var serializer = GetService(); var configuration = GetService(); @@ -36,11 +45,12 @@ public EventBulkSelectionResolverPlanTests(ITestOutputHelper output) : base(outp var mapping = new TypeMappingDescriptor(); configuration.Events.ConfigureIndexMapping(mapping); configuration.Events.QueryParser.Configuration!.UseMappings(() => mapping, configuration.Client.Infer); - _resolver = new EventBulkSelectionResolver(_events.Object, organizations.Object, Mock.Of(), validator, serializer, configuration); + var appOptions = GetService(); + _resolver = new EventBulkSelectionResolver(_events.Object, organizations.Object, Mock.Of(), validator, serializer, configuration, appOptions, _options); } [Fact] - public async Task PrepareAsync_PremiumFeaturesRemovedAfterValidation_RejectsBeforeCounting() + public async Task PrepareAsync_PremiumFeaturesRemovedAfterValidation_RejectsBeforeSearching() { // Arrange var task = NewTask("message:selected"); @@ -75,7 +85,8 @@ public async Task GetPageAsync_PremiumFeaturesRemovedAfterPreparation_RejectsBef var rejection = Assert.IsType(thrown); Assert.Equal(BulkOperationCheckStatus.PlanLimit, rejection.Check.Status); Assert.Equal("Please upgrade your plan to use premium search features.", rejection.Message); - _events.Verify(repository => repository.CountAsync(It.IsAny(), It.IsAny()), Times.Once); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Once); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("prepared"), Times.Once); _events.VerifyNoOtherCalls(); } @@ -93,8 +104,163 @@ public async Task GetPageAsync_PremiumFeaturesRemovedForFreeFilter_StillSearches // Assert Assert.True(preparation.Check.IsOk); Assert.Equal(1, preparation.Total); - Assert.Empty(page.Ids); - _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Once); + Assert.Equal(["selected"], page.Ids); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + + [Theory] + [InlineData(null)] + [InlineData("{}")] + [InlineData("{\"event_filter\":\"type:log\",\"stack_ids\":[\"stack\"]}")] + public async Task GetPageAsync_MissingOrLegacySelection_FailsWithoutSearchingLiveEvents(string? pinned) + { + // Arrange + var task = NewTask("type:log"); + + // Act + var thrown = await Record.ExceptionAsync(async () => await _resolver.GetPageAsync(task, pinned, null, 1, TestCancellationToken)); + + // Assert + var rejection = Assert.IsType(thrown); + Assert.Contains("start the operation again", rejection.Message); + _events.VerifyNoOtherCalls(); + } + + [Fact] + public async Task GetPageAsync_ExpiredSnapshot_FailsWithoutOpeningAnotherSnapshot() + { + // Arrange + var task = NewTask("type:log"); + _events.SetupSequence(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Results("prepared", 2, true, "first")) + // Foundatio returns an empty result without a PIT id for a 404. + .ReturnsAsync(new FindResults()); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var thrown = await Record.ExceptionAsync(async () => await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken)); + + // Assert + Assert.IsType(thrown); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.Is(options => options.GetPointInTimeId() == "prepared")), Times.Once); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task GetPageAsync_RepositoryCapsPageSize_ContinuesWithTheLatestSnapshotAndResumesAfterTheFinalCheckpoint() + { + // Arrange + var task = NewTask("message:selected"); + _events.SetupSequence(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Results("prepared", 3, true, "first")) + .ReturnsAsync(Results("renewed", 0, true, "first")) + .ReturnsAsync(Results("latest", 0, false, "second", "third")); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + // Simulate a repository returning a full capped page even though the requested size is larger. + var first = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 20000, TestCancellationToken); + var last = await _resolver.GetPageAsync(task, preparation.PinnedState, first.NextCursor, 20000, TestCancellationToken); + _organization.HasPremiumFeatures = false; + var resumed = await _resolver.GetPageAsync(task, preparation.PinnedState, last.NextCursor, 20000, TestCancellationToken); + await _resolver.ReleaseAsync(task, preparation.PinnedState, last.NextCursor); + + // Assert + Assert.Equal(["first"], first.Ids); + Assert.Equal(["second", "third"], last.Ids); + Assert.Empty(resumed.Ids); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.Is(options => options.GetPointInTimeId() == "renewed" && !options.ShouldTrackTotalHits())), Times.Once); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(3)); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("latest"), Times.Once); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("prepared"), Times.Never); + } + + [Fact] + public async Task GetPageAsync_SmallSelectionHasAnUnavailablePrefix_ReturnsLaterPinnedIdsAndClosesItsSnapshotImmediately() + { + // Arrange + _options.BatchSize = 3; + var task = NewTask("type:log"); + _events.SetupSequence(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Results("prepared", 3, true, "first", "second", "third")) + .ReturnsAsync(Results(null, 0, false, "third")); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + + // Act + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); + var exhausted = await _resolver.GetPageAsync(task, preparation.PinnedState, page.NextCursor, 1, TestCancellationToken); + + // Assert + Assert.Equal(["third"], page.Ids); + Assert.Empty(exhausted.Ids); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("prepared"), Times.Once); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + + [Fact] + public async Task GetPageAsync_RetentionIncreases_KeepsTheInitialBoundsAndCurrentPolicyWithAnInvariantDateAnchor() + { + // Arrange + var utcNow = new DateTimeOffset(2026, 11, 13, 11, 30, 0, TimeSpan.Zero); + TimeProvider.SetUtcNow(utcNow); + _organization.CreatedUtc = utcNow.UtcDateTime.AddDays(-30); + _organization.RetentionDays = 1; + var task = NewTask("type:log"); + task.Bulk!.Selection.UtcStart = utcNow.UtcDateTime.AddDays(-7); + var configuration = GetService(); + var filters = new List(); + var retentionPolicies = new List(); + _events.Setup(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .Returns((IRepositoryQuery query, ICommandOptions _) => + { + using var stream = new MemoryStream(); + configuration.Client.RequestResponseSerializer.Serialize(query.GetElasticFilters(), stream); + filters.Add(Encoding.UTF8.GetString(stream.ToArray())); + retentionPolicies.Add(query.GetAppFilter()!.Organizations.Single().RetentionDays); + return Task.FromResult(Results(filters.Count == 1 ? "prepared" : "renewed", 2, true, "selected")); + }); + var originalCulture = CultureInfo.CurrentCulture; + BulkSelectionPreparation? preparation = null; + string? cursor = null; + try + { + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("th-TH"); + + // Act + preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + _organization.RetentionDays = 30; + TimeProvider.Advance(TimeSpan.FromHours(1)); + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); + cursor = page.NextCursor; + + // Assert + Assert.Equal(["selected"], page.Ids); + Assert.Equal(2, filters.Count); + Assert.Equal(filters[0], filters[1]); + Assert.Contains("2026-11-13T11:30:00.000Z", filters[0]); + Assert.DoesNotContain("now", filters[0], StringComparison.OrdinalIgnoreCase); + Assert.Equal([1, 30], retentionPolicies); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + if (preparation is not null) + await _resolver.ReleaseAsync(task, preparation.PinnedState, cursor); + } + } + + private static FindResults Results(string? pointInTimeId, long total, bool hasMore, params string[] ids) + { + var data = new Dictionary(); + if (pointInTimeId is not null) + data[ElasticDataKeys.PointInTimeId] = pointInTimeId; + + var hits = ids.Select(id => new FindHit(id, null, 0, + data: new Dictionary { [ElasticDataKeys.Sorts] = new object[] { id } })); + var results = new FindResults(hits, total, data: data); + ((IFindResults)results).HasMore = hasMore; + return results; } private BackgroundTask NewTask(string filter) diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs index 527f95fabf..be0eb29b5e 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/EventBulkSelectionResolverTests.cs @@ -1,10 +1,14 @@ +using Exceptionless.Core; using Exceptionless.Core.Models; using Exceptionless.Core.Models.BackgroundTasks; using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Queries.Validation; using Exceptionless.Core.Services.BulkOperations; using Exceptionless.Core.Utility; using Foundatio.Repositories; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Services.BulkOperations; @@ -16,8 +20,9 @@ public class EventBulkSelectionResolverTests : IntegrationTestsBase public EventBulkSelectionResolverTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { - _resolver = GetService(); _eventRepository = GetService(); + // Exercise retained snapshots with a small fixture that crosses the actual preparation boundary. + _resolver = CreateResolver(batchSize: 2); } protected override async Task ResetDataAsync() @@ -62,16 +67,27 @@ public async Task GetPageAsync_EarlierPagesAreDeletedBetweenPages_ContinuesAfter var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); var ordered = events.OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(); - // Act - var first = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 2, TestCancellationToken); - await _eventRepository.RemoveAsync(events.Where(e => first.Ids.Contains(e.Id)).ToList(), o => o.ImmediateConsistency()); - var second = await _resolver.GetPageAsync(task, preparation.PinnedState, first.NextCursor, 2, TestCancellationToken); - var third = await _resolver.GetPageAsync(task, preparation.PinnedState, second.NextCursor, 2, TestCancellationToken); - - // Assert - Assert.Equal(ordered[..2], first.Ids.ToArray()); - Assert.Equal(ordered[2..4], second.Ids.ToArray()); - Assert.Equal(ordered[4..], third.Ids.ToArray()); + string? cursor = null; + try + { + // Act + var first = await _resolver.GetPageAsync(task, preparation.PinnedState, cursor, 2, TestCancellationToken); + cursor = first.NextCursor; + await _eventRepository.RemoveAsync(events.Where(e => first.Ids.Contains(e.Id)).ToList(), o => o.ImmediateConsistency()); + var second = await _resolver.GetPageAsync(task, preparation.PinnedState, cursor, 2, TestCancellationToken); + cursor = second.NextCursor; + var third = await _resolver.GetPageAsync(task, preparation.PinnedState, cursor, 2, TestCancellationToken); + cursor = third.NextCursor; + + // Assert + Assert.Equal(ordered[..2], first.Ids.ToArray()); + Assert.Equal(ordered[2..4], second.Ids.ToArray()); + Assert.Equal(ordered[4..], third.Ids.ToArray()); + } + finally + { + await _resolver.ReleaseAsync(task, preparation.PinnedState, cursor); + } } [Fact] @@ -118,21 +134,113 @@ public async Task GetPageAsync_StacksChangeStatusAfterPrepare_StillPagesTheEvent Assert.Equal([events.Single(e => e.Message == "open").Id], pages.SelectMany(p => p).ToArray()); } + [Theory] + [InlineData(2)] + [InlineData(10)] + public async Task GetPageAsync_SessionValuesChangeBetweenBatches_KeepsOnlyTheOriginallyMatchingEvents(int preparationSize) + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var (_, events) = await CreateDataAsync(data => + { + for (int i = 0; i < 3; i++) + data.Event().TestProject().Type(Event.KnownTypes.Session).Date(now.AddMinutes(-i - 1)).Value(90).Message($"selected {i}"); + data.Event().TestProject().Type(Event.KnownTypes.Session).Date(now.AddMinutes(-4)).Value(30).Message("not selected"); + }); + var resolver = CreateResolver(preparationSize); + var task = NewFilterTask("type:session value:[60 TO 120]"); + var preparation = await resolver.PrepareAsync(task, TestCancellationToken); + var expected = events.Where(e => e.Value == 90).OrderByDescending(e => e.Date).Select(e => e.Id).ToArray(); + string? cursor = null; + var actual = new List(); + try + { + // Act + var first = await resolver.GetPageAsync(task, preparation.PinnedState, cursor, 1, TestCancellationToken); + cursor = first.NextCursor; + actual.AddRange(first.Ids); + await _eventRepository.RemoveAsync(events.Where(e => first.Ids.Contains(e.Id)).ToList(), options => options.ImmediateConsistency()); + + var newlyMatching = events.Single(e => e.Message == "not selected"); + var noLongerMatching = events.Single(e => e.Message == "selected 1"); + // Active-session processing changes existing values without changing created_utc. + newlyMatching.UpdateSessionStart(newlyMatching.Date.UtcDateTime.AddSeconds(90)); + noLongerMatching.UpdateSessionStart(noLongerMatching.Date.UtcDateTime.AddSeconds(180)); + await _eventRepository.SaveAsync([newlyMatching, noLongerMatching], options => options.ImmediateConsistency()); + + var second = await resolver.GetPageAsync(task, preparation.PinnedState, cursor, 1, TestCancellationToken); + cursor = second.NextCursor; + actual.AddRange(second.Ids); + var third = await resolver.GetPageAsync(task, preparation.PinnedState, cursor, 1, TestCancellationToken); + cursor = third.NextCursor; + actual.AddRange(third.Ids); + var exhausted = await resolver.GetPageAsync(task, preparation.PinnedState, cursor, 1, TestCancellationToken); + + // Assert + Assert.Equal(3, preparation.Total); + Assert.Equal(90m, newlyMatching.Value); + Assert.Equal(180m, noLongerMatching.Value); + Assert.Equal(expected, actual); + Assert.Empty(exhausted.Ids); + Assert.DoesNotContain(newlyMatching.Id, actual); + Assert.Contains(noLongerMatching.Id, actual); + } + finally + { + await resolver.ReleaseAsync(task, preparation.PinnedState, cursor); + } + } + [Fact] - public async Task GetPageAsync_NothingWasPrepared_ReturnsAnEmptyPageInsteadOfSelectingEverything() + public async Task PrepareAsync_NoEventsMatch_ReturnsAnEmptyFixedSelection() { // Arrange - await CreateDataAsync(d => d.Event().TestProject()); - var task = NewFilterTask(""); + await CreateDataAsync(data => data.Event().TestProject().Type(Event.KnownTypes.Log)); + var task = NewFilterTask("type:session"); // Act - var page = await _resolver.GetPageAsync(task, null, null, 10, TestCancellationToken); + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); // Assert + Assert.True(preparation.Check.IsOk); + Assert.Equal(0, preparation.Total); + Assert.NotNull(preparation.PinnedState); Assert.Empty(page.Ids); Assert.Null(page.NextCursor); } + [Theory] + [InlineData(1)] + [InlineData(10)] + public async Task GetPageAsync_RetentionShrinksAfterPreparation_ExcludesOlderSelectedEventsAndKeepsRecentOnes(int preparationSize) + { + // Arrange + var now = TimeProvider.GetUtcNow(); + var organizations = GetService(); + var organization = await organizations.GetByIdAsync(SampleDataService.TEST_ORG_ID); + organization!.CreatedUtc = now.UtcDateTime.AddDays(-30); + organization.RetentionDays = 7; + await organizations.SaveAsync(organization, options => options.ImmediateConsistency()); + var (_, events) = await CreateDataAsync(data => + { + data.Event().TestProject().Date(now.AddHours(-1)).Message("recent"); + data.Event().TestProject().Date(now.AddDays(-4)).Message("older"); + }); + var resolver = CreateResolver(preparationSize); + var task = NewFilterTask(""); + var preparation = await resolver.PrepareAsync(task, TestCancellationToken); + + // Act + organization.RetentionDays = 1; + await organizations.SaveAsync(organization, options => options.ImmediateConsistency()); + var pages = await GetAllPagesAsync(task, preparation.PinnedState, size: 1, resolver); + + // Assert + Assert.Equal(2, preparation.Total); + Assert.Equal([events.Single(e => e.Message == "recent").Id], pages.SelectMany(page => page).ToArray()); + } + [Fact] public async Task GetPageAsync_ProjectScopedTask_OnlySelectsThatProjectsEvents() { @@ -222,21 +330,41 @@ public async Task ValidateSelectionAsync_Filter_IsCheckedForSyntaxAndPlan(string Assert.Equal("selection.filter", result.Field); } - private async Task>> GetAllPagesAsync(BackgroundTask task, string? pinnedState, int size) + private async Task>> GetAllPagesAsync(BackgroundTask task, string? pinnedState, int size, EventBulkSelectionResolver? resolver = null) { + resolver ??= _resolver; var pages = new List>(); string? cursor = null; - for (int guard = 0; guard < 20; guard++) + try { - var page = await _resolver.GetPageAsync(task, pinnedState, cursor, size, TestCancellationToken); - if (page.Ids.Count == 0) - return pages; + for (int guard = 0; guard < 20; guard++) + { + var page = await resolver.GetPageAsync(task, pinnedState, cursor, size, TestCancellationToken); + cursor = page.NextCursor ?? cursor; + if (page.Ids.Count == 0) + return pages; - pages.Add(page.Ids.ToList()); - cursor = page.NextCursor; + pages.Add(page.Ids.ToList()); + } + + throw new InvalidOperationException("Paging did not finish."); + } + finally + { + await resolver.ReleaseAsync(task, pinnedState, cursor); } + } - throw new InvalidOperationException("Paging did not finish."); + private EventBulkSelectionResolver CreateResolver(int batchSize) + { + var organizations = GetService(); + var projects = GetService(); + var validator = GetService(); + var serializer = GetService(); + var configuration = GetService(); + var appOptions = GetService(); + return new EventBulkSelectionResolver(_eventRepository, organizations, projects, validator, serializer, configuration, appOptions, + new BulkOperationOptions { BatchSize = batchSize }); } private BackgroundTask NewFilterTask(string filter, DateTime? createdBefore = null) diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs index c86d485edb..558edbd8f6 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/RelativeDateFreezerTests.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Exceptionless.Core.Services.BulkOperations; using Foundatio.Xunit; using Xunit; @@ -37,6 +38,26 @@ public async Task FreezeAsync_RelativeDatesOnDateFields_AreAnchoredToTheGivenTim Assert.Equal(expected, frozen); } + [Fact] + public async Task FreezeAsync_NonGregorianCulture_UsesAnInvariantUtcAnchor() + { + // Arrange + var originalCulture = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("th-TH"); + try + { + // Act + string? frozen = await RelativeDateFreezer.FreezeAsync("date:>now-1h", UtcNow, IsDateFieldAsync); + + // Assert + Assert.Equal("date:>\"2026-11-03T04:05:06.789Z||-1h\"", frozen); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + } + } + [Theory] [InlineData(null)] [InlineData("")] diff --git a/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs index 43a4b385ed..9cef9506b5 100644 --- a/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs +++ b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs @@ -17,7 +17,7 @@ namespace Exceptionless.Tests.Utility; public sealed class BackgroundTaskTestHarness : IDisposable { public BackgroundTaskTestHarness(TimeProvider timeProvider, ILoggerFactory loggerFactory, BackgroundTaskOptions? options = null, Func? decorateLockProvider = null, - IQueue? queue = null, int lockAttempts = 3, Action? configureCache = null) + IQueue? queue = null, int lockAttempts = 3, Action? configureCache = null, Func? decorateCache = null) { TimeProvider = timeProvider; var cacheOptions = new InMemoryCacheClientOptions { CloneValues = true, TimeProvider = timeProvider, LoggerFactory = loggerFactory }; @@ -32,7 +32,8 @@ public BackgroundTaskTestHarness(TimeProvider timeProvider, ILoggerFactory logge var policies = new ResiliencePolicyProvider(timeProvider, loggerFactory) .WithPolicy(b => b.WithMaxAttempts(lockAttempts).WithDelay(TimeSpan.FromMilliseconds(10))); - Tasks = new BackgroundTaskService(Cache, LockProvider, Publisher, Queue, policies, options ?? new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.FromMilliseconds(100) }, + var taskCache = decorateCache?.Invoke(Cache) ?? Cache; + Tasks = new BackgroundTaskService(taskCache, LockProvider, Publisher, Queue, policies, options ?? new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.FromMilliseconds(100) }, timeProvider, loggerFactory.CreateLogger()); } From aec942e6eee3340d1f5052a626d2b7fd28dc681a Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 07:31:44 -0600 Subject: [PATCH 26/42] Refresh task lists when returning to an organization --- .../src/lib/features/tasks/api.svelte.ts | 4 +- .../src/lib/features/tasks/api.test.ts | 78 ++++++++++++++++++- 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts index b0f8aa24d9..bb979ea090 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts @@ -55,7 +55,9 @@ export function getTasksQuery(request: GetTasksRequest) { return response.data ?? []; }, queryKey: queryKeys.organization(request.route.organizationId), - refetchInterval: (query) => (query.state.data?.some((task) => isTaskActive(task.status)) ? ACTIVE_TASK_POLL_INTERVAL_MS : false) + refetchInterval: (query) => (query.state.data?.some((task) => isTaskActive(task.status)) ? ACTIVE_TASK_POLL_INTERVAL_MS : false), + // Updates for other organizations are not observed. Refresh on return even when the cached list has no active tasks to poll. + staleTime: 0 })); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts index 20600f5ea2..1bf7586c8e 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts @@ -1,25 +1,37 @@ -import { QueryClient, QueryObserver } from '@tanstack/svelte-query'; +import type { ProblemDetails } from '@foundatiofx/fetchclient'; + +import { QueryClient, QueryObserver, type QueryObserverOptions } from '@tanstack/svelte-query'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from './models'; const mocks = vi.hoisted(() => ({ createMutation: vi.fn<(options: () => { onSuccess: (task: ViewBackgroundTask) => Promise | void }) => void>(), + createQuery: vi.fn<(options: () => QueryObserverOptions) => void>(), + getJSON: vi.fn(), useQueryClient: vi.fn() })); vi.mock('#features/auth/index.svelte.ts', () => ({ accessToken: { current: 'access-token' } })); vi.mock('#features/events/api.svelte.ts', () => ({ queryKeys: { type: ['PersistentEvent'] } })); vi.mock('#features/stacks/api.svelte.ts', () => ({ queryKeys: { type: ['Stack'] } })); -vi.mock('@foundatiofx/fetchclient', () => ({ useFetchClient: vi.fn() })); +vi.mock('@foundatiofx/fetchclient', () => ({ useFetchClient: () => ({ getJSON: mocks.getJSON }) })); vi.mock('@tanstack/svelte-query', async (importOriginal) => ({ ...(await importOriginal()), createMutation: mocks.createMutation, - createQuery: vi.fn(), + createQuery: mocks.createQuery, useQueryClient: mocks.useQueryClient })); -import { cancelTask, invalidateTaskAffectedQueries, postBulkOperation, queryKeys, updateTaskFromMessage } from './api.svelte'; +import { + ACTIVE_TASK_POLL_INTERVAL_MS, + cancelTask, + getTasksQuery, + invalidateTaskAffectedQueries, + postBulkOperation, + queryKeys, + updateTaskFromMessage +} from './api.svelte'; function createTask(overrides: Partial = {}): ViewBackgroundTask { return { @@ -59,6 +71,64 @@ function createUpdate(id: string) { }; } +describe('getTasksQuery', () => { + it.each([ + ['empty', BackgroundTaskStatus.Running], + ['empty', BackgroundTaskStatus.Completed], + ['finished', BackgroundTaskStatus.Running], + ['finished', BackgroundTaskStatus.Completed] + ] as const)('refreshes a cached %s list after returning to an organization with a new %s task', async (cachedList, newStatus) => { + // Arrange + vi.clearAllMocks(); + const client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: 5 * 60 * 1000 } } }); + let organizationId = 'organization-id'; + const initialTasks = cachedList === 'finished' ? [createTask({ status: BackgroundTaskStatus.Completed })] : []; + const missedTask = createTask({ id: 'started-elsewhere', status: newStatus }); + mocks.getJSON + .mockReset() + .mockResolvedValueOnce({ data: initialTasks }) + .mockResolvedValueOnce({ data: [] }) + .mockResolvedValueOnce({ data: [...initialTasks, missedTask] }); + getTasksQuery({ + route: { + get organizationId() { + return organizationId; + } + } + }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, options()); + const unsubscribe = observer.subscribe(() => {}); + + try { + await observer.getCurrentQuery().promise; + + // Act + organizationId = 'another-organization'; + observer.setOptions(options()); + await observer.getCurrentQuery().promise; + organizationId = 'organization-id'; + observer.setOptions(options()); + await observer.getCurrentQuery().promise; + + // Assert + expect(mocks.getJSON.mock.calls.map(([url]) => url)).toEqual([ + 'organizations/organization-id/tasks', + 'organizations/another-organization/tasks', + 'organizations/organization-id/tasks' + ]); + expect(observer.getCurrentResult().data).toEqual([...initialTasks, missedTask]); + const refetchInterval = options().refetchInterval; + expect(typeof refetchInterval === 'function' ? refetchInterval(observer.getCurrentQuery()) : refetchInterval).toBe( + newStatus === BackgroundTaskStatus.Running ? ACTIVE_TASK_POLL_INTERVAL_MS : false + ); + } finally { + unsubscribe(); + client.clear(); + } + }); +}); + describe('updateTaskFromMessage', () => { let queryClient: QueryClient; From de51181aa15d4f9c13cc0e47533964446b7b9c63 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 07:47:13 -0600 Subject: [PATCH 27/42] Preserve empty bulk selections through serialization --- .../EventBulkSelectionResolver.cs | 9 ++++-- .../EventBulkSelectionResolverPlanTests.cs | 30 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs index aad44206f6..c091aac15f 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/EventBulkSelectionResolver.cs @@ -84,7 +84,7 @@ public async Task PrepareAsync(BackgroundTask task, Ca if (appFilter is null) return new BulkSelectionPreparation(BulkOperationCheckResult.Ok, 0, - _serializer.SerializeToString(new PinnedEventSelection { Version = 1, Ids = [] })); + _serializer.SerializeToString(new PinnedEventSelection { Version = 1, IsEmpty = true })); // Counting and paging must use the same relative-date anchor, even when preparation runs after the selected day. string? frozenFilter = await RelativeDateFreezer.FreezeAsync(task.Bulk!.Selection.Filter, @@ -114,6 +114,7 @@ public async Task PrepareAsync(BackgroundTask task, Ca pinned.ExcludedStackIds = query.GetExcludedStacks().ToArray(); pinned.StackIds = query.GetStacks().ToArray(); pinned.Total = results.Total; + pinned.IsEmpty = results.Total == 0; string[] ids = GetIds(results); if (results.Total <= ids.Length) @@ -152,7 +153,7 @@ public async Task GetPageAsync(BackgroundTask task, string? p // A final checkpoint can survive a host restart before completion is recorded. There is no remaining action to // authorize in that case, and a later plan change must not turn already completed work into a failed task. - if (position.Exhausted || pinned.Ids is { Length: 0 }) + if (position.Exhausted || pinned.IsEmpty || pinned.Ids is { Length: 0 }) return new BulkSelectionPage([], null); var (appFilter, check) = await GetAppFilterAsync(task); @@ -272,7 +273,7 @@ private PinnedEventSelection ReadPinnedSelection(string? state) var pinned = state is null ? null : _serializer.Deserialize(state); // Records created before snapshots were introduced contain only filters and stack ids. Replaying them would // broaden the confirmed membership, so those tasks must be restarted instead of silently upgrading in place. - if (pinned is null || pinned.Version != 1 || (pinned.Ids is null && String.IsNullOrEmpty(pinned.PointInTimeId))) + if (pinned is null || pinned.Version != 1 || (!pinned.IsEmpty && pinned.Ids is null && String.IsNullOrEmpty(pinned.PointInTimeId))) throw new BulkSelectionRejectedException(SnapshotUnavailable()); return pinned; @@ -351,6 +352,8 @@ private static RepositoryQuery ApplyScope(RepositoryQuery repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(2)); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task PrepareAsync_EmptySelection_RemainsEmptyWhenStoredAndResumed(bool organizationMissing) + { + // Arrange + var task = NewTask("type:log"); + if (organizationMissing) + task.OrganizationId = "537650f3b77efe23a47914f5"; + _events.SetupSequence(repository => repository.FindAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Results("prepared", 0, false)) + .ReturnsAsync(Results("later", 1, false, "newly-matching")); + + // Act + // The resolver uses the configured application serializer for the saved state, just as queued tasks do. + var preparation = await _resolver.PrepareAsync(task, TestCancellationToken); + var page = await _resolver.GetPageAsync(task, preparation.PinnedState, null, 1, TestCancellationToken); + await _resolver.ReleaseAsync(task, preparation.PinnedState, page.NextCursor); + + // Assert + Assert.True(preparation.Check.IsOk); + Assert.Equal(0, preparation.Total); + Assert.NotNull(preparation.PinnedState); + Assert.Empty(page.Ids); + Assert.Null(page.NextCursor); + _events.Verify(repository => repository.FindAsync(It.IsAny(), It.IsAny()), Times.Exactly(organizationMissing ? 0 : 1)); + _snapshots.Verify(repository => repository.ClosePointInTimeAsync("prepared"), Times.Exactly(organizationMissing ? 0 : 1)); + } + [Theory] [InlineData(null)] [InlineData("{}")] [InlineData("{\"event_filter\":\"type:log\",\"stack_ids\":[\"stack\"]}")] + [InlineData("{\"version\":1,\"total\":0}")] public async Task GetPageAsync_MissingOrLegacySelection_FailsWithoutSearchingLiveEvents(string? pinned) { // Arrange From fc804c084506f5f78fc80b11209b49e437423587 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 08:00:29 -0600 Subject: [PATCH 28/42] Separate setup and actions in frontend tests --- .../events-select-all-banner.svelte.test.ts | 9 ++- .../tasks/components/task-item.svelte.test.ts | 76 ++++++++++++------- .../components/task-panel.svelte.test.ts | 5 +- .../src/lib/features/tasks/models.test.ts | 7 +- 4 files changed, 63 insertions(+), 34 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts index f9edce4079..9ebd9f330c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/table/events-select-all-banner.svelte.test.ts @@ -43,9 +43,12 @@ describe('EventsSelectAllBanner', () => { ['only some rows are selected', { isPageSelected: false, pageCount: 20, totalMatching: 1234 }], ['the page already shows every match', { isPageSelected: true, pageCount: 20, totalMatching: 20 }], ['the total is not known yet', { isPageSelected: true, pageCount: 20, totalMatching: undefined }] - ])('shows nothing when %s', (_, props) => { - // Arrange & Act - render(EventsSelectAllBanner, { props: { ...props, onClear: vi.fn(), onSelectAllMatching: vi.fn() } }); + ])('shows nothing when %s', (_, selection) => { + // Arrange + const props = { ...selection, onClear: vi.fn(), onSelectAllMatching: vi.fn() }; + + // Act + render(EventsSelectAllBanner, { props }); // Assert expect(screen.queryByRole('status')).toBeNull(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts index ed3a2e523a..82b4a25bbf 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-item.svelte.test.ts @@ -68,18 +68,27 @@ describe('TaskItem', () => { }); it('labels Cancel with the task it cancels', () => { - // Arrange & Act - render(TaskItem, { props: { canCancel: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() } }); + // Arrange + const props = { canCancel: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() }; + + // Act + render(TaskItem, { props }); // Assert expect(screen.getByRole('button', { name: 'Cancel Delete 1,234 events matching filter' })).toBeTruthy(); }); it('shows the queued state without a count', () => { - // Arrange & Act - render(TaskItem, { - props: { canCancel: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ processed: 0, status: BackgroundTaskStatus.Queued }) } - }); + // Arrange + const props = { + canCancel: true, + onCancel: vi.fn(), + onDismiss: vi.fn(), + task: createTask({ processed: 0, status: BackgroundTaskStatus.Queued }) + }; + + // Act + render(TaskItem, { props }); // Assert expect(statusText()).toBe('Waiting to start'); @@ -87,8 +96,11 @@ describe('TaskItem', () => { }); it('disables Cancel once cancellation was requested', () => { - // Arrange & Act - render(TaskItem, { props: { canCancel: true, isCancelling: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() } }); + // Arrange + const props = { canCancel: true, isCancelling: true, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask() }; + + // Act + render(TaskItem, { props }); // Assert expect((screen.getByRole('button', { name: /^Cancel / }) as HTMLButtonElement).disabled).toBe(true); @@ -110,8 +122,11 @@ describe('TaskItem', () => { }); it('shows how far a cancelled task got', () => { - // Arrange & Act - render(TaskItem, { props: { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ status: BackgroundTaskStatus.Cancelled }) } }); + // Arrange + const props = { onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ status: BackgroundTaskStatus.Cancelled }) }; + + // Act + render(TaskItem, { props }); // Assert expect(statusText()).toBe('Cancelled after 500 of 1,234'); @@ -119,28 +134,30 @@ describe('TaskItem', () => { }); it('shows why a task failed', () => { - // Arrange & Act - render(TaskItem, { - props: { - onCancel: vi.fn(), - onDismiss: vi.fn(), - task: createTask({ message: 'The task stopped because of an unexpected error.', status: BackgroundTaskStatus.Failed }) - } - }); + // Arrange + const props = { + onCancel: vi.fn(), + onDismiss: vi.fn(), + task: createTask({ message: 'The task stopped because of an unexpected error.', status: BackgroundTaskStatus.Failed }) + }; + + // Act + render(TaskItem, { props }); // Assert expect(statusText()).toBe('Failed after 500 of 1,234 · The task stopped because of an unexpected error.'); }); it('shows the percent and message of a task that has no item counts', () => { - // Arrange & Act - render(TaskItem, { - props: { - onCancel: vi.fn(), - onDismiss: vi.fn(), - task: createTask({ can_cancel: false, message: 'Events removed: 12', processed: 0, progress: 50, total: 0, type: 'projects.reset-data' }) - } - }); + // Arrange + const props = { + onCancel: vi.fn(), + onDismiss: vi.fn(), + task: createTask({ can_cancel: false, message: 'Events removed: 12', processed: 0, progress: 50, total: 0, type: 'projects.reset-data' }) + }; + + // Act + render(TaskItem, { props }); // Assert expect(statusText()).toBe('50% · Events removed: 12'); @@ -148,8 +165,11 @@ describe('TaskItem', () => { }); it('does not offer Cancel for a running task when cancelling is not allowed', () => { - // Arrange & Act - render(TaskItem, { props: { canCancel: false, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ can_cancel: false }) } }); + // Arrange + const props = { canCancel: false, onCancel: vi.fn(), onDismiss: vi.fn(), task: createTask({ can_cancel: false }) }; + + // Act + render(TaskItem, { props }); // Assert expect(screen.queryByRole('button', { name: /^Cancel / })).toBeNull(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts index fad835b260..4efb9ae8d2 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts @@ -195,7 +195,10 @@ describe('TaskPanel', () => { }); it('renders nothing when there is nothing to show', () => { - // Arrange & Act + // Arrange + state.clear(); + + // Act render(TaskPanel); // Assert diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts index c2bdd9ed00..7ecb855485 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts @@ -109,8 +109,11 @@ describe('getProgressPercent', () => { [{ processed: 5, progress: 0, status: BackgroundTaskStatus.Running, total: 3 }, 100], [{ processed: 3, progress: 0, status: BackgroundTaskStatus.Completed, total: 0 }, 100], [{ processed: 0, progress: 42, status: BackgroundTaskStatus.Running, total: 0 }, 42] - ])('maps %j to %i percent', (task, expected) => { - // Arrange & Act + ])('maps %j to %i percent', (overrides, expected) => { + // Arrange + const task = createTask(overrides); + + // Act const percent = getProgressPercent(task); // Assert From 27087d9c5160f1e2d8b1971161ae00fcb0caa16a Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 22:21:09 -0600 Subject: [PATCH 29/42] Harden task lifecycle and batch task discovery --- .../BackgroundTaskTrackingWorkItemHandler.cs | 3 - .../BackgroundTasks/BackgroundTaskService.cs | 128 +++++++---- .../BulkOperations/BulkOperationService.cs | 54 ++--- .../BackgroundTaskServiceTests.cs | 198 ++++++++++++++++++ .../BulkOperationServiceTests.cs | 2 +- .../Utility/BackgroundTaskTestHarness.cs | 10 +- 6 files changed, 316 insertions(+), 79 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs index f6823d1e3d..54c4d4c157 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs @@ -145,9 +145,6 @@ private async Task StartAsync(string taskId) return true; }); - if (wasFinished && task is not null) - await TryAsync(taskId, () => _taskService.AddToActiveIndexAsync(task.OrganizationId, task.Id)); - return task is not { Status: BackgroundTaskStatus.Cancelled or BackgroundTaskStatus.Completed }; } diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs index 451d92c865..cd9773fe7f 100644 --- a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -34,8 +34,8 @@ public class BackgroundTaskService /// Every cache key and lock resource of background tasks starts with this prefix. public const string KeyPrefix = "background-tasks"; - private const int IndexPageSize = 500; - private const int MaxIndexPages = 100; + /// Bounds task payloads loaded together, including selections attached to bulk tasks. + internal const int ReadBatchSize = 50; private readonly ICacheClient _cache; private readonly ILockProvider _lockProvider; private readonly IMessagePublisher _publisher; @@ -76,14 +76,36 @@ public BackgroundTaskService(ICacheClient cache, ILockProvider lockProvider, IMe return cached.HasValue ? cached.Value : null; } + /// + /// Returns the existing tasks in input order, once per id. Missing records are omitted. Reads are batched so a + /// remote cache can retrieve several records together; callers remain responsible for checking their scope. + /// + public async Task> GetByIdsAsync(IEnumerable ids) + { + var tasks = new List(); + foreach (var keys in ids.Distinct(StringComparer.Ordinal).Select(GetTaskCacheKey).Chunk(ReadBatchSize)) + { + var cached = await _cache.GetAllAsync(keys); + foreach (string key in keys) + { + if (cached.TryGetValue(key, out var value) && value.HasValue && value.Value is { } task) + tasks.Add(task); + } + } + + return tasks; + } + /// /// All active tasks and the newest finished tasks of the organization, newest first. Only finished history is limited. /// public async Task> GetByOrganizationAsync(string organizationId, int limit = ListLimit) { var activeIds = (await GetActiveIdsAsync(organizationId)).ToHashSet(StringComparer.Ordinal); + // One snapshot avoids repeated full-list cloning in memory and offset shifts while entries expire. + var indexed = await _cache.GetListAsync(GetOrganizationIndexCacheKey(organizationId)); // Ids are ObjectIds, so descending order is newest first no matter how the cache orders its list. - var newestIds = (await GetIndexedIdsAsync(GetOrganizationIndexCacheKey(organizationId))) + var newestIds = (indexed.HasValue ? indexed.Value : []) .Concat(activeIds) .Distinct(StringComparer.Ordinal) .OrderByDescending(id => id, StringComparer.Ordinal); @@ -91,16 +113,15 @@ public async Task> GetByOrganizationAsync(string o var tasks = new List(); var expired = new List(); int finishedCount = 0; - foreach (string id in newestIds) + foreach (var batch in newestIds.Where(id => finishedCount < limit || activeIds.Contains(id)).Chunk(ReadBatchSize)) { - if (finishedCount >= limit && !activeIds.Contains(id)) - continue; - - var task = await GetAsync(id); - if (task is null) - expired.Add(id); - else if (String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal) && (task.IsActive || finishedCount < limit)) + var found = await GetByIdsAsync(batch); + expired.AddRange(batch.Except(found.Select(task => task.Id), StringComparer.Ordinal)); + foreach (var task in found) { + if (!String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal) || (!task.IsActive && finishedCount >= limit)) + continue; + tasks.Add(task); if (!task.IsActive) finishedCount++; @@ -108,7 +129,10 @@ public async Task> GetByOrganizationAsync(string o } if (expired.Count > 0) + { await _cache.ListRemoveAsync(GetOrganizationIndexCacheKey(organizationId), expired); + await RemoveFromActiveIndexAsync(organizationId, expired); + } return tasks.OrderByDescending(t => t.CreatedUtc).ThenByDescending(t => t.Id, StringComparer.Ordinal).ToList(); } @@ -121,12 +145,6 @@ public async Task> GetActiveIdsAsync(string organizationId return ids.HasValue ? ids.Value.Distinct(StringComparer.Ordinal).ToList() : []; } - /// Puts a task that is running again (for example after a queue retry) back into the active index. - public Task AddToActiveIndexAsync(string organizationId, string id) - { - return _cache.ListAddAsync(GetActiveIndexCacheKey(organizationId), [id], TaskLifetime); - } - /// Removes finished (or expired) ids from the active index of the organization. public Task RemoveFromActiveIndexAsync(string organizationId, IReadOnlyCollection ids) { @@ -163,11 +181,25 @@ public async Task AddAsync(BackgroundTask task) /// /// Adds a task and queues the work item that runs it. The work item is created after the task has its id. + /// If preparation fails, the task is marked failed and no work is queued. /// public async Task CreateAndEnqueueAsync(BackgroundTask task, Func createWorkItem) where T : class { await AddAsync(task); - await EnqueueAsync(task, createWorkItem(task)); + + T workItem; + try + { + workItem = createWorkItem(task); + } + catch (Exception ex) + { + _logger.LogError(ex, "Unable to prepare background task {TaskId}: {Message}", task.Id, ex.Message); + await FailToStartAsync(task.Id); + throw; + } + + await EnqueueAsync(task, workItem); return task; } @@ -213,21 +245,37 @@ public Task ContinueAsync(T workItem) where T : class /// public async Task UpdateAsync(string id, Func update) { - await using var updateLock = await _lockPolicy.ExecuteAsync(async ct => - await _lockProvider.AcquireAsync(GetUpdateLockResource(id), TimeSpan.FromSeconds(30), _options.LockAcquireTimeout), CancellationToken.None); - - var task = await GetAsync(id); - if (task is null) - return null; + BackgroundTask? task; + await using (var updateLock = await _lockPolicy.ExecuteAsync(async ct => + await _lockProvider.AcquireAsync(GetUpdateLockResource(id), TimeSpan.FromSeconds(30), _options.LockAcquireTimeout), CancellationToken.None)) + { + task = await GetAsync(id); + if (task is null) + return null; - if (!update(task)) - return task; + var previousUpdatedUtc = task.UpdatedUtc; + if (!update(task)) + return task; - task.UpdatedUtc = _timeProvider.GetUtcNow().UtcDateTime; - await SaveAsync(task); - if (!task.IsActive) - await RemoveFromActiveIndexAsync(task.OrganizationId, [task.Id]); + // Clients use this stamp to reject delayed notifications, including when a clock repeats or moves backwards. + var utcNow = _timeProvider.GetUtcNow().UtcDateTime; + task.UpdatedUtc = utcNow > previousUpdatedUtc ? utcNow : previousUpdatedUtc.AddTicks(1); + await SaveAsync(task); + if (task.IsActive) + { + // Active discovery must live as long as progress, including tasks that run longer than TaskLifetime. + await RefreshIndexAsync(GetActiveIndexCacheKey(task.OrganizationId), task.Id); + } + else + { + // Renew history on completion, rather than rewriting the large history index on every progress report. + // Keep active membership until history is saved so a concurrent list cannot miss a long-running task. + await RefreshIndexAsync(GetOrganizationIndexCacheKey(task.OrganizationId), task.Id); + await RemoveFromActiveIndexAsync(task.OrganizationId, [task.Id]); + } + } + // The state is committed; a slow publisher must not block another update or cancellation. await PublishAsync(task); return task; } @@ -275,7 +323,7 @@ public async Task IsCancelRequestedAsync(string id) return (await _cache.GetAsync(GetCancelCacheKey(id))).HasValue; } - public async Task SaveAsync(BackgroundTask task) + private async Task SaveAsync(BackgroundTask task) { if (!await _cache.SetAsync(GetTaskCacheKey(task.Id), task, TaskLifetime)) throw new InvalidOperationException("The background task could not be saved."); @@ -331,21 +379,11 @@ private Task FailToStartAsync(string id) }); } - private async Task> GetIndexedIdsAsync(string key) + private async Task RefreshIndexAsync(string key, string id) { - var ids = new List(); - for (int page = 1; page <= MaxIndexPages; page++) - { - var batch = await _cache.GetListAsync(key, page, IndexPageSize); - if (!batch.HasValue || batch.Value.Count == 0) - break; - - ids.AddRange(batch.Value); - if (batch.Value.Count < IndexPageSize) - break; - } - - return ids; + await _cache.ListAddAsync(key, [id], TaskLifetime); + // Refreshing an existing list member does not extend the key TTL in every cache provider. + await _cache.SetExpirationAsync(key, TaskLifetime); } } diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index 9984a5505f..11b9a4c468 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -200,42 +200,42 @@ public Task PublishLostAsync(BulkOperationWorkItem workItem) var utcNow = _timeProvider.GetUtcNow().UtcDateTime; var finished = new List(); BackgroundTask? conflict = null; - foreach (string id in await _taskService.GetActiveIdsAsync(candidate.OrganizationId)) + var activeIds = await _taskService.GetActiveIdsAsync(candidate.OrganizationId); + foreach (var batch in activeIds.Chunk(BackgroundTaskService.ReadBatchSize)) { - var existing = await _taskService.GetAsync(id); - if (existing is null || !existing.IsActive) + var tasks = await _taskService.GetByIdsAsync(batch); + finished.AddRange(batch.Except(tasks.Where(task => task.IsActive).Select(task => task.Id), StringComparer.Ordinal)); + foreach (var active in tasks.Where(task => task.IsActive)) { - finished.Add(id); - continue; - } - - if (GetHandler(existing.Type) is not { IsDestructive: true }) - continue; + var existing = active; + if (GetHandler(existing.Type) is not { IsDestructive: true }) + continue; - // An organization wide task can touch every project, so it conflicts with any project's task. - bool overlaps = existing.ProjectId is null || candidate.ProjectId is null || String.Equals(existing.ProjectId, candidate.ProjectId, StringComparison.Ordinal); - if (!overlaps) - continue; + // An organization wide task can touch every project, so it conflicts with any project's task. + bool overlaps = existing.ProjectId is null || candidate.ProjectId is null || String.Equals(existing.ProjectId, candidate.ProjectId, StringComparison.Ordinal); + if (!overlaps) + continue; - if (utcNow - existing.UpdatedUtc >= BackgroundTaskService.StaleAfter) - { - // Retire it before admitting a replacement, so a delayed delivery cannot run it alongside the new one. - var retired = await _taskService.UpdateAsync(existing.Id, current => + if (utcNow - existing.UpdatedUtc >= BackgroundTaskService.StaleAfter) { - if (!current.IsActive || utcNow - current.UpdatedUtc < BackgroundTaskService.StaleAfter) - return false; + // Retire it before admitting a replacement, so a delayed delivery cannot run it alongside the new one. + var retired = await _taskService.UpdateAsync(existing.Id, current => + { + if (!current.IsActive || utcNow - current.UpdatedUtc < BackgroundTaskService.StaleAfter) + return false; - current.Finish(BackgroundTaskStatus.Failed, "Stopped because it made no progress for two hours.", utcNow); - return true; - }); + current.Finish(BackgroundTaskStatus.Failed, "Stopped because it made no progress for two hours.", utcNow); + return true; + }); - if (retired is null || !retired.IsActive) - continue; + if (retired is null || !retired.IsActive) + continue; - existing = retired; - } + existing = retired; + } - conflict ??= existing; + conflict ??= existing; + } } if (finished.Count > 0) diff --git a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs index d12f582c86..4821581548 100644 --- a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs @@ -117,6 +117,27 @@ public async Task CreateAndEnqueueAsync_IndexCannotBeStored_DoesNotQueueOrLeaveA Assert.Equal(0, queued.Enqueued); } + [Fact] + public async Task GetByIdsAsync_MissingAndRepeatedIdsAcrossBatches_ReturnsExistingTasksInInputOrder() + { + // Arrange + var tasks = new List(); + for (int index = 0; index <= BackgroundTaskService.ReadBatchSize; index++) + { + var task = NewTask(); + await Tasks.AddAsync(task); + tasks.Add(task); + } + + string[] requested = [.. tasks.Select(task => task.Id).Reverse(), ObjectId.GenerateNewId().ToString(), tasks[0].Id]; + + // Act + var found = await Tasks.GetByIdsAsync(requested); + + // Assert + Assert.Equal(tasks.Select(task => task.Id).Reverse(), found.Select(task => task.Id)); + } + [Fact] public async Task GetByOrganizationAsync_FinishedTasks_ReturnsTheNewestFirstUpToTheLimit() { @@ -142,6 +163,31 @@ await Tasks.UpdateAsync(task.Id, current => Assert.Equal([ids[3], ids[2]], newestTwo.Select(t => t.Id).ToArray()); } + [Fact] + public async Task GetByOrganizationAsync_HistoryExceedsFormerPageLimit_ReturnsNewestTask() + { + // Arrange + const int historyCount = 50_001; + var utcNow = _time.GetUtcNow().UtcDateTime; + var firstCreatedUtc = utcNow.AddSeconds(1 - historyCount); + var ids = Enumerable.Range(0, historyCount) + .Select(index => ObjectId.GenerateNewId(firstCreatedUtc.AddSeconds(index)).ToString()) + .ToArray(); + var newest = NewTask(); + newest.Id = ids[^1]; + newest.CreatedUtc = utcNow; + newest.UpdatedUtc = utcNow; + newest.Finish(BackgroundTaskStatus.Completed, "Done", utcNow); + await _harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(newest.Id), newest, BackgroundTaskService.TaskLifetime); + await _harness.Cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId), ids, BackgroundTaskService.TaskLifetime); + + // Act + var listed = await Tasks.GetByOrganizationAsync(OrganizationId, limit: 1); + + // Assert + Assert.Equal(newest.Id, Assert.Single(listed).Id); + } + [Fact] public async Task GetByOrganizationAsync_ActiveTasks_DoNotConsumeTheFinishedHistoryLimit() { @@ -203,6 +249,60 @@ public async Task GetByOrganizationAsync_ExpiredAndForeignIds_AreSkippedAndExpir Assert.Contains(kept.Id, indexed.Value); } + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task UpdateAsync_NotificationWaits_PersistsOrderedUpdatesWithoutHoldingTheLock(int clockAdjustmentSeconds) + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.Zero }, lockAttempts: 1); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + var publishing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releasePublication = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + harness.Publisher.BeforePublish = () => + { + harness.Publisher.BeforePublish = null; + publishing.SetResult(); + return releasePublication.Task; + }; + + // Act + var firstUpdate = harness.Tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.Progress = 20; + return true; + }); + Exception? exception; + try + { + await publishing.Task.WaitAsync(TestCancellationToken); + _time.AdjustTime(_time.GetUtcNow().AddSeconds(clockAdjustmentSeconds)); + exception = await Record.ExceptionAsync(async () => await harness.Tasks.UpdateAsync(task.Id, current => + { + current.Progress = 40; + return true; + })); + } + finally + { + releasePublication.SetResult(); + await firstUpdate; + } + + var stored = await harness.Tasks.GetAsync(task.Id); + var updates = harness.Publisher.Updates; + + // Assert + Assert.Null(exception); + Assert.NotNull(stored); + Assert.Equal(40, stored.Progress); + Assert.Equal([40, 20], updates.Select(update => update.Progress)); + Assert.Equal(stored.UpdatedUtc, updates[0].UpdatedUtc); + Assert.True(updates[0].UpdatedUtc > updates[1].UpdatedUtc); + } + [Fact] public async Task UpdateAsync_UpdateReturnsFalse_SavesAndPublishesNothing() { @@ -259,6 +359,65 @@ public async Task UpdateAsync_ChangeThatLeavesTheTaskActive_SavesStampsAndPublis Assert.Equal(task.CanCancel, update.CanCancel); } + [Fact] + public async Task UpdateAsync_LongRunningTask_KeepsIndexesForTheSavedRecordLifetime() + { + // Arrange + ObservingHistoryCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingHistoryCacheClient(inner)); + var tasks = harness.Tasks; + var task = NewTask(); + await tasks.AddAsync(task); + _time.Advance(TimeSpan.FromDays(6)); + + // Act + await tasks.UpdateAsync(task.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.Progress = 50; + return true; + }); + _time.Advance(TimeSpan.FromDays(2)); + var stored = await tasks.GetAsync(task.Id); + var active = await tasks.GetActiveIdsAsync(OrganizationId); + var listed = await tasks.GetByOrganizationAsync(OrganizationId); + + // Assert + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Running, stored.Status); + Assert.Equal([task.Id], active); + Assert.Equal(task.Id, Assert.Single(listed).Id); + + // Arrange + IReadOnlyList duringHistoryRefresh = []; + cache!.BeforeHistoryRefresh = async () => duringHistoryRefresh = await tasks.GetByOrganizationAsync(OrganizationId); + + // Act + await tasks.UpdateAsync(task.Id, current => + { + current.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); + return true; + }); + _time.Advance(TimeSpan.FromDays(6)); + var finished = await tasks.GetByOrganizationAsync(OrganizationId); + var finishedActive = await tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.Equal(task.Id, Assert.Single(duringHistoryRefresh).Id); + Assert.Equal(BackgroundTaskStatus.Completed, Assert.Single(finished).Status); + Assert.Empty(finishedActive); + + // Act + _time.Advance(TimeSpan.FromDays(1) + TimeSpan.FromSeconds(1)); + var expired = await tasks.GetAsync(task.Id); + var expiredList = await tasks.GetByOrganizationAsync(OrganizationId); + + // Assert + Assert.Null(expired); + Assert.Empty(expiredList); + } + [Fact] public async Task UpdateAsync_TaskFinishes_RemovesItFromTheActiveIndexAndPublishesTheFinalState() { @@ -594,6 +753,29 @@ public async Task CreateAndEnqueueAsync_Success_QueuesTheWorkItemWithTheTaskIdAn Assert.Equal(task.Id, update.Id); } + [Fact] + public async Task CreateAndEnqueueAsync_WorkItemFactoryThrows_FailsTheTaskWithoutQueueingIt() + { + // Arrange + var task = NewTask(); + var failure = new InvalidOperationException("Unable to prepare the work item"); + + // Act + var thrown = await Record.ExceptionAsync(async () => await Tasks.CreateAndEnqueueAsync(task, _ => throw failure)); + var stored = await Tasks.GetAsync(task.Id); + var active = await Tasks.GetActiveIdsAsync(OrganizationId); + var queued = await _harness.Queue.GetQueueStatsAsync(); + + // Assert + Assert.Same(failure, thrown); + Assert.NotNull(stored); + Assert.Equal(BackgroundTaskStatus.Failed, stored.Status); + Assert.NotNull(stored.CompletedUtc); + Assert.Empty(active); + Assert.Equal(0, queued.Enqueued); + Assert.Equal(BackgroundTaskStatus.Failed, Assert.Single(_harness.Publisher.Updates).Status); + } + [Fact] public async Task EnqueueAsync_QueueRejectsTheWorkItem_FailsTheTaskSoItIsNotLeftWaiting() { @@ -717,6 +899,22 @@ private static BackgroundTask NewTask(string organizationId = OrganizationId, bo }; } + private sealed class ObservingHistoryCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + public Func? BeforeHistoryRefresh { get; set; } + + public new async Task ListAddAsync(string key, IEnumerable values, TimeSpan? expiresIn = null) where T : notnull + { + if (key == BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId) && BeforeHistoryRefresh is { } observe) + { + BeforeHistoryRefresh = null; + await observe(); + } + + return await base.ListAddAsync(key, values, expiresIn); + } + } + private sealed class LostAcknowledgementQueue(InMemoryQueueOptions options) : InMemoryQueue(options) { public Func? AfterEnqueue { get; set; } diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs index 14ec5dfddc..4396e5d3b9 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs @@ -205,7 +205,7 @@ public async Task CreateAsync_FinishedTasksStillInTheActiveIndex_AreDroppedFromI await _harness.Tasks.AddAsync(expired); // Simulate an index that was not cleaned: the task finished without being removed, and another record expired. finished.Finish(BackgroundTaskStatus.Completed, "Done", _time.GetUtcNow().UtcDateTime); - await _harness.Tasks.SaveAsync(finished); + await _harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(finished.Id), finished, BackgroundTaskService.TaskLifetime); await _harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(expired.Id)); // Act diff --git a/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs index 9cef9506b5..37e089d69e 100644 --- a/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs +++ b/tests/Exceptionless.Tests/Utility/BackgroundTaskTestHarness.cs @@ -59,6 +59,9 @@ public sealed class RecordingPublisher : IMessagePublisher /// When set, publishing throws, like a message bus that is down. public Exception? Failure { get; set; } + /// Lets a test hold publication while another task update proceeds. + public Func? BeforePublish { get; set; } + public IReadOnlyList Updates { get @@ -68,15 +71,16 @@ public IReadOnlyList Updates } } - public Task PublishAsync(Type messageType, object message, MessageOptions? options = null, CancellationToken cancellationToken = default) + public async Task PublishAsync(Type messageType, object message, MessageOptions? options = null, CancellationToken cancellationToken = default) { if (Failure is not null) throw Failure; + if (BeforePublish is { } beforePublish) + await beforePublish(); + lock (_messages) _messages.Add(message); - - return Task.CompletedTask; } } } From 4566256adb7dd8824da5efc6958128f531067172 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 22:23:25 -0600 Subject: [PATCH 30/42] Load only required event fields for bulk deletion --- .../DeleteEventsBulkOperationHandler.cs | 4 +- .../Endpoints/BulkOperationEndpointTests.cs | 47 ++++++++++++++----- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs index b8c9a3fc89..148414a812 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/DeleteEventsBulkOperationHandler.cs @@ -55,7 +55,9 @@ public string Describe(BackgroundTask task) public async Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) { - var events = await _eventRepository.GetByIdsAsync(ids.ToArray(), o => o.Cache(false)); + // Deletion needs scope, daily-index routing and notification fields, not the event's potentially large payload. + var events = await _eventRepository.GetByIdsAsync(ids.ToArray(), o => o.Cache(false) + .Include(e => e.Id, e => e.Date, e => e.OrganizationId, e => e.ProjectId, e => e.StackId)); // The task may only touch its own organization (and project), no matter what ids it was given. var inScope = events diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index ec6e2a13af..f2adebaf3f 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -197,32 +197,53 @@ public async Task Create_FilterSelection_DoesNotIncludeEventsStoredAfterCreation public async Task Create_IdSelection_DeletesTheIdsThroughTheSameSideEffectsAsTheSingleAction() { // Arrange + var now = TimeProvider.GetUtcNow(); var (_, events) = await CreateDataAsync(d => { - d.Event().TestProject(); - d.Event().TestProject(); + d.Event().TestProject().Date(now.AddHours(-1)).Message("First selected event") + .Mutate(e => e.Data = new() { ["payload"] = new string('x', 4096) }); + d.Event().TestProject().Date(now.AddDays(-2)).Message("Second selected event") + .Mutate(e => e.Data = new() { ["payload"] = new string('y', 4096) }); d.Event().TestProject(); }); - - // Act - var created = await CreateOperationAsync(new NewBulkOperation + var selected = events.Take(2).ToDictionary(e => e.Id); + var removed = new List(); + using var removing = _eventRepository.DocumentsRemoving.AddHandler((_, args) => + { + removed.AddRange(args.Documents); + return Task.CompletedTask; + }); + var usageService = GetService(); + var request = new NewBulkOperation { Type = "events.delete", OrganizationId = SampleDataService.TEST_ORG_ID, - Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = events.Take(2).Select(e => e.Id).ToList() } - }); + Selection = new NewBulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = selected.Keys.ToList() } + }; + + // Act + var created = await CreateOperationAsync(request); await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); + var finished = await GetOperationAsync(created.Id); + var remaining = await _eventRepository.GetByIdsAsync(events.Select(e => e.Id).ToArray()); + var usage = await usageService.GetUsageAsync(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID); // Assert - var finished = await GetOperationAsync(created.Id); Assert.Equal(BackgroundTaskStatus.Completed, finished.Status); Assert.Equal(2, finished.Succeeded); - Assert.Null(await _eventRepository.GetByIdAsync(events[0].Id)); - Assert.Null(await _eventRepository.GetByIdAsync(events[1].Id)); - Assert.NotNull(await _eventRepository.GetByIdAsync(events[2].Id)); - - var usage = await GetService().GetUsageAsync(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID); + Assert.Equal(events[2].Id, Assert.Single(remaining).Id); Assert.Equal(2, usage.CurrentUsage.Deleted); + Assert.Equal(selected.Keys.Order(), removed.Select(e => e.Id).Order()); + Assert.All(removed, deleted => + { + var original = selected[deleted.Id]; + Assert.Equal(original.Date, deleted.Date); + Assert.Equal(original.OrganizationId, deleted.OrganizationId); + Assert.Equal(original.ProjectId, deleted.ProjectId); + Assert.Equal(original.StackId, deleted.StackId); + Assert.Null(deleted.Message); + Assert.Empty(deleted.Data!); + }); } [Fact] From aa494c31d9c3f41ed88e725f4dd9155f5ec8a978 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 22:23:26 -0600 Subject: [PATCH 31/42] Keep task controls reachable and coalesce completion refreshes --- .../e2e/tests/bulk-delete-matching.e2e.ts | 70 +++++++++++++++++++ .../tasks/components/task-panel.svelte | 7 +- .../components/task-panel.svelte.test.ts | 13 ++-- 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts index 8d6ff533c1..0d17da7145 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/bulk-delete-matching.e2e.ts @@ -1,9 +1,79 @@ +import { BackgroundTaskStatus, type ViewBackgroundTask } from '#generated/api.ts'; + import { expect, test } from '../fixtures/e2e-test'; import { createRepresentativeEvent } from '../support/synthetic-event'; const EVENT_COUNT = 5; const PAGE_SIZE = 2; +test('many running tasks stay within the viewport and the oldest task can be cancelled after scrolling', async ({ e2eScenario, page }) => { + // Arrange + const now = Date.now(); + const tasks: ViewBackgroundTask[] = Array.from({ length: 12 }, (_, index) => ({ + can_cancel: true, + created_by_user_id: '000000000000000000000001', + created_utc: new Date(now - index * 1000).toISOString(), + description: `Synthetic bulk task ${index + 1}`, + failed: 0, + id: (index + 1).toString(16).padStart(24, '0'), + organization_id: e2eScenario.organizationId, + processed: 25, + progress: 25, + project_id: e2eScenario.projectId, + status: BackgroundTaskStatus.Running, + succeeded: 25, + total: 100, + type: 'events.delete', + updated_utc: new Date(now).toISOString() + })); + const oldest = tasks.at(-1)!; + await page.route( + (url) => url.pathname === `/api/v2/organizations/${e2eScenario.organizationId}/tasks`, + (route) => route.fulfill({ json: tasks }) + ); + await page.route(`**/api/v2/tasks/${oldest.id}/cancel`, async (route) => { + oldest.status = BackgroundTaskStatus.Cancelled; + oldest.updated_utc = new Date().toISOString(); + await route.fulfill({ json: oldest, status: 202 }); + }); + await page.setViewportSize({ height: 400, width: 390 }); + const panel = page.getByRole('region', { name: 'Background tasks' }); + const oldestCancel = panel.getByRole('button', { exact: true, name: `Cancel ${oldest.description}` }); + + // Act + await page.goto(`/event?project=${e2eScenario.projectId}&time=all`); + + // Assert + await expect(panel.getByRole('button', { name: /^Cancel Synthetic bulk task / })).toHaveCount(tasks.length); + const metrics = await panel.evaluate((element) => { + const bounds = element.getBoundingClientRect(); + return { + bottom: bounds.bottom, + clientHeight: element.clientHeight, + scrollHeight: element.scrollHeight, + top: bounds.top, + viewportHeight: window.innerHeight + }; + }); + expect(metrics.top).toBeGreaterThanOrEqual(0); + expect(metrics.bottom).toBeLessThanOrEqual(metrics.viewportHeight); + expect(metrics.scrollHeight).toBeGreaterThan(metrics.clientHeight); + await expect(oldestCancel).not.toBeInViewport(); + + // Act + await oldestCancel.scrollIntoViewIfNeeded(); + + // Assert + await expect(oldestCancel).toBeInViewport(); + expect(await panel.evaluate((element) => element.scrollTop)).toBeGreaterThan(0); + + // Act + await oldestCancel.click(); + + // Assert + await expect(panel.getByRole('button', { exact: true, name: `Dismiss ${oldest.description}` })).toBeVisible(); +}); + test('deleting everything matching the filter runs in the background and shows progress', async ({ e2eApi, e2eScenario, page }) => { // Arrange: more events than fit on one page, all in this test's own project. const references = Array.from({ length: EVENT_COUNT }, (_, index) => `${e2eScenario.referenceId}-bulk-${index}`); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte index a76c99c505..f1a240513f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte @@ -42,6 +42,7 @@ // A task that was seen running and is now finished changed events and stacks, whichever way the update arrived. $effect(() => { + let hasFinishedTasks = false; for (const task of tasksQuery.data ?? []) { if (isTaskActive(task.status)) { // A failed task can be retried with the same id; its new attempt and outcome must be visible. @@ -55,6 +56,10 @@ } cancellingIds.delete(task.id); + hasFinishedTasks = true; + } + + if (hasFinishedTasks) { void invalidateTaskAffectedQueries(queryClient); } }); @@ -89,7 +94,7 @@ {#if visibleTasks.length > 0}
{#each visibleTasks as task (task.id)} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts index 4efb9ae8d2..5a444d502b 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts @@ -98,16 +98,21 @@ describe('TaskPanel', () => { expect(mocks.updateFromMessage).toHaveBeenCalledWith(mocks.queryClient, update); }); - it('refreshes events and stacks once when a running task finishes', async () => { + it.each([1, 3])('refreshes events and stacks once when %i running tasks finish in one update', async (count) => { // Arrange - state.set('tasks', [createTask()]); + const running = Array.from({ length: count }, (_, index) => createTask({ id: `task-${index}` })); + const completed = running.map((task) => ({ ...task, processed: 50, status: BackgroundTaskStatus.Completed, succeeded: 50 })); + state.set('tasks', running); render(TaskPanel); await tick(); // Act - state.set('tasks', [createTask({ processed: 50, status: BackgroundTaskStatus.Completed, succeeded: 50 })]); + state.set('tasks', completed); await tick(); - state.set('tasks', [createTask({ processed: 50, status: BackgroundTaskStatus.Completed, succeeded: 50, updated_utc: new Date().toISOString() })]); + state.set( + 'tasks', + completed.map((task) => ({ ...task, updated_utc: new Date().toISOString() })) + ); await tick(); // Assert From 557b4fad8c7fe2ed77f31955b2a2ba14b939aba5 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 22:23:27 -0600 Subject: [PATCH 32/42] Document task contracts and reproducible performance measurements --- README.md | 2 + .../BackgroundTasks/BackgroundTasks.csproj | 12 + benchmarks/BackgroundTasks/Program.cs | 152 +++++++++++ benchmarks/BackgroundTasks/README.md | 56 +++++ benchmarks/BackgroundTasks/comparison.json | 238 ++++++++++++++++++ docs/background-tasks.md | 159 ++++++++++++ 6 files changed, 619 insertions(+) create mode 100644 benchmarks/BackgroundTasks/BackgroundTasks.csproj create mode 100644 benchmarks/BackgroundTasks/Program.cs create mode 100644 benchmarks/BackgroundTasks/README.md create mode 100644 benchmarks/BackgroundTasks/comparison.json create mode 100644 docs/background-tasks.md diff --git a/README.md b/README.md index 57afbe85fe..30e94dbebe 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ Notes: 3. Use Aspire CLI 13.6.5 or newer (`aspire update --self` to update). The AppHost uses the CLI bundle for the dashboard and resource orchestrator; when no CLI is installed, the SDK downloads the matching version through DNX. 4. Try `dotnet build -mt` for multithreaded builds. Performance depends on the workload and machine. +See the [background task developer guide](docs/background-tasks.md) for task lifecycle, API contracts, cancellation, bulk execution, and extension points. + ## UI Development The Svelte 5 UI in `src/Exceptionless.Web/ClientApp` is the only application frontend and serves at `/`. Aspire starts Svelte together with the API. Primary hash-route bookmarks resolve to the current pages, and existing signed-in sessions are retained. diff --git a/benchmarks/BackgroundTasks/BackgroundTasks.csproj b/benchmarks/BackgroundTasks/BackgroundTasks.csproj new file mode 100644 index 0000000000..64784b27ff --- /dev/null +++ b/benchmarks/BackgroundTasks/BackgroundTasks.csproj @@ -0,0 +1,12 @@ + + + Exe + net11.0 + enable + enable + true + + + + + diff --git a/benchmarks/BackgroundTasks/Program.cs b/benchmarks/BackgroundTasks/Program.cs new file mode 100644 index 0000000000..edcf949bdb --- /dev/null +++ b/benchmarks/BackgroundTasks/Program.cs @@ -0,0 +1,152 @@ +using System.Diagnostics; +using System.Text.Json; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Services.BackgroundTasks; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Messaging; +using Foundatio.Queues; +using Foundatio.Repositories.Utility; +using Foundatio.Resilience; +using Microsoft.Extensions.Logging.Abstractions; + +// A manual service microbenchmark, deliberately outside the correctness test suite. +// Compare the same command before and after a change. This does not measure Redis network latency. +string label = args.FirstOrDefault() ?? "current"; +int[] sizes = args.Length > 1 ? args.Skip(1).Select(Int32.Parse).ToArray() : [100, 1_000, 10_000, 50_001]; +foreach (int historyCount in sizes) +{ + using var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { CloneValues = true, MaxItems = null }); + using var messageBus = new InMemoryMessageBus(); + using var queue = new InMemoryQueue(); + var measuredCache = new MeasuredCacheClient(cache); + var locks = new CacheLockProvider(cache, messageBus); + var tasks = new BackgroundTaskService(measuredCache, locks, messageBus, queue, + new ResiliencePolicyProvider(), new BackgroundTaskOptions(), TimeProvider.System, NullLogger.Instance); + + const string organizationId = "537650f3b77efe23a47914f4"; + var createdUtc = DateTime.UtcNow.AddSeconds(-historyCount - 2); + var ids = new List(historyCount + 2); + for (int i = 0; i < historyCount + 2; i++) + { + var task = new BackgroundTask + { + Id = ObjectId.GenerateNewId(createdUtc.AddSeconds(i)).ToString(), + OrganizationId = organizationId, + Type = "benchmark.work", + Description = "Benchmark task", + CreatedByUserId = "537650f3b77efe23a47914f5", + CreatedUtc = createdUtc.AddSeconds(i), + UpdatedUtc = createdUtc.AddSeconds(i), + Status = i < 2 ? BackgroundTaskStatus.Running : BackgroundTaskStatus.Completed, + Progress = i < 2 ? 10 : 100 + }; + ids.Add(task.Id); + await cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(task.Id), task, BackgroundTaskService.TaskLifetime); + } + + // Seed equivalent persisted state directly so index construction is outside the measurement. + await cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(organizationId), ids, BackgroundTaskService.TaskLifetime); + await cache.ListAddAsync(BackgroundTaskService.GetActiveIndexCacheKey(organizationId), ids.Take(2), BackgroundTaskService.TaskLifetime); + async Task RunAsync(string operation) + { + if (operation == "list") + { + var result = await tasks.GetByOrganizationAsync(organizationId); + return result.Count == Math.Min(historyCount, BackgroundTaskService.ListLimit) + 2 + && result[0].Id == ids[^1] + && result.Count(t => t.IsActive) == 2; + } + + var updated = await tasks.UpdateAsync(ids[0], task => + { + task.Progress = 20; + return true; + }); + return updated is { Progress: 20, IsActive: true }; + } + + foreach (string operation in new[] { "list", "progress" }) + { + for (int i = 0; i < 3; i++) + await RunAsync(operation); + + int iterations = historyCount >= 10_000 ? 7 : 21; + var milliseconds = new List(iterations); + var allocatedBytes = new List(iterations); + bool correct = true; + for (int i = 0; i < iterations; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + measuredCache.Reset(); + long beforeAllocation = GC.GetTotalAllocatedBytes(precise: true); + long started = Stopwatch.GetTimestamp(); + correct &= await RunAsync(operation); + milliseconds.Add(Stopwatch.GetElapsedTime(started).TotalMilliseconds); + allocatedBytes.Add(GC.GetTotalAllocatedBytes(precise: true) - beforeAllocation); + } + + milliseconds.Sort(); + allocatedBytes.Sort(); + Console.WriteLine(JsonSerializer.Serialize(new + { + label, + operation, + historyCount, + iterations, + medianMilliseconds = milliseconds[iterations / 2], + medianAllocatedBytes = allocatedBytes[iterations / 2], + measuredCache.ListReads, + measuredCache.SingleReads, + measuredCache.BatchReads, + measuredCache.ListWrites, + measuredCache.ExpirationWrites, + correct + })); + } +} + +// Instrument calls while preserving the real cache implementation and its cloning behavior. +sealed class MeasuredCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient +{ + public int ListReads { get; private set; } + public int SingleReads { get; private set; } + public int BatchReads { get; private set; } + public int ListWrites { get; private set; } + public int ExpirationWrites { get; private set; } + + public void Reset() => ListReads = SingleReads = BatchReads = ListWrites = ExpirationWrites = 0; + + Task> ICacheClient.GetAsync(string key) + { + SingleReads++; + return UnscopedCache.GetAsync(key); + } + + Task>> ICacheClient.GetAllAsync(IEnumerable keys) + { + BatchReads++; + return UnscopedCache.GetAllAsync(keys); + } + + Task>> ICacheClient.GetListAsync(string key, int? page, int pageSize) + { + ListReads++; + return UnscopedCache.GetListAsync(key, page, pageSize); + } + + Task ICacheClient.ListAddAsync(string key, IEnumerable values, TimeSpan? expiresIn) + { + ListWrites++; + return UnscopedCache.ListAddAsync(key, values, expiresIn); + } + + Task ICacheClient.SetExpirationAsync(string key, TimeSpan expiresIn) + { + ExpirationWrites++; + return UnscopedCache.SetExpirationAsync(key, expiresIn); + } +} diff --git a/benchmarks/BackgroundTasks/README.md b/benchmarks/BackgroundTasks/README.md new file mode 100644 index 0000000000..9213df9c23 --- /dev/null +++ b/benchmarks/BackgroundTasks/README.md @@ -0,0 +1,56 @@ +# Background task service benchmark + +This manual executable measures task listing and active progress updates using the real `BackgroundTaskService` and Foundatio in-memory cache. It is outside the normal test suite and solution. It adds no benchmark package or production dependency. + +## Run + +From the repository root with the .NET 11 SDK: + +```sh +dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current +``` + +The first argument labels the results. Optional following arguments replace the finished-history sizes: + +```sh +dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current 100 1000 10000 50001 +``` + +To compare revisions, create a separate worktree at the baseline, copy this folder's `Program.cs` and `.csproj` into the same location there, and run the same command in each checkout. Keep the SDK, configuration, dependencies, and machine identical. Run the measurements sequentially without simultaneous builds or test suites. + +## What is measured + +- Seed each history size plus two older active tasks. Setup is outside the measurement. Cache values are cloned as in application use; capacity eviction is disabled so it does not change the selected workload. +- Warm each operation three times, then take 21 samples for smaller histories or seven for histories of 10,000 or more. +- Report median elapsed time and total managed allocation per call, including asynchronous work. Force collection outside each sample to make retained setup state comparable. +- Count service-level index reads, individual and batched record reads, index renewals, and expiry updates. The measuring decorator delegates directly to the underlying cache. +- Check that listing returns the newest finished task, the expected count, and both active tasks. Check that progress updates retain the running task and requested progress. + +Allocation means bytes allocated during an operation, not retained process memory. This is a service microbenchmark, not an Elasticsearch throughput test or a Redis latency forecast. `GetAllAsync` enables batching, but provider behavior and Redis Cluster hash-slot placement determine actual network requests. Progress measurements use two active entries; costs for a larger active index must be measured separately. + +## Recorded comparison + +Measured on Linux x64, .NET SDK 11.0.100, Release, Foundatio 13.0.5. The baseline is [`fc804c08`](https://github.com/exceptionless/Exceptionless/commit/fc804c084506f5f78fc80b11209b49e437423587). The candidate is the service implementation accompanying this report. [comparison.json](comparison.json) contains every recorded sample summary and the candidate service blob identifier. + +| Finished history | Baseline listing | Candidate listing | Baseline allocation | Candidate allocation | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 0.1340 ms | 0.1573 ms | 0.050 MB | 0.061 MB | +| 1,000 | 0.9845 ms | 0.6104 ms | 0.308 MB | 0.188 MB | +| 10,000 | 16.6519 ms | 5.2518 ms | 10.578 MB | 1.364 MB | +| 50,001 | 152.1701 ms | 17.0455 ms | 233.967 MB | 6.241 MB | + +The 50,001 baseline result is **incorrect**: the old page cap omits the newest task. Its timing is included to expose the old work and failure, not as a comparison between two correct implementations. All candidate result checks passed. + +At 10,000 finished tasks, both versions return the correct result; the candidate used about 87% fewer allocated bytes and 68% less elapsed time. The candidate performs two index reads and two batched record reads for these fixtures. The baseline performs 22 index reads and 52 individual record reads at 10,000 tasks, rising to 101 index reads at 50,001. + +The small-history tradeoff is visible: at 100 tasks, dictionary batching adds about 12 KB of allocation and 0.023 ms of in-process work. It reduces sequential cache calls, which this local benchmark does not price as remote latency. Avoid adding provider-specific branches or caching layers to optimize that small absolute cost without a representative workload. + +### Retention cost + +Progress updates now renew the small active index and its key expiry so a task remains discoverable for the lifetime of its saved record. They do not rewrite the history index. With two active tasks, median allocation increased from 6,640 to 9,128 bytes at the first three history sizes: about 2.5 KB per update. At 1,000 history entries, the observed median was 0.0773 ms before and 0.1047 ms after. Submillisecond timings vary with runtime and host scheduling; the extra renewal and expiry calls are the stable change. + +Terminal updates renew history before removing active membership. These measurements cover active progress, not terminal-history writes. Index retention is a correctness obligation; report its write cost alongside read improvements instead of claiming that every operation became faster. + +## Remaining scaling boundary + +The portable cache API has no ordered recent-history query. Listing still reads and sorts all live history IDs, and returning all active tasks requires materializing that active set. The improvement removes repeated index copies, the page cutoff, and sequential record reads. A future store with ordered discovery can change that access pattern when real volume warrants it; the [task guide](../../docs/background-tasks.md) describes that extraction boundary. diff --git a/benchmarks/BackgroundTasks/comparison.json b/benchmarks/BackgroundTasks/comparison.json new file mode 100644 index 0000000000..29bae3a484 --- /dev/null +++ b/benchmarks/BackgroundTasks/comparison.json @@ -0,0 +1,238 @@ +{ + "environment": { + "platform": "Linux x64", + "sdk": "11.0.100", + "configuration": "Release", + "foundatio": "13.0.5" + }, + "baseline_commit": "fc804c084506f5f78fc80b11209b49e437423587", + "candidate_service_blob": "cd9773fe7feafda0cbabb1e44edf1527b37536f0", + "baseline": [ + { + "label": "fc804c08", + "operation": "list", + "historyCount": 100, + "iterations": 21, + "medianMilliseconds": 0.134, + "medianAllocatedBytes": 49640, + "ListReads": 2, + "SingleReads": 52, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "progress", + "historyCount": 100, + "iterations": 21, + "medianMilliseconds": 0.0717, + "medianAllocatedBytes": 6640, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "list", + "historyCount": 1000, + "iterations": 21, + "medianMilliseconds": 0.9845, + "medianAllocatedBytes": 308328, + "ListReads": 4, + "SingleReads": 52, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "progress", + "historyCount": 1000, + "iterations": 21, + "medianMilliseconds": 0.0773, + "medianAllocatedBytes": 6640, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "list", + "historyCount": 10000, + "iterations": 7, + "medianMilliseconds": 16.6519, + "medianAllocatedBytes": 10577920, + "ListReads": 22, + "SingleReads": 52, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "progress", + "historyCount": 10000, + "iterations": 7, + "medianMilliseconds": 0.1462, + "medianAllocatedBytes": 6640, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "fc804c08", + "operation": "list", + "historyCount": 50001, + "iterations": 7, + "medianMilliseconds": 152.1701, + "medianAllocatedBytes": 233967176, + "ListReads": 101, + "SingleReads": 52, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "fc804c08", + "operation": "progress", + "historyCount": 50001, + "iterations": 7, + "medianMilliseconds": 0.1139, + "medianAllocatedBytes": 6584, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + } + ], + "candidate": [ + { + "label": "candidate", + "operation": "list", + "historyCount": 100, + "iterations": 21, + "medianMilliseconds": 0.1573, + "medianAllocatedBytes": 61232, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate", + "operation": "progress", + "historyCount": 100, + "iterations": 21, + "medianMilliseconds": 0.0643, + "medianAllocatedBytes": 9128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate", + "operation": "list", + "historyCount": 1000, + "iterations": 21, + "medianMilliseconds": 0.6104, + "medianAllocatedBytes": 188400, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate", + "operation": "progress", + "historyCount": 1000, + "iterations": 21, + "medianMilliseconds": 0.1047, + "medianAllocatedBytes": 9128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate", + "operation": "list", + "historyCount": 10000, + "iterations": 7, + "medianMilliseconds": 5.2518, + "medianAllocatedBytes": 1364296, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate", + "operation": "progress", + "historyCount": 10000, + "iterations": 7, + "medianMilliseconds": 0.1774, + "medianAllocatedBytes": 9128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate", + "operation": "list", + "historyCount": 50001, + "iterations": 7, + "medianMilliseconds": 17.0455, + "medianAllocatedBytes": 6241400, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate", + "operation": "progress", + "historyCount": 50001, + "iterations": 7, + "medianMilliseconds": 0.1606, + "medianAllocatedBytes": 9128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + } + ] +} diff --git a/docs/background-tasks.md b/docs/background-tasks.md new file mode 100644 index 0000000000..8567e29915 --- /dev/null +++ b/docs/background-tasks.md @@ -0,0 +1,159 @@ +# Background tasks + +Background tasks let users follow work performed by `WorkItemJob`: bulk actions, project data resets, and sample-data generation. The task record describes progress; the work queue controls delivery. This guide covers the application contracts and the boundary for a future Foundatio extraction. + +## Ownership and persistence + +| Component | Owns | Source | +| --- | --- | --- | +| `BackgroundTaskService` | Cached records, organization indexes, serialized updates, cancellation requests, progress notifications | [Service](../src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs) | +| `WorkItemJob` and its queue | Delivery, acknowledgment, work-item retries, execution locks | [Registration](../src/Exceptionless.Core/Bootstrapper.cs) | +| `BackgroundTaskTrackingWorkItemHandler` | Progress and lifecycle tracking around an existing ordinary work-item handler | [Tracking wrapper](../src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs) | +| `BulkOperationService` and `BulkOperationWorkItemHandler` | Destructive-operation admission, batch execution, checkpoints, bounded continuation, selection cleanup | [Admission](../src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs), [worker](../src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs) | +| Selection resolver and action handler | Event membership, current access/retention rules, and the action's domain side effects | [Contracts](../src/Exceptionless.Core/Services/BulkOperations/IBulkOperationHandler.cs) | +| HTTP and WebSocket adapters | Authentication, organization access, public projections, client discovery and reconciliation | [API handler](../src/Exceptionless.Web/Api/Handlers/BackgroundTaskHandler.cs), [client](../src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts) | + +The task record is the source of truth for saved progress. Counts and the bulk cursor share one record and are saved together. The queue payload carries the task identifier and the information required by its handler. Notifications are best effort; polling reads the saved state when a notification is delayed or lost. + +Task records use the configured `ICacheClient` and expire seven days after their last save. They provide limited history rather than a durable audit log. Index entries have independent per-member expiration. Every active update renews active membership; a terminal update renews finished-history membership. This keeps long-running work discoverable without rewriting the larger history index on every progress report. The service also refreshes the index key's lifetime because renewing an existing member does not extend the key in every provider. + +Record and index writes are separate operations. A cache failure can leave incomplete discovery even when a record was saved; a no-op mutation does not repair every prior index failure. History refresh precedes active removal so normal completion has overlapping membership and a failed refresh preserves the existing fallback. Treat this cache-backed history as best effort, and use a store with stronger indexing guarantees if a future consumer requires transactional discovery. + +## Creating and updating a task + +Ordinary callers use `CreateAndEnqueueAsync`. Authorization and project validation happen before this call. The factory runs after the task has its identifier, so the work item can reference it: + +```csharp +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Services.BackgroundTasks; + +static Task QueueSampleDataAsync( + BackgroundTaskService tasks, + string organizationId, + string projectId, + string userId) +{ + return tasks.CreateAndEnqueueAsync(new BackgroundTask + { + OrganizationId = organizationId, + ProjectId = projectId, + Type = BackgroundTaskTypes.GenerateProjectSampleData, + Description = "Generate sample data", + CreatedByUserId = userId + }, task => new GenerateSampleEventsWorkItem + { + OrganizationId = organizationId, + ProjectId = projectId, + TaskId = task.Id, + EventCount = 250, + DaysBack = 7 + }); +} +``` + +This uses the same work item as [SampleDataService](../src/Exceptionless.Core/Utility/SampleDataService.cs). `CanCancel` defaults to false because this handler cannot stop midway safely. Set it to true only for an implementation that honors cancellation. + +| Service method | Contract | +| --- | --- | +| `CreateAndEnqueueAsync` | Creates and indexes a queued task, then prepares and enqueues its work item. Factory or queue failures fail a still-queued task. | +| `GetAsync` | Returns the stored record, or null when absent. Internal callers must enforce their own access policy. | +| `GetByIdsAsync` | Returns existing tasks once per ID in input order, omitting missing records. Cache reads contain at most 50 records; callers enforce scope. | +| `GetByOrganizationAsync` | Returns all discoverable active tasks plus up to `limit` finished tasks, newest first; the default finished limit is 50. | +| `UpdateAsync` | Locks one task, reloads it, runs a synchronous mutation, and saves only when the callback returns true. Returns null when the record is absent. | +| `CancelAsync` | Records a cancellation request and returns the current task outcome, including a distinct non-cancellable result. | +| `IsCancelRequestedAsync` | Lets a cooperative worker check the persisted cancellation request at a safe boundary. | + +`AddAsync` followed by `EnqueueAsync` is the lower-level path used by bulk admission, where selection state must be saved before enqueueing. Keep that sequence inside the owning operation service. Raw persistence is private so callers use the locking, indexing, and notification path. + +Task storage and queue submission are separate operations. An enqueue exception can occur after the queue accepted the item; cleanup changes only a task that is still queued and preserves an outcome already recorded by the worker. Keep work-item factories free of domain side effects. The returned creation snapshot may already be older than worker progress by the time it reaches the client. + +An `UpdateAsync` callback should only inspect and change the supplied task. Perform expensive work outside it, then checkpoint the result. The service owns `UpdatedUtc`: every committed mutation receives a timestamp strictly newer than the previous record, including when a clock repeats or moves backward. The persistence lock is released before publishing; a slow message bus does not hold up another mutation. + +## Lifecycle and retry rules + +`IsActive` means `Queued` or `Running`. The worker owns valid transitions for its task type; the service supplies serialized mutations. + +| Event | Saved behavior | +| --- | --- | +| Accepted submission | `Queued`, with creation and update timestamps | +| Work starts or reports progress | `Running`; the first start timestamp is retained | +| Successful completion | `Completed`, completion timestamp, progress 100 | +| Ordinary handler throws | `Failed` with a generic public message; the queue may retry it | +| Ordinary queue retries a failed task | `Running` again; completion time, message, and percent are reset, and active membership is restored | +| Bulk transient error | Keep the task active and continue from its saved cursor; three consecutive transient failures end it as `Failed` | +| Bulk permanent error, lost selection, or exhausted checkpoint retries | `Failed`; the task does not reopen automatically | +| Duplicate delivery of completed or cancelled tracked work | Skip execution | + +A non-success ordinary work-item result, such as shutdown, leaves tracking active so queue delivery can resume it. Bulk shutdown also retains its saved progress and selection. An ordinary work item with no `TaskId` runs without tracking. If an ordinary task record disappears, its existing queue payload can still run; a bulk task requires its saved record and selection and stops if either is lost. + +`Failed` therefore describes a failed attempt for an ordinary tracked work item and a terminal outcome for a bulk operation. Consumers must allow a newer ordinary-work update to move `Failed` back to `Running`. Retry policy belongs to the queue or bulk worker, not to a status label alone. + +## Cancellation and staleness + +Cancellation is cooperative. A queued cancellable task can be marked `Cancelled` immediately. A running bulk task checks the request between batches, finishes the batch already in flight, then records `Cancelled`. Completed side effects are retained. + +`StaleAfter` is two hours without a saved update. Staleness is computed when needed; it is not a separate status or a timer that automatically kills work. An abandoned task can be cleared through cancellation even when its type normally cannot be stopped. Bulk admission can retire an abandoned conflicting operation before accepting a replacement. + +Cancelling a finished task is idempotent. A successful cancellation request can still return `Running` while the worker finishes its current batch. Continue observing the task until its state changes. The task snapshot does not expose a separate cancellation-pending field. + +## HTTP and notification contracts + +All routes are under `/api/v2`. Task lookup and cancellation enforce organization access; inaccessible records return the same not-found response as missing records. + +| Route | Result | +| --- | --- | +| `POST /bulk-operations` | HTTP 202 with the accepted task; conflicting destructive work returns 409 | +| `GET /tasks/{id}` | One task snapshot; missing or inaccessible tasks return 404 | +| `GET /organizations/{id}/tasks` | All active tasks plus the newest 50 finished tasks | +| `POST /tasks/{id}/cancel` | HTTP 202 with the current snapshot; a live non-cancellable task returns 422 | + +The HTTP [view model](../src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs) includes status, counts, message, creator, timestamps, and a selection summary for bulk work. That summary can contain the filter, original time expression, resolved UTC bounds, and explicit ID count. It omits the ID array, operation parameters, pinned membership, snapshot identifier, and resume cursor. The smaller [BackgroundTaskUpdated message](../src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs) also omits the selection summary. + +Messages can be duplicated, delayed, or delivered out of order. Compare `UpdatedUtc` at .NET tick precision before applying them; JavaScript's `Date` alone discards submillisecond precision. Unknown task identifiers trigger rediscovery. The client polls while tasks are active and refetches when returning to an organization. A message is a hint to reconcile saved state, not an instruction to execute work. + +## Bulk selection and execution + +Explicit ID selection accepts at most 1,000 distinct IDs. Larger requests use a filter. The resolver captures filter membership during preparation: a complete small selection keeps fixed IDs, while a larger selection keeps an Elasticsearch point-in-time snapshot and search-after cursor. Relative dates use a fixed UTC anchor and invariant formatting. + +For filter selections, current organization/project scope, search-plan rules, and retention can narrow the eligible set. Later arrivals or edits cannot expand it. Missing, expired, or legacy selection state fails safely and requires starting a new operation; the worker does not silently re-run a live filter. Empty selections have an explicit persisted representation because the application serializer omits empty collections. + +Preparation runs before the organization admission lock. Destructive bulk operations that overlap the same project cannot be admitted together; an organization-wide operation overlaps every project. Each batch executes at most the configured number of IDs, with a default batch size of 500. Event deletion fetches only identity, daily-index routing, organization, project, and stack fields, then uses the same deletion service as individual deletes. + +The snapshot has a two-hour idle lease, renewed while paging. Terminal or rejected work releases it best effort; active retries and shutdown retain it. Expiry bounds the resource when cleanup cannot reach the backend. + +Deletion and checkpoint persistence are separate writes. The worker retries saving a known batch result before considering another execution. A process can still stop after a side effect but before its checkpoint is saved, so handlers must tolerate replay. Missing items are reported as failed; counts are not an exactly-once ledger of external side effects. + +## Adding work + +For an ordinary tracked job: + +1. Implement the existing work-item handler and give its work-item type an optional `TaskId` through `IBackgroundTaskWorkItem`. +2. Register it through `TrackWithBackgroundTask` in `Bootstrapper`, following reset-data or sample-data generation. +3. Create it through `CreateAndEnqueueAsync` after authorization, and report progress through `WorkItemContext.ReportProgressAsync` so execution-lock renewal still happens. +4. Define which side effects may replay, whether cancellation is safe, and which queue failures are retryable. + +For a bulk action, implement `IBulkOperationHandler` with a unique operation type, authorization, parameter validation, description, and bounded batch execution. Reuse the existing entity resolver when membership semantics match. A new entity family also needs an `IBulkSelectionResolver` that prepares stable membership, returns opaque resumable cursors, and releases its resources. Keep all domain side effects in the existing domain service. + +## Scaling and a future Foundatio boundary + +Keep immutable selection data separate from frequently updated progress where the existing format permits it. Process one batch at a time, project only required fields, and use the cache's bulk-read operation for groups of task records. Listing reads each index once, orders the IDs, and loads records in batches of at most 50. It has no silent history-page cutoff. Conflict detection uses the same bounded record reads while checking the active index. + +Returning all active tasks inherently requires work proportional to that active set. The generic cache-list API does not provide a portable ordered recent-history query, so listing still reads and orders the live history IDs. `GetAllAsync` lets providers combine reads; it does not promise one network round trip on every provider or Redis Cluster topology. Active-index renewal adds writes to a progress update, while the history index is renewed only on terminal updates. Measure these operations separately when changing retention or storage. + +| Reusable responsibility | Keep in the application adapter | +| --- | --- | +| Task identity, status, progress, timestamps, and serialized state updates | ObjectId HTTP routes and Exceptionless model interfaces | +| Store retention, discovery, bounded reads, and cancellation signaling | Organization/project/user authorization and admission policy | +| Worker tracking and progress reporting using existing Foundatio primitives | Event filters, plan limits, retention policy, Elasticsearch snapshots, deletion accounting | +| Optional advisory change notifications | Public HTTP/WebSocket projections, UI text, and task-panel behavior | + +The current `BackgroundTask` includes organization/project/user fields and `BulkOperationDetails`; it is still an application model. Preserve that honest boundary until another consumer establishes the common contract. An extraction should define identity, owner scope, private checkpoint serialization, retention/index consistency, and retry ownership explicitly, while reusing `ICacheClient`, queues, locks, messaging, resilience, and `TimeProvider`. Introduce a storage abstraction when it is needed to offer indexed discovery or durability that the cache contract cannot provide. Avoid a generic payload bag or a parallel queue/state-machine framework solely to anticipate reuse. + +## Verification and operations + +Service tests use real Foundatio in-memory infrastructure and controlled time. Regression coverage belongs at the failure boundary: factory/index/queue failures, concurrent updates, delayed notifications, cancellation, retention, full selection paging, and checkpoint recovery. Integration tests exercise real serialization, Elasticsearch routing, authorization, usage accounting, and selection mutation. Browser tests cover discovery, concurrent task completion, and cancellation access. + +Start with the focused classes under [BackgroundTasks](../tests/Exceptionless.Tests/Services/BackgroundTasks) and [BulkOperations](../tests/Exceptionless.Tests/Services/BulkOperations), then the [worker tests](../tests/Exceptionless.Tests/Jobs/WorkItemHandlers), endpoint tests, and the full CI run. Manual performance measurements live in [benchmarks/BackgroundTasks](../benchmarks/BackgroundTasks); they report service time, allocation, and cache calls separately from provider/network throughput. + +Deploy the updated job before or with the web application. Cache loss can remove progress and prepared selections; treat that as an interrupted operation requiring user-visible recovery. This feature adds no database migration or index. Preserve public DTOs and message shapes when changing internal storage or moving reusable components to Foundatio. From 4e2826f0893c436e94d1fdac8a551c77a78866c0 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 22:26:55 -0600 Subject: [PATCH 33/42] Keep background task architecture guide out of the public site --- docs/_config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_config.ts b/docs/_config.ts index 20f2af36dc..3667a21c17 100644 --- a/docs/_config.ts +++ b/docs/_config.ts @@ -15,6 +15,7 @@ site.ignore( ".gitignore", ".npmrc", "_site", + "background-tasks.md", "deno.json", "deno.lock", "news/index.md", From 01b02c3b8099618abc740635653645b6f5b9f064 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 22:43:29 -0600 Subject: [PATCH 34/42] Wait for dependent event metrics before browser remounts --- .../e2e/support/api-request-monitor.ts | 8 ++++--- .../e2e/tests/event-effects-chaos.e2e.ts | 21 ++++++++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts b/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts index d6493e25c0..758aeb9fd8 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/support/api-request-monitor.ts @@ -38,10 +38,12 @@ export function waitForEventDetailResponses(page: Page, eventId: string, stackId const url = new URL(response.url()); return url.pathname === `/api/v2/events/${eventId}` && url.searchParams.get('expected_stack_id') === stackId; }); - const projectCountResponse = page.waitForResponse((response) => { + return Promise.all([eventResponse, waitForProjectUserCountResponse(page, projectId)]); +} + +export function waitForProjectUserCountResponse(page: Page, projectId: string): Promise { + return page.waitForResponse((response) => { const url = new URL(response.url()); return url.pathname === `/api/v2/projects/${projectId}/events/count` && url.searchParams.get('aggregations') === 'cardinality:user'; }); - - return Promise.all([eventResponse, projectCountResponse]); } diff --git a/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts b/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts index 6ecf710fb7..07a85b557e 100644 --- a/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts +++ b/src/Exceptionless.Web/ClientApp/e2e/tests/event-effects-chaos.e2e.ts @@ -1,7 +1,7 @@ import type { ConsoleMessage, Page, Request, Response } from '@playwright/test'; import { expect, test } from '../fixtures/e2e-test'; -import { isApplicationApiRequest, trackPendingApiRequests, waitForEventDetailResponses } from '../support/api-request-monitor'; +import { isApplicationApiRequest, trackPendingApiRequests, waitForEventDetailResponses, waitForProjectUserCountResponse } from '../support/api-request-monitor'; import { ExceptionlessE2EJourney } from '../support/exceptionless-journey'; import { createRepresentativeEvent } from '../support/synthetic-event'; import { @@ -206,9 +206,19 @@ test('event list and detail effects stay bounded through paging and background c await measureAction(diagnostics, 'stack detail discovery route remounts', async () => { for (let index = 0; index < 5; index++) { + // Arrange await pendingApiRequests.waitForPendingRequests(); + // Overview can render before the dependent project metric request starts. + const projectCountResponse = waitForProjectUserCountResponse(page, e2eScenario.projectId); + + // Act await page.goto(`/stack/${journey.stackId}`); + + // Assert await expect(page.getByRole('tab', { name: 'Overview' })).toBeVisible(); + const response = await projectCountResponse; + expect(response.ok()).toBe(true); + expect(await response.finished()).toBeNull(); } }); expect(actionSample(diagnostics, 'stack detail discovery route remounts').eventDetails).toBeLessThanOrEqual(10); @@ -217,9 +227,18 @@ test('event list and detail effects stay bounded through paging and background c await measureAction(diagnostics, 'canonical stack event route remounts', async () => { for (let index = 0; index < 5; index++) { + // Arrange await pendingApiRequests.waitForPendingRequests(); + const projectCountResponse = waitForProjectUserCountResponse(page, e2eScenario.projectId); + + // Act await page.goto(`/stack/${journey.stackId}/event/${journey.eventId}`); + + // Assert await expect(page.getByRole('tab', { name: 'Overview' })).toBeVisible(); + const response = await projectCountResponse; + expect(response.ok()).toBe(true); + expect(await response.finished()).toBeNull(); } }); expect(actionSample(diagnostics, 'canonical stack event route remounts').eventDetails).toBeLessThanOrEqual(10); From 54f7cf264bbeb3f6a79227edd5e1c3972d56bf55 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Fri, 13 Nov 2026 22:55:37 -0600 Subject: [PATCH 35/42] Synchronize deletion assertions with search visibility --- docs/background-tasks.md | 2 ++ .../Api/Endpoints/BulkOperationEndpointTests.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/background-tasks.md b/docs/background-tasks.md index 8567e29915..04484e5b53 100644 --- a/docs/background-tasks.md +++ b/docs/background-tasks.md @@ -122,6 +122,8 @@ Preparation runs before the organization admission lock. Destructive bulk operat The snapshot has a two-hour idle lease, renewed while paging. Terminal or rejected work releases it best effort; active retries and shutdown retain it. Expiry bounds the resource when cleanup cannot reach the backend. +Completion does not force an Elasticsearch index refresh. Deletion uses the repository's normal eventual search consistency, so search results can briefly retain deleted events until the index refreshes. + Deletion and checkpoint persistence are separate writes. The worker retries saving a known batch result before considering another execution. A process can still stop after a side effect but before its checkpoint is saved, so handlers must tolerate replay. Missing items are reported as failed; counts are not an exactly-once ledger of external side effects. ## Adding work diff --git a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs index f2adebaf3f..b24c5b9a0d 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/BulkOperationEndpointTests.cs @@ -225,6 +225,8 @@ public async Task Create_IdSelection_DeletesTheIdsThroughTheSameSideEffectsAsThe var created = await CreateOperationAsync(request); await _workItemJob.RunUntilEmptyAsync(TestCancellationToken); var finished = await GetOperationAsync(created.Id); + // Missing ids fall back to a search across daily indexes, which must observe the completed deletions. + await RefreshDataAsync(); var remaining = await _eventRepository.GetByIdsAsync(events.Select(e => e.Id).ToArray()); var usage = await usageService.GetUsageAsync(SampleDataService.TEST_ORG_ID, SampleDataService.TEST_PROJECT_ID); From 826d4f43da94e30379ebc9a725c57889f180bb3f Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sat, 14 Nov 2026 16:56:53 -0600 Subject: [PATCH 36/42] Harden task state and bulk recovery boundaries --- .../BackgroundTaskTrackingWorkItemHandler.cs | 6 +- .../BulkOperationWorkItemHandler.cs | 15 +++ .../BackgroundTasks/BackgroundTaskService.cs | 50 +++++++--- .../BulkOperations/BulkOperationService.cs | 50 ++++++++-- ...kgroundTaskTrackingWorkItemHandlerTests.cs | 80 +++++++++++++++- .../BulkOperationWorkItemHandlerTests.cs | 93 ++++++++++++++++++- .../BackgroundTaskServiceTests.cs | 62 ++++++++++++- .../BulkOperationServiceTests.cs | 72 +++++++++++++- 8 files changed, 394 insertions(+), 34 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs index 54c4d4c157..45459c3070 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs @@ -155,17 +155,13 @@ private async Task StartAsync(string taskId) return result; } - // Tracking is best effort: a cache or lock problem must not fail the work the user asked for. + // Tracking is best effort and uses no caller cancellation token. A transport failure must not replace the worker's outcome. private async Task TryAsync(string taskId, Func action) { try { await action(); } - catch (OperationCanceledException) - { - throw; - } catch (Exception ex) { Log.LogWarning(ex, "Unable to update background task {TaskId}: {Message}", taskId, ex.Message); diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs index 7b8e0b8283..2ea69febab 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BulkOperationWorkItemHandler.cs @@ -113,6 +113,8 @@ public override async Task HandleItemAsync(WorkItemContext context) await FinishAsync(workItem, BackgroundTaskStatus.Failed, "The saved selection expired, so the operation could not continue."); return; } + + await _service.RefreshSelectionExpirationAsync(task); } task = await _taskService.UpdateAsync(task.Id, current => @@ -145,6 +147,18 @@ public override async Task HandleItemAsync(WorkItemContext context) return; } + // Admission can retire a stale worker while it is between batches. Observe the saved outcome before + // starting another action, including when the old worker resumes after its replacement was admitted. + task = await _taskService.GetAsync(workItem.TaskId); + if (task is null) + { + await _service.PublishLostAsync(workItem); + return; + } + + if (!task.IsActive) + return; + if (await _taskService.IsCancelRequestedAsync(task.Id)) { await FinishAsync(workItem, BackgroundTaskStatus.Cancelled, $"Cancelled by user after {task.Processed:N0} of {task.Total:N0}."); @@ -172,6 +186,7 @@ public override async Task HandleItemAsync(WorkItemContext context) if (!task.IsActive) return; + await _service.RefreshSelectionExpirationAsync(task); await context.ReportProgressAsync(GetPercent(task), $"{task.Processed:N0} of {task.Total:N0}"); if (page.NextCursor is null) diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs index cd9773fe7f..2224e29333 100644 --- a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -101,30 +101,46 @@ public async Task> GetByIdsAsync(IEnumerable public async Task> GetByOrganizationAsync(string organizationId, int limit = ListLimit) { + limit = Math.Max(0, limit); var activeIds = (await GetActiveIdsAsync(organizationId)).ToHashSet(StringComparer.Ordinal); // One snapshot avoids repeated full-list cloning in memory and offset shifts while entries expire. var indexed = await _cache.GetListAsync(GetOrganizationIndexCacheKey(organizationId)); - // Ids are ObjectIds, so descending order is newest first no matter how the cache orders its list. + // ObjectIds order whole seconds; their machine/process bytes can reverse creation order within a second. var newestIds = (indexed.HasValue ? indexed.Value : []) .Concat(activeIds) .Distinct(StringComparer.Ordinal) .OrderByDescending(id => id, StringComparer.Ordinal); var tasks = new List(); + var finished = new List(); var expired = new List(); - int finishedCount = 0; - foreach (var batch in newestIds.Where(id => finishedCount < limit || activeIds.Contains(id)).Chunk(ReadBatchSize)) + string? cutoffSecond = null; + // Read the entire cutoff second before choosing its newest tasks by CreatedUtc, including across cache batches. + foreach (var batch in newestIds.Where(id => finished.Count < limit || activeIds.Contains(id) + || (cutoffSecond is not null && id.StartsWith(cutoffSecond, StringComparison.Ordinal))).Chunk(ReadBatchSize)) { var found = await GetByIdsAsync(batch); expired.AddRange(batch.Except(found.Select(task => task.Id), StringComparer.Ordinal)); foreach (var task in found) { - if (!String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal) || (!task.IsActive && finishedCount >= limit)) + if (!String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal)) continue; - tasks.Add(task); - if (!task.IsActive) - finishedCount++; + if (task.IsActive) + tasks.Add(task); + else + { + finished.Add(task); + if (finished.Count == limit) + cutoffSecond = task.Id[..8]; + } + } + + // A burst within one second must not retain every task's payload while its remaining batches are read. + if (finished.Count > limit) + { + finished.Sort(CompareNewestFirst); + finished.RemoveRange(limit, finished.Count - limit); } } @@ -134,7 +150,9 @@ public async Task> GetByOrganizationAsync(string o await RemoveFromActiveIndexAsync(organizationId, expired); } - return tasks.OrderByDescending(t => t.CreatedUtc).ThenByDescending(t => t.Id, StringComparer.Ordinal).ToList(); + tasks.AddRange(finished); + tasks.Sort(CompareNewestFirst); + return tasks; } /// The ids of the organization's tasks that have not finished. @@ -255,7 +273,10 @@ await _lockProvider.AcquireAsync(GetUpdateLockResource(id), TimeSpan.FromSeconds var previousUpdatedUtc = task.UpdatedUtc; if (!update(task)) - return task; + { + // A declined mutation returns the stored state even if the callback changed its copy. + return await GetAsync(id); + } // Clients use this stamp to reject delayed notifications, including when a clock repeats or moves backwards. var utcNow = _timeProvider.GetUtcNow().UtcDateTime; @@ -356,12 +377,9 @@ public async Task PublishAsync(BackgroundTaskUpdated message) { await _publisher.PublishAsync(message); } - catch (OperationCanceledException) - { - throw; - } catch (Exception ex) { + // No caller token is passed, so a transport cancellation is also an advisory notification failure. _logger.LogWarning(ex, "Failed to publish background task update for {TaskId}: {Message}", message.Id, ex.Message); } } @@ -385,6 +403,12 @@ private async Task RefreshIndexAsync(string key, string id) // Refreshing an existing list member does not extend the key TTL in every cache provider. await _cache.SetExpirationAsync(key, TaskLifetime); } + + private static int CompareNewestFirst(BackgroundTask left, BackgroundTask right) + { + int byCreated = right.CreatedUtc.CompareTo(left.CreatedUtc); + return byCreated != 0 ? byCreated : StringComparer.Ordinal.Compare(right.Id, left.Id); + } } public record BackgroundTaskCancelResult diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index 11b9a4c468..8911aa4344 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -26,6 +26,7 @@ public class BulkOperationOptions ///
public class BulkOperationService { + private static readonly TimeSpan CreateLockLifetime = TimeSpan.FromSeconds(30); private readonly BackgroundTaskService _taskService; private readonly ICacheClient _cache; private readonly ILockProvider _lockProvider; @@ -62,6 +63,14 @@ public BulkOperationService(BackgroundTaskService taskService, ICacheClient cach return cached.HasValue ? cached.Value : null; } + /// Keeps an active task's immutable selection available without rewriting its payload. + public Task RefreshSelectionExpirationAsync(BackgroundTask task) + { + return task.Bulk?.Selection.Kind is BulkOperationSelectionKind.Filter + ? _cache.SetExpirationAsync(GetPinnedSelectionCacheKey(task.Id), BackgroundTaskService.TaskLifetime) + : Task.CompletedTask; + } + /// Releases a rejected or terminal task's selection without changing its durable outcome. public async Task ReleaseSelectionAsync(BackgroundTask task, string? pinnedSelection = null, string? cursor = null) { @@ -82,7 +91,7 @@ public async Task ReleaseSelectionAsync(BackgroundTask task, string? pinnedSelec /// /// Stores and enqueues the task unless it conflicts with an active destructive task on the same project. - /// The check and the write happen under a per-organization lock so two requests cannot both pass it. + /// Admission uses a renewed per-organization lease; a scan that outlives its lease returns busy. /// public async Task CreateAsync(BackgroundTask task) { @@ -117,17 +126,40 @@ public async Task CreateAsync(BackgroundTask task) task.Description = handler.Describe(task); - await using var createLock = await _lockProvider.TryAcquireAsync(BackgroundTaskService.GetCreateLockResource(task.OrganizationId), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(10)); + long leaseStarted = _timeProvider.GetTimestamp(); + await using var createLock = await _lockProvider.TryAcquireAsync(BackgroundTaskService.GetCreateLockResource(task.OrganizationId), CreateLockLifetime, TimeSpan.FromSeconds(10)); if (createLock is null) return BulkOperationCreateResult.Busy(); + async Task RenewCreateLockAsync() + { + long renewalStarted = _timeProvider.GetTimestamp(); + if (_timeProvider.GetElapsedTime(leaseStarted, renewalStarted) >= CreateLockLifetime) + return false; + + await createLock.RenewAsync(CreateLockLifetime); + if (_timeProvider.GetElapsedTime(leaseStarted) >= CreateLockLifetime) + return false; + + leaseStarted = renewalStarted; + return true; + } + + // Start conservatively before acquisition, then establish a fresh lease. A slow scan may renew it, but an + // expired request must not admit work from its earlier snapshot after another creator has acquired the lock. + if (!await RenewCreateLockAsync()) + return BulkOperationCreateResult.Busy(); + if (handler.IsDestructive) { - var conflict = await FindConflictingTaskAsync(task); + var conflict = await FindConflictingTaskAsync(task, RenewCreateLockAsync); if (conflict is not null) - return BulkOperationCreateResult.Conflicting(conflict); + return conflict; } + if (!await RenewCreateLockAsync()) + return BulkOperationCreateResult.Busy(); + await _taskService.AddAsync(task); enqueueAttempted = true; await _taskService.EnqueueAsync(task, new BulkOperationWorkItem @@ -193,7 +225,7 @@ public Task PublishLostAsync(BulkOperationWorkItem workItem) }); } - private async Task FindConflictingTaskAsync(BackgroundTask candidate) + private async Task FindConflictingTaskAsync(BackgroundTask candidate, Func> renewCreateLock) { // Only active tasks can conflict, and every one of them is in the active index no matter how much history // the organization has. @@ -203,6 +235,9 @@ public Task PublishLostAsync(BulkOperationWorkItem workItem) var activeIds = await _taskService.GetActiveIdsAsync(candidate.OrganizationId); foreach (var batch in activeIds.Chunk(BackgroundTaskService.ReadBatchSize)) { + if (!await renewCreateLock()) + return BulkOperationCreateResult.Busy(); + var tasks = await _taskService.GetByIdsAsync(batch); finished.AddRange(batch.Except(tasks.Where(task => task.IsActive).Select(task => task.Id), StringComparer.Ordinal)); foreach (var active in tasks.Where(task => task.IsActive)) @@ -218,6 +253,9 @@ public Task PublishLostAsync(BulkOperationWorkItem workItem) if (utcNow - existing.UpdatedUtc >= BackgroundTaskService.StaleAfter) { + if (!await renewCreateLock()) + return BulkOperationCreateResult.Busy(); + // Retire it before admitting a replacement, so a delayed delivery cannot run it alongside the new one. var retired = await _taskService.UpdateAsync(existing.Id, current => { @@ -241,7 +279,7 @@ public Task PublishLostAsync(BulkOperationWorkItem workItem) if (finished.Count > 0) await _taskService.RemoveFromActiveIndexAsync(candidate.OrganizationId, finished); - return conflict; + return conflict is null ? null : BulkOperationCreateResult.Conflicting(conflict); } } diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs index 805092d8a2..9ac6bc9a8d 100644 --- a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs @@ -3,6 +3,7 @@ using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Services.BackgroundTasks; using Exceptionless.Tests.Utility; +using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Lock; using Foundatio.Utility; @@ -60,6 +61,70 @@ await handler.HandleItemAsync(CreateContext(task.Id, (progress, message) => Assert.Contains(_harness.Publisher.Updates, u => u.Status == BackgroundTaskStatus.Running && u.Progress == 50); } + [Fact] + public async Task HandleItemAsync_ProgressTrackingIsCancelled_StillReportsProgressAndCompletesTheWork() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => new CancellingTaskCacheClient(inner, task => task.Status is BackgroundTaskStatus.Running && task.Progress == 50)); + var task = await AddTaskAsync(harness: harness); + bool continued = false; + var reported = new List<(int Progress, string? Message)>(); + var handler = CreateHandler(async context => + { + await context.ReportProgressAsync(50, "Halfway"); + continued = true; + }, harness); + var context = CreateContext(task.Id, (progress, message) => + { + reported.Add((progress, message)); + return Task.CompletedTask; + }); + + // Act + var exception = await Record.ExceptionAsync(() => handler.HandleItemAsync(context)); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Null(exception); + Assert.True(continued); + Assert.Equal([(50, "Halfway")], reported); + Assert.Equal(BackgroundTaskStatus.Completed, stored!.Status); + } + + [Fact] + public async Task HandleItemAsync_JobProgressCallbackIsCancelled_RethrowsCancellation() + { + // Arrange + var task = await AddTaskAsync(); + var failure = new TaskCanceledException("The job progress callback was cancelled"); + var handler = CreateHandler(context => context.ReportProgressAsync(50, "Halfway")); + var context = CreateContext(task.Id, (_, _) => Task.FromException(failure)); + + // Act + var thrown = await Record.ExceptionAsync(() => handler.HandleItemAsync(context)); + + // Assert + Assert.Same(failure, thrown); + } + + [Fact] + public async Task HandleItemAsync_FailureTrackingIsCancelled_RethrowsTheOriginalHandlerException() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => new CancellingTaskCacheClient(inner, task => task.Status is BackgroundTaskStatus.Failed)); + var task = await AddTaskAsync(harness: harness); + var failure = new InvalidOperationException("The work failed"); + var handler = CreateHandler(_ => Task.FromException(failure), harness); + + // Act + var thrown = await Record.ExceptionAsync(() => handler.HandleItemAsync(CreateContext(task.Id))); + + // Assert + Assert.Same(failure, thrown); + } + [Fact] public async Task HandleItemAsync_WorkSucceeds_CompletesTheTaskAtOneHundredPercentWithTheLastMessage() { @@ -275,9 +340,9 @@ private async Task AddTaskAsync(bool canCancel = false, Backgrou return task; } - private BackgroundTaskTrackingWorkItemHandler CreateHandler(Func work) + private BackgroundTaskTrackingWorkItemHandler CreateHandler(Func work, BackgroundTaskTestHarness? harness = null) { - return new BackgroundTaskTrackingWorkItemHandler(new DelegateWorkItemHandler(work), _harness.Tasks, _time); + return new BackgroundTaskTrackingWorkItemHandler(new DelegateWorkItemHandler(work), (harness ?? _harness).Tasks, _time); } private WorkItemContext CreateContext(string taskId, Func? progress = null) @@ -285,4 +350,15 @@ private WorkItemContext CreateContext(string taskId, Func? p var workItem = new ResetProjectDataWorkItem { OrganizationId = OrganizationId, ProjectId = ProjectId, TaskId = taskId }; return new WorkItemContext(workItem, "job", EmptyLock.Empty, TestCancellationToken, progress ?? ((_, _) => Task.CompletedTask)); } + + private sealed class CancellingTaskCacheClient(ICacheClient inner, Func shouldCancel) : ScopedCacheClient(inner, null), ICacheClient + { + Task ICacheClient.SetAsync(string key, T value, TimeSpan? expiresIn) + { + if (value is BackgroundTask task && shouldCancel(task)) + throw new TaskCanceledException("The task cache timed out"); + + return UnscopedCache.SetAsync(key, value, expiresIn); + } + } } diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs index 6c83237130..cac57a9277 100644 --- a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BulkOperationWorkItemHandlerTests.cs @@ -459,6 +459,95 @@ public async Task HandleItemAsync_TransientFailure_KeepsTheSelectionUntilContinu Assert.Single(resolver.Releases); } + [Fact] + public async Task HandleItemAsync_RetiredBetweenBatches_DoesNotExecuteAnotherBatch() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + var executedIds = new List(); + var operationHandler = new DelegateBulkHandler(batch => + { + executedIds.AddRange(batch); + return Task.FromResult(new BulkBatchResult(batch.Count, 0)); + }); + var resolver = new RemainingIdsResolver([]); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(["first", "second"])); + var handler = CreateHandler(harness, service, resolver, batchSize: 1); + BulkOperationCreateResult? replacement = null; + int reports = 0; + var context = CreateContext(created.Task!.Id, TestCancellationToken, async (_, _) => + { + if (++reports != 2) + return; + + // The first batch is saved, but this worker has not started the next one. + _time.Advance(BackgroundTaskService.StaleAfter + TimeSpan.FromMinutes(1)); + replacement = await service.CreateAsync(NewTask(["replacement"])); + }); + + // Act + await handler.HandleItemAsync(context); + var retired = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(replacement); + Assert.NotNull(replacement.Task); + Assert.NotNull(retired); + Assert.Equal(BackgroundTaskStatus.Failed, retired.Status); + Assert.Equal(1, retired.Processed); + Assert.Equal(["first"], executedIds); + Assert.Equal([replacement.Task.Id], await harness.Tasks.GetActiveIdsAsync(OrganizationId)); + } + + [Fact] + public async Task HandleItemAsync_ProgressExceedsSelectionLifetime_ResumesAfterShutdown() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log); + using var shutdown = new CancellationTokenSource(); + int batchesBeforeShutdown = (int)BackgroundTaskService.TaskLifetime.TotalHours + 1; + string[] ids = Enumerable.Range(0, batchesBeforeShutdown + 1).Select(index => index.ToString("D3")).ToArray(); + var remaining = ids.ToHashSet(StringComparer.Ordinal); + int batches = 0; + var operationHandler = new DelegateBulkHandler(batch => + { + batches++; + return Task.FromResult(new BulkBatchResult(batch.Count(remaining.Remove), 0)); + }); + var resolver = new RemainingIdsResolver(remaining); + var service = CreateService(harness, operationHandler, resolver); + var created = await service.CreateAsync(NewTask(ids, BulkOperationSelectionKind.Filter)); + var handler = CreateHandler(harness, service, resolver, batchSize: 1); + var context = CreateContext(created.Task!.Id, shutdown.Token, (_, _) => + { + if (batches > 0) + _time.Advance(TimeSpan.FromHours(1)); + if (batches == batchesBeforeShutdown) + shutdown.Cancel(); + + return Task.CompletedTask; + }); + + // Act + await handler.HandleItemAsync(context); + var interrupted = await harness.Tasks.GetAsync(created.Task.Id); + string? selection = await service.GetPinnedSelectionAsync(created.Task.Id); + await handler.HandleItemAsync(CreateContext(created.Task.Id, TestCancellationToken)); + var completed = await harness.Tasks.GetAsync(created.Task.Id); + + // Assert + Assert.NotNull(interrupted); + Assert.Equal(BackgroundTaskStatus.Running, interrupted.Status); + Assert.Equal(batchesBeforeShutdown, interrupted.Processed); + Assert.NotNull(selection); + Assert.NotNull(completed); + Assert.Equal(BackgroundTaskStatus.Completed, completed.Status); + Assert.Equal(ids.Length, completed.Succeeded); + Assert.Equal(0, completed.Failed); + Assert.Empty(remaining); + } + private BulkOperationService CreateService(BackgroundTaskTestHarness harness, IBulkOperationHandler handler, IBulkSelectionResolver resolver) { return new BulkOperationService(harness.Tasks, harness.Cache, harness.LockProvider, [handler], [resolver], _time); @@ -478,10 +567,10 @@ private BulkOperationWorkItemHandler CreateHandler(BackgroundTaskTestHarness har new BulkOperationOptions { BatchSize = batchSize, RetryDelay = retryDelay ?? TimeSpan.Zero }, harness.LockProvider, _time, Log); } - private static WorkItemContext CreateContext(string taskId, CancellationToken cancellationToken) + private static WorkItemContext CreateContext(string taskId, CancellationToken cancellationToken, Func? progressCallback = null) { var workItem = new BulkOperationWorkItem { TaskId = taskId, OrganizationId = OrganizationId }; - return new WorkItemContext(workItem, "test-job", EmptyLock.Empty, cancellationToken, static (_, _) => Task.CompletedTask); + return new WorkItemContext(workItem, "test-job", EmptyLock.Empty, cancellationToken, progressCallback ?? (static (_, _) => Task.CompletedTask)); } private static BackgroundTask NewTask(IReadOnlyCollection ids, BulkOperationSelectionKind kind = BulkOperationSelectionKind.Ids) diff --git a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs index 4821581548..397c707123 100644 --- a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs @@ -163,6 +163,45 @@ await Tasks.UpdateAsync(task.Id, current => Assert.Equal([ids[3], ids[2]], newestTwo.Select(t => t.Id).ToArray()); } + [Fact] + public async Task GetByOrganizationAsync_SameSecondHistoryAcrossBatches_ReturnsNewestCreatedTaskWithoutReadingOlderHistory() + { + // Arrange + RecordingReadCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new RecordingReadCacheClient(inner)); + var utcNow = _time.GetUtcNow().UtcDateTime; + var tasks = new List(); + for (int index = 0; index <= BackgroundTaskService.ReadBatchSize; index++) + { + var task = NewTask(); + // Different servers can create ObjectIds whose machine bytes reverse creation order within one second. + task.Id = new ObjectId(utcNow, machine: index + 1, pid: 1, increment: 0).ToString(); + task.CreatedUtc = utcNow.AddMilliseconds(BackgroundTaskService.ReadBatchSize - index); + task.UpdatedUtc = task.CreatedUtc; + task.Finish(BackgroundTaskStatus.Completed, "Done", task.CreatedUtc); + await harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(task.Id), task, BackgroundTaskService.TaskLifetime); + tasks.Add(task); + } + + var older = NewTask(); + older.Id = ObjectId.GenerateNewId(utcNow.AddSeconds(-1)).ToString(); + older.CreatedUtc = utcNow.AddSeconds(-1); + older.UpdatedUtc = older.CreatedUtc; + older.Finish(BackgroundTaskStatus.Completed, "Done", older.CreatedUtc); + await harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(older.Id), older, BackgroundTaskService.TaskLifetime); + await harness.Cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId), tasks.Select(task => task.Id).Append(older.Id), BackgroundTaskService.TaskLifetime); + + // Act + var listed = await harness.Tasks.GetByOrganizationAsync(OrganizationId, limit: 1); + + // Assert + Assert.Equal(tasks[0].Id, Assert.Single(listed).Id); + Assert.Equal(2, cache!.ReadBatches.Count); + Assert.All(cache.ReadBatches, batch => Assert.InRange(batch.Length, 1, BackgroundTaskService.ReadBatchSize)); + Assert.DoesNotContain(BackgroundTaskService.GetTaskCacheKey(older.Id), cache.ReadBatches.SelectMany(batch => batch)); + } + [Fact] public async Task GetByOrganizationAsync_HistoryExceedsFormerPageLimit_ReturnsNewestTask() { @@ -322,6 +361,7 @@ public async Task UpdateAsync_UpdateReturnsFalse_SavesAndPublishesNothing() // Assert Assert.NotNull(result); Assert.Equal(0, stored!.Progress); + Assert.Equal(stored.Progress, result.Progress); Assert.Equal(stored.CreatedUtc, stored.UpdatedUtc); Assert.Empty(_harness.Publisher.Updates); } @@ -493,13 +533,17 @@ public async Task UpdateAsync_UnknownTask_ReturnsNullWithoutRunningTheUpdate() Assert.Empty(_harness.Publisher.Updates); } - [Fact] - public async Task UpdateAsync_PublisherIsDown_StillSavesTheChange() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task UpdateAsync_PublisherIsDown_StillSavesTheChange(bool transportCancelled) { // Arrange var task = NewTask(); await Tasks.AddAsync(task); - _harness.Publisher.Failure = new TimeoutException("Message bus unavailable"); + _harness.Publisher.Failure = transportCancelled + ? new TaskCanceledException("Message bus timed out") + : new TimeoutException("Message bus unavailable"); // Act var result = await Tasks.UpdateAsync(task.Id, current => @@ -915,6 +959,18 @@ private sealed class ObservingHistoryCacheClient(ICacheClient inner) : ScopedCac } } + private sealed class RecordingReadCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + public List ReadBatches { get; } = []; + + Task>> ICacheClient.GetAllAsync(IEnumerable keys) + { + var batch = keys.ToArray(); + ReadBatches.Add(batch); + return UnscopedCache.GetAllAsync(batch); + } + } + private sealed class LostAcknowledgementQueue(InMemoryQueueOptions options) : InMemoryQueue(options) { public Func? AfterEnqueue { get; set; } diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs index 4396e5d3b9..1c3274e2e3 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs @@ -96,9 +96,9 @@ public async Task CreateAsync_DestructiveTaskActiveOnTheSameProject_Conflicts() public async Task CreateAsync_TaskCompletesDuringActiveIndexRead_StillFindsTheConflictingTask() { // Arrange - CompletingTaskOnReadCacheClient? cache = null; + ObservingTaskReadsCacheClient? cache = null; using var harness = new BackgroundTaskTestHarness(_time, Log, - decorateCache: inner => cache = new CompletingTaskOnReadCacheClient(inner)); + decorateCache: inner => cache = new ObservingTaskReadsCacheClient(inner)); var service = CreateService(harness); var completing = NewTask("test.safe", ProjectId); await harness.Tasks.AddAsync(completing); @@ -126,6 +126,62 @@ await harness.Tasks.UpdateAsync(completing.Id, current => Assert.Equal(BackgroundTaskStatus.Completed, (await harness.Tasks.GetAsync(completing.Id))!.Status); } + [Fact] + public async Task CreateAsync_LongConflictScan_KeepsTheAdmissionLeaseUntilTheTaskIsIndexed() + { + // Arrange + ObservingTaskReadsCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingTaskReadsCacheClient(inner)); + var service = CreateService(harness); + for (int index = 0; index < BackgroundTaskService.ReadBatchSize + 1; index++) + await harness.Tasks.AddAsync(NewTask("test.safe", ProjectId)); + + var heldDuringScan = new List(); + cache!.AfterBatchRead = async () => + { + _time.Advance(TimeSpan.FromSeconds(16)); + heldDuringScan.Add(await harness.LockProvider.IsLockedAsync(BackgroundTaskService.GetCreateLockResource(OrganizationId))); + }; + + // Act + var result = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + + // Assert + Assert.NotNull(result.Task); + Assert.Equal(2, heldDuringScan.Count); + Assert.All(heldDuringScan, held => Assert.True(held)); + Assert.Equal(1, (await harness.Queue.GetQueueStatsAsync()).Enqueued); + } + + [Fact] + public async Task CreateAsync_AdmissionReadOutlivesLease_DoesNotAdmitAfterAnotherCreator() + { + // Arrange + ObservingTaskReadsCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingTaskReadsCacheClient(inner)); + var service = CreateService(harness); + BulkOperationCreateResult? competing = null; + cache!.AfterRead = async () => + { + _time.Advance(TimeSpan.FromSeconds(31)); + competing = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + }; + + // Act + var expired = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + + // Assert + Assert.NotNull(competing); + Assert.NotNull(competing.Task); + Assert.True(expired.IsBusy); + Assert.Null(expired.Task); + Assert.Equal([competing.Task.Id], active); + Assert.Equal(1, (await harness.Queue.GetQueueStatsAsync()).Enqueued); + } + [Theory] [InlineData(null, ProjectId)] [InlineData(ProjectId, null)] @@ -367,9 +423,19 @@ private sealed class StubHandler(string type, bool isDestructive) : IBulkOperati public Task ExecuteBatchAsync(BackgroundTask task, IReadOnlyCollection ids, CancellationToken cancellationToken) => Task.FromResult(new BulkBatchResult(ids.Count, 0)); } - private sealed class CompletingTaskOnReadCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + private sealed class ObservingTaskReadsCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient { public Func? AfterRead { get; set; } + public Func? AfterBatchRead { get; set; } + + public new async Task>> GetAllAsync(IEnumerable keys) + { + var result = await base.GetAllAsync(keys); + if (AfterBatchRead is { } afterRead) + await afterRead(); + + return result; + } public new async Task>> GetListAsync(string key, int? page = null, int pageSize = 100) where T : notnull { From aec2b9734837b0b3da2a519c186d94be42106428 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sat, 14 Nov 2026 17:19:50 -0600 Subject: [PATCH 37/42] Keep task discovery reliable during continuous progress --- .../features/tasks/api-polling.svelte.test.ts | 222 ++++++++++++++++++ .../src/lib/features/tasks/api.svelte.ts | 179 ++++++++++---- .../src/lib/features/tasks/api.test.ts | 176 ++++++++++++-- .../tasks/components/task-panel.svelte | 77 ++++-- .../components/task-panel.svelte.test.ts | 144 +++++++++++- .../src/lib/features/tasks/models.test.ts | 53 ++++- .../src/lib/features/tasks/models.ts | 29 ++- 7 files changed, 792 insertions(+), 88 deletions(-) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api-polling.svelte.test.ts diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api-polling.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api-polling.svelte.test.ts new file mode 100644 index 0000000000..cddeb214d4 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api-polling.svelte.test.ts @@ -0,0 +1,222 @@ +import type { ProblemDetails } from '@foundatiofx/fetchclient'; + +import { focusManager, QueryClient, QueryObserver, type QueryObserverOptions } from '@tanstack/svelte-query'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { BackgroundTaskStatus, type ViewBackgroundTask } from './models'; + +const mocks = vi.hoisted(() => ({ + createQuery: vi.fn<(options: () => QueryObserverOptions) => void>(), + getJSON: vi.fn(), + useQueryClient: vi.fn() +})); + +vi.mock('#features/auth/index.svelte.ts', () => ({ accessToken: { current: 'access-token' } })); +vi.mock('#features/events/api.svelte.ts', () => ({ queryKeys: { type: ['PersistentEvent'] } })); +vi.mock('#features/stacks/api.svelte.ts', () => ({ queryKeys: { type: ['Stack'] } })); +vi.mock('@foundatiofx/fetchclient', () => ({ useFetchClient: () => ({ getJSON: mocks.getJSON }) })); +vi.mock('@tanstack/svelte-query', async (importOriginal) => ({ + ...(await importOriginal()), + createQuery: mocks.createQuery, + useQueryClient: mocks.useQueryClient +})); + +import { ACTIVE_TASK_POLL_INTERVAL_MS, getTasksQuery, queryKeys, startTaskPolling, updateTaskFromMessage } from './api.svelte'; + +function createTask(status: BackgroundTaskStatus): ViewBackgroundTask { + return { + can_cancel: false, + created_by_user_id: 'user-id', + created_utc: '2026-01-01T10:00:00Z', + description: 'Generate sample data', + failed: 0, + id: 'task-1', + organization_id: 'organization-id', + processed: 0, + progress: 0, + status, + succeeded: 0, + total: 0, + type: 'projects.sample-data', + updated_utc: '2026-01-01T10:00:00Z' + }; +} + +describe('task polling', () => { + beforeEach(() => { + vi.useFakeTimers(); + focusManager.setFocused(true); + }); + + afterEach(() => { + focusManager.setFocused(undefined); + vi.useRealTimers(); + }); + + it.each([false, true])('polls on schedule during continuous progress with a preseeded cache: %s', async (preseeded) => { + // Arrange + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + mocks.useQueryClient.mockReturnValue(client); + const running = createTask(BackgroundTaskStatus.Running); + const unknown = { ...running, id: 'started-elsewhere' }; + mocks.getJSON.mockReset().mockResolvedValue({ data: [running, unknown] }); + if (preseeded) { + client.setQueryData(queryKeys.organization(running.organization_id), [running]); + } else { + mocks.getJSON.mockResolvedValueOnce({ data: [running] }); + } + getTasksQuery({ route: { organizationId: 'organization-id' } }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, { ...options(), refetchOnMount: !preseeded }); + const unsubscribe = observer.subscribe(() => {}); + const stopPolling = startTaskPolling(client, running.organization_id, true); + + try { + await observer.getCurrentQuery().promise; + + // Act + for (let progress = 1; progress <= 5; progress++) { + await vi.advanceTimersByTimeAsync(500); + await updateTaskFromMessage(client, { + ...running, + message: running.message ?? undefined, + processed: progress, + project_id: running.project_id ?? undefined, + updated_utc: new Date().toISOString() + }); + } + await vi.advanceTimersByTimeAsync(500); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(preseeded ? 1 : 2); + expect(observer.getCurrentResult().data?.map((task) => task.id)).toEqual([running.id, unknown.id]); + + // Act + focusManager.setFocused(false); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS * 2); + await updateTaskFromMessage(client, { + ...running, + message: running.message ?? undefined, + processed: 6, + project_id: running.project_id ?? undefined, + updated_utc: new Date().toISOString() + }); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(preseeded ? 1 : 2); + + // Act + focusManager.setFocused(true); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(preseeded ? 2 : 3); + + // Act + stopPolling?.(); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(preseeded ? 2 : 3); + } finally { + stopPolling?.(); + unsubscribe(); + client.clear(); + } + }); + + it('keeps its polling cadence after a failed HTTP attempt', async () => { + // Arrange + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + mocks.useQueryClient.mockReturnValue(client); + const running = createTask(BackgroundTaskStatus.Running); + mocks.getJSON + .mockReset() + .mockResolvedValueOnce({ data: [running] }) + .mockRejectedValueOnce(new Error('Unavailable')) + .mockResolvedValue({ data: [running] }); + getTasksQuery({ route: { organizationId: 'organization-id' } }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, options()); + const unsubscribe = observer.subscribe(() => {}); + const stopPolling = startTaskPolling(client, running.organization_id, true); + + try { + await observer.getCurrentQuery().promise; + + // Act + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS - 1); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + + // Act + await vi.advanceTimersByTimeAsync(1); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(3); + } finally { + stopPolling?.(); + unsubscribe(); + client.clear(); + } + }); + + it.each(['empty', 'failed'] as const)('reconciles a list that is %s at a slower rate only while the tab is visible', async (initialState) => { + // Arrange + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + mocks.useQueryClient.mockReturnValue(client); + const initialTasks = initialState === 'failed' ? [createTask(BackgroundTaskStatus.Failed)] : []; + const running = createTask(BackgroundTaskStatus.Running); + const completed = createTask(BackgroundTaskStatus.Completed); + mocks.getJSON + .mockReset() + .mockResolvedValueOnce({ data: initialTasks }) + .mockResolvedValueOnce({ data: [running] }) + .mockResolvedValue({ data: [completed] }); + getTasksQuery({ route: { organizationId: 'organization-id' } }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, options()); + const unsubscribe = observer.subscribe(() => {}); + let stopPolling = startTaskPolling(client, 'organization-id', false); + + try { + await observer.getCurrentQuery().promise; + + // Act + focusManager.setFocused(false); + await vi.advanceTimersByTimeAsync(60_000); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledOnce(); + + // Act + focusManager.setFocused(true); + await vi.advanceTimersByTimeAsync(60_000 - 1); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledOnce(); + + // Act + await vi.advanceTimersByTimeAsync(1); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + expect(observer.getCurrentResult().data).toEqual([running]); + + // Act + stopPolling?.(); + stopPolling = startTaskPolling(client, 'organization-id', true); + await vi.advanceTimersByTimeAsync(ACTIVE_TASK_POLL_INTERVAL_MS); + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(3); + expect(observer.getCurrentResult().data).toEqual([completed]); + } finally { + stopPolling?.(); + unsubscribe(); + client.clear(); + } + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts index bb979ea090..daccfa36a7 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts @@ -1,7 +1,7 @@ -import type { QueryClient } from '@tanstack/svelte-query'; +import type { Query, QueryClient } from '@tanstack/svelte-query'; import { type ProblemDetails, useFetchClient } from '@foundatiofx/fetchclient'; -import { createMutation, createQuery, useQueryClient } from '@tanstack/svelte-query'; +import { createMutation, createQuery, focusManager, replaceEqualDeep, useQueryClient } from '@tanstack/svelte-query'; import type { BackgroundTaskUpdated } from '#features/websockets/models.ts'; @@ -9,10 +9,12 @@ import { accessToken } from '#features/auth/index.svelte.ts'; import { queryKeys as eventQueryKeys } from '#features/events/api.svelte.ts'; import { queryKeys as stackQueryKeys } from '#features/stacks/api.svelte.ts'; -import { applyTaskUpdate, compareTaskUpdateTimes, isTaskActive, type NewBulkOperation, type ViewBackgroundTask } from './models'; +import { applyTaskUpdate, compareTaskUpdateTimes, mergeTaskList, type NewBulkOperation, type ViewBackgroundTask } from './models'; /** While a task is active the list is polled as well, so progress still moves when push messages are unavailable. */ export const ACTIVE_TASK_POLL_INTERVAL_MS = 3000; +/** Reconcile missed notifications, tasks started elsewhere, and ordinary retries while the tab is visible. */ +export const IDLE_TASK_POLL_INTERVAL_MS = 60_000; export const queryKeys = { cancel: (id: string | undefined) => [...queryKeys.type, id, 'cancel'] as const, @@ -27,6 +29,16 @@ export interface GetTasksRequest { }; } +interface TaskListRead { + baseline: readonly ViewBackgroundTask[]; + result?: ViewBackgroundTask[]; +} + +type TaskQuery = Query; + +// Query identity survives observer/options changes. Each settled read releases its baseline. +const taskListReads = new WeakMap(); + export function cancelTask() { const queryClient = useQueryClient(); @@ -44,21 +56,54 @@ export function cancelTask() { } export function getTasksQuery(request: GetTasksRequest) { - return createQuery(() => ({ - enabled: () => !!accessToken.current && !!request.route.organizationId, - queryFn: async ({ signal }: { signal: AbortSignal }) => { - const client = useFetchClient(); - const response = await client.getJSON(`organizations/${request.route.organizationId}/tasks`, { - signal - }); + const queryClient = useQueryClient(); - return response.data ?? []; - }, - queryKey: queryKeys.organization(request.route.organizationId), - refetchInterval: (query) => (query.state.data?.some((task) => isTaskActive(task.status)) ? ACTIVE_TASK_POLL_INTERVAL_MS : false), - // Updates for other organizations are not observed. Refresh on return even when the cached list has no active tasks to poll. - staleTime: 0 - })); + return createQuery(() => { + const organizationId = request.route.organizationId; + const key = queryKeys.organization(organizationId); + const getQuery = () => + queryClient.getQueryCache().find({ + exact: true, + queryKey: key + })!; + + return { + enabled: () => !!accessToken.current && !!organizationId, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const query = getQuery(); + const read: TaskListRead = { + baseline: query.state.data ?? [] + }; + taskListReads.set(query, read); + try { + const client = useFetchClient(); + const response = await client.getJSON(`organizations/${organizationId}/tasks`, { + signal + }); + read.result = response.data ?? []; + return read.result; + } finally { + if (taskListReads.get(query) === read && (signal.aborted || !read.result)) { + taskListReads.delete(query); + } + } + }, + queryKey: key, + // Refresh immediately on return; updates for other organizations are not observed. + staleTime: 0, + structuralSharing: (current, incoming) => { + const query = getQuery(); + const read = taskListReads.get(query); + if (!read || read.result !== incoming) { + return replaceEqualDeep(current, incoming); + } + + // Merge at the synchronous cache commit: a push can arrive after queryFn returns its response. + taskListReads.delete(query); + return replaceEqualDeep(current, mergeTaskList(incoming as ViewBackgroundTask[], current as undefined | ViewBackgroundTask[], read.baseline)); + } + }; + }); } /** Refreshes everything a finished bulk task may have changed. */ @@ -74,10 +119,26 @@ export async function invalidateTaskAffectedQueries(queryClient: QueryClient) { } /** Refetches the task lists, e.g. after a request that starts a task. */ -export function invalidateTaskQueries(queryClient: QueryClient) { - return queryClient.invalidateQueries({ - queryKey: queryKeys.type - }); +export async function invalidateTaskQueries(queryClient: QueryClient) { + // Replace a read that began before creation, including an uncached initial fetch. Once invalidated, + // let its reconciliation finish so a burst of notifications cannot keep restarting the request. + await queryClient.cancelQueries( + { + predicate: (query) => !query.state.isInvalidated, + queryKey: queryKeys.type + }, + { + revert: false + } + ); + return queryClient.invalidateQueries( + { + queryKey: queryKeys.type + }, + { + cancelRefetch: false + } + ); } export function postBulkOperation() { @@ -96,15 +157,60 @@ export function postBulkOperation() { })); } +/** Poll on a fixed cadence owned by the panel, so pushed cache writes cannot postpone reconciliation. */ +export function startTaskPolling(queryClient: QueryClient, organizationId: string | undefined, hasActiveTasks: boolean) { + if (!organizationId) { + return; + } + + const key = queryKeys.organization(organizationId); + const timer = window.setInterval( + () => { + if (!focusManager.isFocused()) { + return; + } + + void queryClient.refetchQueries( + { + exact: true, + fetchStatus: 'idle', + queryKey: key, + type: 'active' + }, + { + cancelRefetch: false + } + ); + }, + hasActiveTasks ? ACTIVE_TASK_POLL_INTERVAL_MS : IDLE_TASK_POLL_INTERVAL_MS + ); + return () => window.clearInterval(timer); +} + /** Applies a pushed update to the cached list. Unknown tasks (started elsewhere) trigger a refetch instead. */ export async function updateTaskFromMessage(queryClient: QueryClient, update: BackgroundTaskUpdated) { const key = queryKeys.organization(update.organization_id); const current = queryClient.getQueryData(key); const next = current ? applyTaskUpdate(current, update) : undefined; if (!next) { - void queryClient.invalidateQueries({ - queryKey: key - }); + // Replace an earlier read once, then share the reconciliation during a burst of unknown-task messages. + await queryClient.cancelQueries( + { + predicate: (query) => !query.state.isInvalidated, + queryKey: key + }, + { + revert: false + } + ); + void queryClient.invalidateQueries( + { + queryKey: key + }, + { + cancelRefetch: false + } + ); return; } @@ -112,29 +218,16 @@ export async function updateTaskFromMessage(queryClient: QueryClient, update: Ba return; } - await updateCachedTasks(queryClient, key, (latest) => (latest ? applyTaskUpdate(latest, update) : undefined)); + updateCachedTasks(queryClient, key, () => next); } -async function updateCachedTasks( +function updateCachedTasks( queryClient: QueryClient, key: ReturnType, update: (current: undefined | ViewBackgroundTask[]) => undefined | ViewBackgroundTask[] ) { - const wasInvalidated = queryClient.getQueryState(key)?.isInvalidated; - // An older list response must not remove newly queued work and stop polling. Keep any pushed updates received - // since that read started instead of reverting the cache to its pre-fetch snapshot. - await queryClient.cancelQueries( - { - queryKey: key - }, - { - revert: false - } - ); - - // An unknown-task message can invalidate the list while cancellation settles. A cache write clears that flag, - // so preserve the required reconciliation even if this update finishes the last task we already know about. - const needsReconciliation = wasInvalidated || queryClient.getQueryState(key)?.isInvalidated; + // Cache writes clear invalidation. Keep any required list reconciliation running while applying pushed progress. + const needsReconciliation = queryClient.getQueryState(key)?.isInvalidated; queryClient.setQueryData(key, update); if (needsReconciliation) { void queryClient.invalidateQueries( @@ -148,9 +241,9 @@ async function updateCachedTasks( } } -async function upsertTask(queryClient: QueryClient, task: ViewBackgroundTask) { +function upsertTask(queryClient: QueryClient, task: ViewBackgroundTask) { const key = queryKeys.organization(task.organization_id); - await updateCachedTasks(queryClient, key, (current) => { + updateCachedTasks(queryClient, key, (current) => { const existing = current?.find((item) => item.id === task.id); if (existing && compareTaskUpdateTimes(existing.updated_utc, task.updated_utc) > 0) { return current; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts index 1bf7586c8e..faa68ffbb2 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts @@ -24,10 +24,10 @@ vi.mock('@tanstack/svelte-query', async (importOriginal) => ({ })); import { - ACTIVE_TASK_POLL_INTERVAL_MS, cancelTask, getTasksQuery, invalidateTaskAffectedQueries, + invalidateTaskQueries, postBulkOperation, queryKeys, updateTaskFromMessage @@ -71,6 +71,11 @@ function createUpdate(id: string) { }; } +function getTaskQueryOptions() { + getTasksQuery({ route: { organizationId: 'organization-id' } }); + return mocks.createQuery.mock.calls.at(-1)![0](); +} + describe('getTasksQuery', () => { it.each([ ['empty', BackgroundTaskStatus.Running], @@ -81,6 +86,7 @@ describe('getTasksQuery', () => { // Arrange vi.clearAllMocks(); const client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: 5 * 60 * 1000 } } }); + mocks.useQueryClient.mockReturnValue(client); let organizationId = 'organization-id'; const initialTasks = cachedList === 'finished' ? [createTask({ status: BackgroundTaskStatus.Completed })] : []; const missedTask = createTask({ id: 'started-elsewhere', status: newStatus }); @@ -118,10 +124,6 @@ describe('getTasksQuery', () => { 'organizations/organization-id/tasks' ]); expect(observer.getCurrentResult().data).toEqual([...initialTasks, missedTask]); - const refetchInterval = options().refetchInterval; - expect(typeof refetchInterval === 'function' ? refetchInterval(observer.getCurrentQuery()) : refetchInterval).toBe( - newStatus === BackgroundTaskStatus.Running ? ACTIVE_TASK_POLL_INTERVAL_MS : false - ); } finally { unsubscribe(); client.clear(); @@ -162,7 +164,7 @@ describe('updateTaskFromMessage', () => { // Assert expect(queryClient.getQueryData(queryKeys.organization('organization-id'))).toEqual(tasks); - expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: queryKeys.organization('organization-id') }); + expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: queryKeys.organization('organization-id') }, { cancelRefetch: false }); }); }); @@ -192,10 +194,68 @@ describe('task cache lifecycle', () => { afterEach(() => client.clear()); + it.each([ + ['pushed update', false], + ['task creation invalidation', false], + ['pushed update', true], + ['task creation invalidation', true] + ] as const)('coalesces unknown-task notifications after a %s replaces a pending list request with cached data: %s', async (source, cached) => { + // Arrange + const previousRead = Promise.withResolvers(); + const nextRead = Promise.withResolvers(); + const nextReadStarted = Promise.withResolvers(); + const completed = createTask({ id: 'started-elsewhere', status: BackgroundTaskStatus.Completed }); + const queued = createTask({ id: 'queued-during-read', status: BackgroundTaskStatus.Queued }); + const signals: AbortSignal[] = []; + mocks.getJSON.mockReset().mockImplementation((_url: string, { signal }: { signal: AbortSignal }) => { + signals.push(signal); + if (signals.length === 1) { + return previousRead.promise.then((data) => ({ data })); + } + + nextReadStarted.resolve(); + return nextRead.promise.then((data) => ({ data })); + }); + if (cached) { + client.setQueryData(key, []); + } + + const observer = new QueryObserver(client, getTaskQueryOptions()); + const unsubscribe = observer.subscribe(() => {}); + + try { + // Act + const reconciliation = source === 'pushed update' ? updateTaskFromMessage(client, createUpdate(completed.id)) : invalidateTaskQueries(client); + await nextReadStarted.promise; + for (let notification = 0; notification < 5; notification++) { + await updateTaskFromMessage(client, createUpdate(completed.id)); + } + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + expect(signals.map((signal) => signal.aborted)).toEqual([true, false]); + + // Act + postBulkOperation(); + await mocks.createMutation.mock.calls[0]![0]().onSuccess(queued); + previousRead.resolve([]); + nextRead.resolve([completed]); + await reconciliation; + await observer.getCurrentQuery().promise; + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + expect(observer.getCurrentResult().data).toEqual([completed, queued]); + } finally { + unsubscribe(); + } + }); + it('keeps a newly queued task when an earlier list request returns without it', async () => { // Arrange const pending = Promise.withResolvers(); - const read = client.fetchQuery({ queryFn: () => pending.promise, queryKey: key }).catch(() => undefined); + mocks.getJSON.mockReset().mockImplementation(() => pending.promise.then((data) => ({ data }))); + const read = client.fetchQuery(getTaskQueryOptions()); const task = createTask({ status: BackgroundTaskStatus.Queued }); postBulkOperation(); const options = mocks.createMutation.mock.calls[0]![0](); @@ -209,6 +269,38 @@ describe('task cache lifecycle', () => { expect(client.getQueryData(key)).toEqual([task]); }); + it('merges a task inserted after HTTP completion and drops it when a later read omits it unchanged', async () => { + // Arrange + mocks.getJSON.mockReset().mockResolvedValue({ data: [] }); + const task = createTask({ status: BackgroundTaskStatus.Queued }); + const queryOptions = getTaskQueryOptions(); + const queryFn = queryOptions.queryFn; + if (typeof queryFn !== 'function') { + throw new Error('The task list requires a query function.'); + } + postBulkOperation(); + const mutationOptions = mocks.createMutation.mock.calls[0]![0](); + + // Act + await client.fetchQuery({ + ...queryOptions, + queryFn: async (context) => { + const data = await queryFn(context); + await mutationOptions.onSuccess(task); + return data; + } + }); + + // Assert + expect(client.getQueryData(key)).toEqual([task]); + + // Act + await client.fetchQuery(queryOptions); + + // Assert + expect(client.getQueryData(key)).toEqual([]); + }); + it.each([ ['2026-01-01T10:00:00Z', '2026-01-01T10:01:00Z'], ['2026-01-01T10:00:00.1234566Z', '2026-01-01T10:00:00.1234567Z'] @@ -217,7 +309,8 @@ describe('task cache lifecycle', () => { const running = createTask({ updated_utc: older }); client.setQueryData(key, [running]); const pending = Promise.withResolvers(); - const read = client.fetchQuery({ queryFn: () => pending.promise, queryKey: key }).catch(() => undefined); + mocks.getJSON.mockReset().mockImplementation(() => pending.promise.then((data) => ({ data }))); + const read = client.fetchQuery(getTaskQueryOptions()); const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: newer }); client.setQueryData(key, [completed]); cancelTask(); @@ -240,7 +333,8 @@ describe('task cache lifecycle', () => { const previous = createTask({ status: previousStatus }); client.setQueryData(key, [previous]); const pending = Promise.withResolvers(); - const read = client.fetchQuery({ queryFn: () => pending.promise, queryKey: key }).catch(() => undefined); + mocks.getJSON.mockReset().mockImplementation(() => pending.promise.then((data) => ({ data }))); + const read = client.fetchQuery(getTaskQueryOptions()); // Act await updateTaskFromMessage(client, { ...createUpdate(previous.id), status: nextStatus }); @@ -262,18 +356,67 @@ describe('task cache lifecycle', () => { expect(client.getQueryData(key)?.map((task) => task.processed)).toEqual([75, 75]); }); + it('finishes reconciliation during continuous progress without rolling it back or retaining unchanged omitted tasks', async () => { + // Arrange + const running = createTask(); + const expired = createTask({ id: 'expired', status: BackgroundTaskStatus.Completed }); + const unknown = createTask({ id: 'started-elsewhere' }); + const pending = Promise.withResolvers<{ data: ViewBackgroundTask[] }>(); + const signals: AbortSignal[] = []; + mocks.getJSON + .mockReset() + .mockResolvedValueOnce({ data: [running, expired] }) + .mockImplementation((_url: string, { signal }: { signal: AbortSignal }) => { + signals.push(signal); + return pending.promise; + }); + getTasksQuery({ route: { organizationId: 'organization-id' } }); + const options = mocks.createQuery.mock.calls[0]![0]; + const observer = new QueryObserver(client, options()); + const unsubscribe = observer.subscribe(() => {}); + + try { + await observer.getCurrentQuery().promise; + + // Act + await updateTaskFromMessage(client, createUpdate(unknown.id)); + for (let progress = 1; progress <= 5; progress++) { + await updateTaskFromMessage(client, { + ...createUpdate(running.id), + processed: progress, + updated_utc: `2026-01-01T10:00:0${progress}Z` + }); + } + + // Assert + expect(mocks.getJSON).toHaveBeenCalledTimes(2); + expect(signals.map((signal) => signal.aborted)).toEqual([false]); + + // Act + pending.resolve({ data: [running, unknown] }); + await observer.getCurrentQuery().promise; + + // Assert + expect(observer.getCurrentResult().data).toEqual([ + { ...running, message: null, processed: 5, progress: 75, succeeded: 75, updated_utc: '2026-01-01T10:00:05Z' }, + unknown + ]); + } finally { + unsubscribe(); + } + }); + it.each(['pushed completion', 'cancellation response'] as const)( - 'finishes discovering an unknown task after a %s cancels the list request', + 'finishes discovering an unknown task when a %s arrives during the list request', async (source) => { // Arrange const running = createTask(); const completed = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: '2026-01-01T10:00:09Z' }); const unknown = createTask({ id: 'started-elsewhere' }); client.setQueryData(key, [running]); - const previousRead = Promise.withResolvers(); - const nextRead = Promise.withResolvers(); - const queryFn = vi.fn().mockReturnValueOnce(previousRead.promise).mockReturnValueOnce(nextRead.promise); - const observer = new QueryObserver(client, { queryFn, queryKey: key, staleTime: Infinity }); + const pending = Promise.withResolvers(); + mocks.getJSON.mockReset().mockImplementation(() => pending.promise.then((data) => ({ data }))); + const observer = new QueryObserver(client, { ...getTaskQueryOptions(), staleTime: Infinity }); const unsubscribe = observer.subscribe(() => {}); // Act @@ -287,11 +430,10 @@ describe('task cache lifecycle', () => { // Assert expect(client.getQueryData(key)).toMatchObject([completed]); - expect(queryFn).toHaveBeenCalledTimes(2); + expect(mocks.getJSON).toHaveBeenCalledOnce(); // Act - previousRead.resolve([running, unknown]); - nextRead.resolve([completed, unknown]); + pending.resolve([completed, unknown]); await client.getQueryCache().find({ queryKey: key })?.promise; unsubscribe(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte index f1a240513f..81edaf4374 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte @@ -1,15 +1,16 @@ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts index 5a444d502b..54a80e3fa5 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts @@ -2,18 +2,35 @@ import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/sv import { tick } from 'svelte'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { organization } from '#features/organizations/context.svelte.ts'; + import { BackgroundTaskStatus, BulkOperationSelectionKind, type ViewBackgroundTask } from '../models'; const mocks = vi.hoisted(() => ({ cancelMutateAsync: vi.fn(), invalidateAffected: vi.fn(), queryClient: { id: 'query-client' }, + startPolling: vi.fn(), + stopPolling: vi.fn(), updateFromMessage: vi.fn() })); vi.mock('@tanstack/svelte-query', () => ({ useQueryClient: () => mocks.queryClient })); vi.mock('svelte-sonner', () => ({ toast: { error: vi.fn(), success: vi.fn() } })); -vi.mock('#features/organizations/context.svelte.ts', () => ({ organization: { current: 'organization-id' } })); +vi.mock('#features/organizations/context.svelte.ts', async () => { + const { SvelteMap } = await import('svelte/reactivity'); + const state = new SvelteMap([['current', 'organization-id']]); + return { + organization: { + get current() { + return state.get('current'); + }, + set current(value: string | undefined) { + state.set('current', value); + } + } + }; +}); vi.mock('../api.svelte', async () => { const { SvelteMap } = await import('svelte/reactivity'); const state = new SvelteMap(); @@ -27,6 +44,7 @@ vi.mock('../api.svelte', async () => { } }), invalidateTaskAffectedQueries: mocks.invalidateAffected, + startTaskPolling: mocks.startPolling, updateTaskFromMessage: mocks.updateFromMessage }; }); @@ -61,7 +79,9 @@ function createTask(overrides: Partial = {}): ViewBackground describe('TaskPanel', () => { beforeEach(() => { state.clear(); + organization.current = 'organization-id'; mocks.cancelMutateAsync.mockResolvedValue(undefined); + mocks.startPolling.mockReturnValue(mocks.stopPolling); }); afterEach(() => { @@ -83,6 +103,47 @@ describe('TaskPanel', () => { expect(mocks.invalidateAffected).not.toHaveBeenCalled(); }); + it('keeps polling across progress and replaces it only when activity or organization changes', async () => { + // Arrange + const running = createTask(); + state.set('tasks', [running]); + render(TaskPanel); + await tick(); + + // Act + for (let progress = 1; progress <= 5; progress++) { + state.set('tasks', [{ ...running, processed: progress }]); + await tick(); + } + + // Assert + expect(mocks.startPolling).toHaveBeenCalledExactlyOnceWith(mocks.queryClient, 'organization-id', true); + expect(mocks.stopPolling).not.toHaveBeenCalled(); + + // Act + state.set('tasks', [{ ...running, status: BackgroundTaskStatus.Completed }]); + await tick(); + + // Assert + expect(mocks.startPolling).toHaveBeenLastCalledWith(mocks.queryClient, 'organization-id', false); + expect(mocks.stopPolling).toHaveBeenCalledOnce(); + + // Act + organization.current = 'another-organization'; + state.set('tasks', []); + await tick(); + + // Assert + expect(mocks.startPolling).toHaveBeenLastCalledWith(mocks.queryClient, 'another-organization', false); + expect(mocks.stopPolling).toHaveBeenCalledTimes(2); + + // Act + cleanup(); + + // Assert + expect(mocks.stopPolling).toHaveBeenCalledTimes(3); + }); + it('applies pushed updates for its organization and ignores other organizations', async () => { // Arrange state.set('tasks', [createTask()]); @@ -111,7 +172,7 @@ describe('TaskPanel', () => { await tick(); state.set( 'tasks', - completed.map((task) => ({ ...task, updated_utc: new Date().toISOString() })) + completed.map((task) => ({ ...task })) ); await tick(); @@ -199,6 +260,85 @@ describe('TaskPanel', () => { expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); }); + it.each([BackgroundTaskStatus.Completed, BackgroundTaskStatus.Failed])( + 'shows and reconciles a newer %s retry outcome when the running update was missed', + async (status) => { + // Arrange + vi.useFakeTimers(); + vi.setSystemTime('2026-01-01T10:00:00.123Z'); + const failed = createTask({ status: BackgroundTaskStatus.Failed, updated_utc: '2026-01-01T10:00:00.1234566Z' }); + const outcome = { ...failed, status, updated_utc: '2026-01-01T10:00:00.1234567Z' }; + state.set('tasks', [failed]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + state.set('tasks', [outcome]); + await tick(); + + // Assert + expect(screen.getByRole('button', { name: /^Dismiss / })).toBeVisible(); + expect(document.querySelector('[data-slot="task"]')).toHaveAttribute('data-status', status); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + state.set('tasks', [{ ...outcome }]); + await tick(); + + // Assert + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + } + ); + + it('reconciles a newly discovered completed task even when it never saw the task running', async () => { + // Arrange + state.set('tasks', []); + render(TaskPanel); + await tick(); + const completed = createTask({ status: BackgroundTaskStatus.Completed }); + + // Act + state.set('tasks', [completed]); + await tick(); + state.set('tasks', [{ ...completed }]); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + }); + + it('preserves a dismissal across organization changes and shows a retry that finished while away', async () => { + // Arrange + const failed = createTask({ status: BackgroundTaskStatus.Failed }); + const completed = { ...failed, status: BackgroundTaskStatus.Completed, updated_utc: new Date(Date.now() + 1000).toISOString() }; + state.set('tasks', [failed]); + render(TaskPanel); + + // Act + await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); + organization.current = 'another-organization'; + state.set('tasks', []); + await tick(); + document.dispatchEvent(new CustomEvent('BackgroundTaskUpdated', { detail: completed })); + organization.current = 'organization-id'; + state.set('tasks', [failed]); + await tick(); + + // Assert + expect(mocks.updateFromMessage).not.toHaveBeenCalled(); + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); + + // Act + state.set('tasks', [completed]); + await tick(); + + // Assert + expect(screen.getByRole('button', { name: /^Dismiss / })).toBeVisible(); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + }); + it('renders nothing when there is nothing to show', () => { // Arrange state.clear(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts index 7ecb855485..76cd5beb65 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.test.ts @@ -7,6 +7,7 @@ import { canCancelTask, FINISHED_TASK_VISIBLE_MS, getProgressPercent, + mergeTaskList, selectVisibleTasks, STALE_TASK_MS, type ViewBackgroundTask @@ -102,6 +103,36 @@ describe('applyTaskUpdate', () => { }); }); +describe('mergeTaskList', () => { + it('keeps newer cached revisions and accepts complete server metadata on equal revisions', () => { + // Arrange + const older = createTask({ updated_utc: '2026-01-01T10:00:00.1234566Z' }); + const newer = { ...older, processed: 50, updated_utc: '2026-01-01T10:00:00.1234567Z' }; + const cached = createTask({ id: 'complete-metadata', status: BackgroundTaskStatus.Completed }); + const fromServer = { ...cached, completed_utc: '2026-01-01T10:00:00Z' }; + + // Act + const merged = mergeTaskList([older, fromServer], [newer, cached], [older, cached]); + + // Assert + expect(merged).toEqual([newer, fromServer]); + }); + + it('preserves only omitted tasks added or advanced during the read', () => { + // Arrange + const unchanged = createTask({ id: 'unchanged' }); + const previous = createTask({ id: 'advanced', updated_utc: '2026-01-01T10:00:00.1234566Z' }); + const advanced = { ...previous, updated_utc: '2026-01-01T10:00:00.1234567Z' }; + const inserted = createTask({ id: 'inserted' }); + + // Act + const merged = mergeTaskList([], [unchanged, advanced, inserted], [unchanged, previous]); + + // Assert + expect(merged).toEqual([advanced, inserted]); + }); +}); + describe('getProgressPercent', () => { it.each([ [{ processed: 0, progress: 0, status: BackgroundTaskStatus.Queued, total: 0 }, 0], @@ -160,7 +191,7 @@ describe('selectVisibleTasks', () => { }); // Act - const result = selectVisibleTasks([active, old, recent], new Set(), now); + const result = selectVisibleTasks([active, old, recent], new Map(), now); // Assert expect(result.map((task) => task.id)).toEqual(['recent', 'active']); @@ -171,7 +202,7 @@ describe('selectVisibleTasks', () => { const task = createTask({ id: 'dismissed', status: BackgroundTaskStatus.Failed }); // Act - const result = selectVisibleTasks([task], new Set(['dismissed']), now); + const result = selectVisibleTasks([task], new Map([['dismissed', task.updated_utc]]), now); // Assert expect(result).toEqual([]); @@ -182,9 +213,25 @@ describe('selectVisibleTasks', () => { const task = createTask({ id: 'retried', status }); // Act - const result = selectVisibleTasks([task], new Set(['retried']), now); + const result = selectVisibleTasks([task], new Map([['retried', task.updated_utc]]), now); // Assert expect(result).toEqual([task]); }); + + it.each([ + ['2026-01-01T10:29:59.1234566Z', false], + ['2026-01-01T10:29:59.1234567Z', false], + ['2026-01-01T10:29:59.1234568Z', true] + ] as const)('only shows a terminal outcome at %s when it is newer than the dismissed revision', (updatedUtc, visible) => { + // Arrange + const task = createTask({ status: BackgroundTaskStatus.Completed, updated_utc: updatedUtc }); + const dismissed = new Map([[task.id, '2026-01-01T10:29:59.1234567Z']]); + + // Act + const result = selectVisibleTasks([task], dismissed, now); + + // Assert + expect(result).toEqual(visible ? [task] : []); + }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts index 6d0962104b..59958a2414 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/models.ts @@ -67,6 +67,30 @@ export function hasCounts(task: Pick): boolean { return task.total > 0; } +/** Reconciles a list read without discarding task changes received while that read was in flight. */ +export function mergeTaskList( + tasks: ViewBackgroundTask[], + current: readonly ViewBackgroundTask[] | undefined, + baseline: readonly ViewBackgroundTask[] +): ViewBackgroundTask[] { + const remaining = new Map((current ?? []).map((task) => [task.id, task])); + const merged = tasks.map((task) => { + const cached = remaining.get(task.id); + remaining.delete(task.id); + return cached && compareTaskUpdateTimes(cached.updated_utc, task.updated_utc) > 0 ? cached : task; + }); + + const initialUpdates = new Map(baseline.map((task) => [task.id, task.updated_utc])); + for (const task of remaining.values()) { + const initialUpdatedUtc = initialUpdates.get(task.id); + if (!initialUpdatedUtc || compareTaskUpdateTimes(task.updated_utc, initialUpdatedUtc) > 0) { + merged.push(task); + } + } + + return merged; +} + /** An active task that has not reported progress for this long is abandoned, and the server lets anyone clear it. */ export const STALE_TASK_MS = 2 * 60 * 60 * 1000; @@ -84,14 +108,15 @@ export function isTaskActive(status: BackgroundTaskStatus): boolean { } /** Active tasks plus recently finished ones the user has not dismissed, newest first. */ -export function selectVisibleTasks(tasks: ViewBackgroundTask[], dismissedIds: ReadonlySet, now: number): ViewBackgroundTask[] { +export function selectVisibleTasks(tasks: ViewBackgroundTask[], dismissedUpdates: ReadonlyMap, now: number): ViewBackgroundTask[] { return tasks .filter((task) => { if (isTaskActive(task.status)) { return true; } - if (dismissedIds.has(task.id)) { + const dismissedUtc = dismissedUpdates.get(task.id); + if (dismissedUtc && compareTaskUpdateTimes(task.updated_utc, dismissedUtc) <= 0) { return false; } From ba4c3b313f14de3d7b37d46256db166f1ee197e7 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sat, 14 Nov 2026 17:19:52 -0600 Subject: [PATCH 38/42] Document task boundaries and expanded performance measurements --- benchmarks/BackgroundTasks/Program.cs | 56 +- benchmarks/BackgroundTasks/README.md | 41 +- .../BackgroundTasks/extended-comparison.json | 741 ++++++++++++++++++ docs/background-tasks.md | 32 +- 4 files changed, 843 insertions(+), 27 deletions(-) create mode 100644 benchmarks/BackgroundTasks/extended-comparison.json diff --git a/benchmarks/BackgroundTasks/Program.cs b/benchmarks/BackgroundTasks/Program.cs index edcf949bdb..2e3049ba50 100644 --- a/benchmarks/BackgroundTasks/Program.cs +++ b/benchmarks/BackgroundTasks/Program.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Text.Json; using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.BulkOperations; using Exceptionless.Core.Services.BackgroundTasks; using Foundatio.Caching; using Foundatio.Jobs; @@ -14,9 +15,14 @@ // A manual service microbenchmark, deliberately outside the correctness test suite. // Compare the same command before and after a change. This does not measure Redis network latency. string label = args.FirstOrDefault() ?? "current"; -int[] sizes = args.Length > 1 ? args.Skip(1).Select(Int32.Parse).ToArray() : [100, 1_000, 10_000, 50_001]; -foreach (int historyCount in sizes) +bool stress = args.Skip(1).Contains("--stress", StringComparer.Ordinal); +int[] sizes = args.Length > 1 && !stress ? args.Skip(1).Select(Int32.Parse).ToArray() : [100, 1_000, 10_000, 50_001]; +Workload[] workloads = stress + ? [new("large-selection-history", 1_000, 2, 1_000), new("many-active-tasks", 1_000, 200, 0), new("same-second-history", 1_000, 2, 1_000, SameSecond: true)] + : sizes.Select(size => new Workload("history", size, 2, 0)).ToArray(); +foreach (var workload in workloads) { + int historyCount = workload.HistoryCount; using var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { CloneValues = true, MaxItems = null }); using var messageBus = new InMemoryMessageBus(); using var queue = new InMemoryQueue(); @@ -26,21 +32,34 @@ new ResiliencePolicyProvider(), new BackgroundTaskOptions(), TimeProvider.System, NullLogger.Instance); const string organizationId = "537650f3b77efe23a47914f4"; - var createdUtc = DateTime.UtcNow.AddSeconds(-historyCount - 2); - var ids = new List(historyCount + 2); - for (int i = 0; i < historyCount + 2; i++) + var createdUtc = DateTime.UtcNow.AddSeconds(-historyCount - workload.ActiveCount); + createdUtc = new DateTime(createdUtc.Ticks - createdUtc.Ticks % TimeSpan.TicksPerSecond, DateTimeKind.Utc); + var ids = new List(historyCount + workload.ActiveCount); + var selectionIds = Enumerable.Range(0, workload.SelectionIds).Select(_ => ObjectId.GenerateNewId().ToString()).ToList(); + for (int i = 0; i < historyCount + workload.ActiveCount; i++) { + bool active = i < workload.ActiveCount; + var taskCreatedUtc = workload.SameSecond && !active + ? createdUtc.AddSeconds(workload.ActiveCount).AddTicks(i - workload.ActiveCount) + : createdUtc.AddSeconds(i); var task = new BackgroundTask { - Id = ObjectId.GenerateNewId(createdUtc.AddSeconds(i)).ToString(), + // Model servers whose ObjectId machine bytes disagree with precise creation order inside one second. + Id = workload.SameSecond && !active + ? new ObjectId(taskCreatedUtc, machine: historyCount + workload.ActiveCount - i, pid: 1, increment: 0).ToString() + : ObjectId.GenerateNewId(taskCreatedUtc).ToString(), OrganizationId = organizationId, Type = "benchmark.work", Description = "Benchmark task", CreatedByUserId = "537650f3b77efe23a47914f5", - CreatedUtc = createdUtc.AddSeconds(i), - UpdatedUtc = createdUtc.AddSeconds(i), - Status = i < 2 ? BackgroundTaskStatus.Running : BackgroundTaskStatus.Completed, - Progress = i < 2 ? 10 : 100 + CreatedUtc = taskCreatedUtc, + UpdatedUtc = taskCreatedUtc, + Status = active ? BackgroundTaskStatus.Running : BackgroundTaskStatus.Completed, + Progress = active ? 10 : 100, + Bulk = selectionIds.Count == 0 ? null : new BulkOperationDetails + { + Selection = new BulkOperationSelection { Kind = BulkOperationSelectionKind.Ids, Ids = selectionIds } + } }; ids.Add(task.Id); await cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(task.Id), task, BackgroundTaskService.TaskLifetime); @@ -48,26 +67,26 @@ // Seed equivalent persisted state directly so index construction is outside the measurement. await cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(organizationId), ids, BackgroundTaskService.TaskLifetime); - await cache.ListAddAsync(BackgroundTaskService.GetActiveIndexCacheKey(organizationId), ids.Take(2), BackgroundTaskService.TaskLifetime); + await cache.ListAddAsync(BackgroundTaskService.GetActiveIndexCacheKey(organizationId), ids.Take(workload.ActiveCount), BackgroundTaskService.TaskLifetime); async Task RunAsync(string operation) { if (operation == "list") { var result = await tasks.GetByOrganizationAsync(organizationId); - return result.Count == Math.Min(historyCount, BackgroundTaskService.ListLimit) + 2 + return result.Count == Math.Min(historyCount, BackgroundTaskService.ListLimit) + workload.ActiveCount && result[0].Id == ids[^1] - && result.Count(t => t.IsActive) == 2; + && result.Count(t => t.IsActive) == workload.ActiveCount; } var updated = await tasks.UpdateAsync(ids[0], task => { - task.Progress = 20; - return true; + task.Progress = operation == "progress" ? 20 : 99; + return operation == "progress"; }); return updated is { Progress: 20, IsActive: true }; } - foreach (string operation in new[] { "list", "progress" }) + foreach (string operation in new[] { "list", "progress", "no-op" }) { for (int i = 0; i < 3; i++) await RunAsync(operation); @@ -94,8 +113,11 @@ async Task RunAsync(string operation) Console.WriteLine(JsonSerializer.Serialize(new { label, + workload = workload.Name, operation, historyCount, + activeCount = workload.ActiveCount, + selectionIds = workload.SelectionIds, iterations, medianMilliseconds = milliseconds[iterations / 2], medianAllocatedBytes = allocatedBytes[iterations / 2], @@ -109,6 +131,8 @@ async Task RunAsync(string operation) } } +sealed record Workload(string Name, int HistoryCount, int ActiveCount, int SelectionIds, bool SameSecond = false); + // Instrument calls while preserving the real cache implementation and its cloning behavior. sealed class MeasuredCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient { diff --git a/benchmarks/BackgroundTasks/README.md b/benchmarks/BackgroundTasks/README.md index 9213df9c23..fa42f7b29e 100644 --- a/benchmarks/BackgroundTasks/README.md +++ b/benchmarks/BackgroundTasks/README.md @@ -1,6 +1,6 @@ # Background task service benchmark -This manual executable measures task listing and active progress updates using the real `BackgroundTaskService` and Foundatio in-memory cache. It is outside the normal test suite and solution. It adds no benchmark package or production dependency. +This manual executable measures task listing, active progress updates, and skipped mutations using the real `BackgroundTaskService` and Foundatio in-memory cache. It is outside the normal test suite and solution. It adds no benchmark package or production dependency. ## Run @@ -16,21 +16,33 @@ The first argument labels the results. Optional following arguments replace the dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current 100 1000 10000 50001 ``` +The stress profile checks larger task payloads, a larger active index, and creation bursts across servers: + +```sh +dotnet run --project benchmarks/BackgroundTasks/BackgroundTasks.csproj --configuration Release -- current --stress +``` + +| Stress workload | Finished tasks | Active tasks | Explicit selection IDs per task | Creation order | +| --- | ---: | ---: | ---: | --- | +| Large selection history | 1,000 | 2 | 1,000 | Separate seconds | +| Many active tasks | 1,000 | 200 | 0 | Separate seconds | +| Same-second history | 1,000 | 2 | 1,000 | All finished tasks in one second, with ID order opposite to precise creation order | + To compare revisions, create a separate worktree at the baseline, copy this folder's `Program.cs` and `.csproj` into the same location there, and run the same command in each checkout. Keep the SDK, configuration, dependencies, and machine identical. Run the measurements sequentially without simultaneous builds or test suites. ## What is measured -- Seed each history size plus two older active tasks. Setup is outside the measurement. Cache values are cloned as in application use; capacity eviction is disabled so it does not change the selected workload. +- Seed each history size plus the profile's older active tasks. Setup is outside the measurement. Cache values are cloned as in application use; capacity eviction is disabled so it does not change the selected workload. - Warm each operation three times, then take 21 samples for smaller histories or seven for histories of 10,000 or more. - Report median elapsed time and total managed allocation per call, including asynchronous work. Force collection outside each sample to make retained setup state comparable. - Count service-level index reads, individual and batched record reads, index renewals, and expiry updates. The measuring decorator delegates directly to the underlying cache. -- Check that listing returns the newest finished task, the expected count, and both active tasks. Check that progress updates retain the running task and requested progress. +- Check that listing returns the newest finished task, the expected count, and every active task. Check that progress updates retain the running task and requested progress. A skipped mutation deliberately changes its local copy before returning false; the returned result must still describe the stored progress. -Allocation means bytes allocated during an operation, not retained process memory. This is a service microbenchmark, not an Elasticsearch throughput test or a Redis latency forecast. `GetAllAsync` enables batching, but provider behavior and Redis Cluster hash-slot placement determine actual network requests. Progress measurements use two active entries; costs for a larger active index must be measured separately. +Allocation means bytes allocated during an operation, not retained process memory. This is a service microbenchmark, not an Elasticsearch throughput test or a Redis latency forecast. `GetAllAsync` enables batching, but provider behavior and Redis Cluster hash-slot placement determine actual network requests. The default profile uses two active entries; the stress profile measures a larger active index separately. ## Recorded comparison -Measured on Linux x64, .NET SDK 11.0.100, Release, Foundatio 13.0.5. The baseline is [`fc804c08`](https://github.com/exceptionless/Exceptionless/commit/fc804c084506f5f78fc80b11209b49e437423587). The candidate is the service implementation accompanying this report. [comparison.json](comparison.json) contains every recorded sample summary and the candidate service blob identifier. +Measured on Linux x64, .NET SDK 11.0.100, Release, Foundatio 13.0.5. The baseline is [`fc804c08`](https://github.com/exceptionless/Exceptionless/commit/fc804c084506f5f78fc80b11209b49e437423587). The candidate is the service implementation in [`54f7cf26`](https://github.com/exceptionless/Exceptionless/commit/54f7cf264bbeb3f6a79227edd5e1c3972d56bf55), before the same-second ordering and skipped-mutation corrections. [comparison.json](comparison.json) contains every recorded sample summary and the candidate service blob identifier. | Finished history | Baseline listing | Candidate listing | Baseline allocation | Candidate allocation | | ---: | ---: | ---: | ---: | ---: | @@ -53,4 +65,21 @@ Terminal updates renew history before removing active membership. These measurem ## Remaining scaling boundary -The portable cache API has no ordered recent-history query. Listing still reads and sorts all live history IDs, and returning all active tasks requires materializing that active set. The improvement removes repeated index copies, the page cutoff, and sequential record reads. A future store with ordered discovery can change that access pattern when real volume warrants it; the [task guide](../../docs/background-tasks.md) describes that extraction boundary. +The portable cache API has no ordered recent-history query. Listing still reads and sorts all live history IDs, and returning all active tasks requires materializing that active set. Correct ordering across servers also requires reading every candidate in the cutoff second, even when only a few finished records are requested. The service trims finished candidates between bounded read batches, so retained task payloads scale with active tasks, the requested limit, and one batch rather than the entire burst. A future store with ordered discovery can change that access pattern when real volume warrants it; the [task guide](../../docs/background-tasks.md) describes that extraction boundary. + +## Correctness and stress comparison + +[extended-comparison.json](extended-comparison.json) compares `54f7cf26` with the corrected service, using identical expanded benchmark source. It records both service blob identifiers, the benchmark blob, all 42 summaries, and the same environment and sampling rules. All 21 candidate checks passed. The baseline fails every skipped-mutation return check and the same-second listing check. + +| Listing workload | Before | After | Allocation before | Allocation after | Record batches after | +| --- | ---: | ---: | ---: | ---: | ---: | +| 1,000 history, each with 1,000 selection IDs | 0.5614 ms | 0.5584 ms | 625,304 B | 623,520 B | 2 | +| 1,000 history, 200 active tasks | 1.0354 ms | 0.8396 ms | 371,792 B | 363,496 B | 5 | +| 10,000 history, 2 active tasks | 3.0749 ms | 3.0660 ms | 1,364,320 B | 1,362,536 B | 2 | +| 1,000 same-second tasks with large selections | 0.7746 ms **incorrect** | 5.0146 ms | 625,640 B **incorrect** | 9,282,976 B | 21 | + +Correct same-second ordering costs more reads and decoding allocation because the old version silently discarded newer records. The corrected service retains only 50 finished candidates between batches, with at most another 50 candidates during trimming; it does not retain all 1,000 task payloads. This is an algorithmic bound, not a measured peak-memory claim. + +The successful progress path keeps the same cache-call counts and allocation in these fixtures. At 200 active tasks it allocates 26,248 bytes per update, versus 9,208 with two active tasks; the cache's active-index work is a real scaling cost. A skipped mutation adds one read only on that path: 488 bytes for the small record and 8,896 bytes when its selection contains 1,000 IDs. This upholds the stored-result contract without cloning every successful update. + +Small-history elapsed times varied in both directions: the 100-task list measured 0.1344 ms before and 0.1722 ms after, while its allocation fell by 1,784 bytes. These results do not support a claim that every operation became faster. The stable benefits are bounded retained history payloads, fewer temporary sorting allocations, unchanged successful-update cache calls, and correct results under creation bursts. diff --git a/benchmarks/BackgroundTasks/extended-comparison.json b/benchmarks/BackgroundTasks/extended-comparison.json new file mode 100644 index 0000000000..6450439704 --- /dev/null +++ b/benchmarks/BackgroundTasks/extended-comparison.json @@ -0,0 +1,741 @@ +{ + "recordedUtcDate": "2026-11-14", + "environment": { + "os": "Linux x64", + "sdk": "11.0.100", + "configuration": "Release", + "foundatio": "13.0.5", + "cache": "InMemoryCacheClient", + "cloneValues": true, + "maxItems": null + }, + "baselineCommit": "54f7cf264bbeb3f6a79227edd5e1c3972d56bf55", + "baselineServiceBlob": "cd9773fe7feafda0cbabb1e44edf1527b37536f0", + "candidateServiceBlob": "2224e293332b8927e8852a8ddddeb62c9111fe4d", + "benchmarkProgramBlob": "2e3049ba50050610ae519bdc5865aefa1b3717de", + "method": "Three warmups, then 21 samples per operation (seven for 10,000 or more history entries); median elapsed time and total managed allocated bytes; GC outside each sample; baseline and candidate run sequentially with identical benchmark source and no concurrent test suites or builds.", + "limitations": [ + "Allocation is total bytes allocated per call, not retained or peak memory.", + "In-process cache measurements do not forecast Redis latency or Elasticsearch throughput.", + "The baseline same-second listing is incorrect; its lower cost omits required candidates.", + "Baseline no-op mutation returns the uncommitted callback copy and fails the contract check.", + "Timings are host-sensitive; operation counts and allocation expose stable work differences.", + "Terminal-history writes and bulk execution are outside this benchmark." + ], + "measurements": [ + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "list", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.1344, + "medianAllocatedBytes": 61256, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "progress", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0616, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "no-op", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0404, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.4538, + "medianAllocatedBytes": 188424, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0663, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0433, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "list", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 3.0749, + "medianAllocatedBytes": 1364320, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "progress", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.1048, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "no-op", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.0474, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "list", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 27.6162, + "medianAllocatedBytes": 6241424, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "progress", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.2345, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "history", + "operation": "no-op", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.1602, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "large-selection-history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.5614, + "medianAllocatedBytes": 625304, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "large-selection-history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0973, + "medianAllocatedBytes": 26024, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "large-selection-history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0729, + "medianAllocatedBytes": 14128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "many-active-tasks", + "operation": "list", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 1.0354, + "medianAllocatedBytes": 371792, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 5, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "many-active-tasks", + "operation": "progress", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.1029, + "medianAllocatedBytes": 26248, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "many-active-tasks", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0432, + "medianAllocatedBytes": 5720, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "same-second-history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.7746, + "medianAllocatedBytes": 625640, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "baseline-54f7cf26", + "workload": "same-second-history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.1199, + "medianAllocatedBytes": 26024, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "baseline-54f7cf26", + "workload": "same-second-history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0734, + "medianAllocatedBytes": 14128, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": false + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "list", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.1722, + "medianAllocatedBytes": 59472, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "progress", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0456, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "no-op", + "historyCount": 100, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.03, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.6504, + "medianAllocatedBytes": 186640, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0692, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0418, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "list", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 3.066, + "medianAllocatedBytes": 1362536, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "progress", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.0656, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "no-op", + "historyCount": 10000, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.0393, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "list", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 21.9808, + "medianAllocatedBytes": 6239640, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "progress", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.234, + "medianAllocatedBytes": 9208, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "history", + "operation": "no-op", + "historyCount": 50001, + "activeCount": 2, + "selectionIds": 0, + "iterations": 7, + "medianMilliseconds": 0.1751, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "large-selection-history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.5584, + "medianAllocatedBytes": 623520, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 2, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "large-selection-history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0947, + "medianAllocatedBytes": 26024, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "large-selection-history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0528, + "medianAllocatedBytes": 23024, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "many-active-tasks", + "operation": "list", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.8396, + "medianAllocatedBytes": 363496, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 5, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "many-active-tasks", + "operation": "progress", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.1004, + "medianAllocatedBytes": 26248, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "many-active-tasks", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 200, + "selectionIds": 0, + "iterations": 21, + "medianMilliseconds": 0.0434, + "medianAllocatedBytes": 6208, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "same-second-history", + "operation": "list", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 5.0146, + "medianAllocatedBytes": 9282976, + "ListReads": 2, + "SingleReads": 0, + "BatchReads": 21, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "same-second-history", + "operation": "progress", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0854, + "medianAllocatedBytes": 26024, + "ListReads": 0, + "SingleReads": 1, + "BatchReads": 0, + "ListWrites": 1, + "ExpirationWrites": 1, + "correct": true + }, + { + "label": "candidate-2224e293", + "workload": "same-second-history", + "operation": "no-op", + "historyCount": 1000, + "activeCount": 2, + "selectionIds": 1000, + "iterations": 21, + "medianMilliseconds": 0.0584, + "medianAllocatedBytes": 23024, + "ListReads": 0, + "SingleReads": 2, + "BatchReads": 0, + "ListWrites": 0, + "ExpirationWrites": 0, + "correct": true + } + ] +} diff --git a/docs/background-tasks.md b/docs/background-tasks.md index 04484e5b53..70d4755aad 100644 --- a/docs/background-tasks.md +++ b/docs/background-tasks.md @@ -17,6 +17,8 @@ The task record is the source of truth for saved progress. Counts and the bulk c Task records use the configured `ICacheClient` and expire seven days after their last save. They provide limited history rather than a durable audit log. Index entries have independent per-member expiration. Every active update renews active membership; a terminal update renews finished-history membership. This keeps long-running work discoverable without rewriting the larger history index on every progress report. The service also refreshes the index key's lifetime because renewing an existing member does not extend the key in every provider. +Task mutations require isolated snapshots from the cache: serialized providers provide this boundary, and the in-memory provider must use `CloneValues = true`. A callback must retain the task's identity and ownership. A future store adapter must preserve these requirements rather than exposing a shared mutable task instance. + Record and index writes are separate operations. A cache failure can leave incomplete discovery even when a record was saved; a no-op mutation does not repair every prior index failure. History refresh precedes active removal so normal completion has overlapping membership and a failed refresh preserves the existing fallback. Treat this cache-backed history as best effort, and use a store with stronger indexing guarantees if a future consumer requires transactional discovery. ## Creating and updating a task @@ -60,7 +62,7 @@ This uses the same work item as [SampleDataService](../src/Exceptionless.Core/Ut | `GetAsync` | Returns the stored record, or null when absent. Internal callers must enforce their own access policy. | | `GetByIdsAsync` | Returns existing tasks once per ID in input order, omitting missing records. Cache reads contain at most 50 records; callers enforce scope. | | `GetByOrganizationAsync` | Returns all discoverable active tasks plus up to `limit` finished tasks, newest first; the default finished limit is 50. | -| `UpdateAsync` | Locks one task, reloads it, runs a synchronous mutation, and saves only when the callback returns true. Returns null when the record is absent. | +| `UpdateAsync` | Locks one task, reloads it, runs a synchronous mutation, and saves only when the callback returns true. Returns the stored state, including after a skipped mutation, or null when the record is absent. | | `CancelAsync` | Records a cancellation request and returns the current task outcome, including a distinct non-cancellable result. | | `IsCancelRequestedAsync` | Lets a cooperative worker check the persisted cancellation request at a safe boundary. | @@ -70,6 +72,8 @@ Task storage and queue submission are separate operations. An enqueue exception An `UpdateAsync` callback should only inspect and change the supplied task. Perform expensive work outside it, then checkpoint the result. The service owns `UpdatedUtc`: every committed mutation receives a timestamp strictly newer than the previous record, including when a clock repeats or moves backward. The persistence lock is released before publishing; a slow message bus does not hold up another mutation. +Returning false discards the mutation and reloads the stored state for the result. This extra read occurs only on skipped updates; successful progress updates do not clone another task just to support rollback. Advisory publication has no caller cancellation token, so a transport timeout reported as `TaskCanceledException` is logged like other publication failures. It cannot reverse an accepted enqueue or a committed update. + ## Lifecycle and retry rules `IsActive` means `Queued` or `Running`. The worker owns valid transitions for its task type; the service supplies serialized mutations. @@ -85,16 +89,20 @@ An `UpdateAsync` callback should only inspect and change the supplied task. Perf | Bulk permanent error, lost selection, or exhausted checkpoint retries | `Failed`; the task does not reopen automatically | | Duplicate delivery of completed or cancelled tracked work | Skip execution | -A non-success ordinary work-item result, such as shutdown, leaves tracking active so queue delivery can resume it. Bulk shutdown also retains its saved progress and selection. An ordinary work item with no `TaskId` runs without tracking. If an ordinary task record disappears, its existing queue payload can still run; a bulk task requires its saved record and selection and stops if either is lost. +A non-success ordinary work-item result, such as shutdown, leaves tracking active so queue delivery can resume it. The wrapper has no provider-neutral retry-exhaustion signal; an arbitrary handler that repeatedly returns non-success results could exhaust deliveries while its task remains active. The current tracked reset/sample handlers use success or exceptions. Bulk shutdown also retains its saved progress and selection. An ordinary work item with no `TaskId` runs without tracking. If an ordinary task record disappears, its existing queue payload can still run; a bulk task requires its saved record and selection and stops if either is lost. `Failed` therefore describes a failed attempt for an ordinary tracked work item and a terminal outcome for a bulk operation. Consumers must allow a newer ordinary-work update to move `Failed` back to `Running`. Retry policy belongs to the queue or bulk worker, not to a status label alone. +The tracking wrapper keeps the initial start decision strict so a storage failure cannot accidentally run completed or cancelled work. Subsequent progress and outcome tracking is best effort; a tracking transport failure cannot replace the handler's original exception or prevent the job's progress callback from renewing execution locks. Cancellation of the actual work and its original progress callback still propagates normally. + ## Cancellation and staleness Cancellation is cooperative. A queued cancellable task can be marked `Cancelled` immediately. A running bulk task checks the request between batches, finishes the batch already in flight, then records `Cancelled`. Completed side effects are retained. `StaleAfter` is two hours without a saved update. Staleness is computed when needed; it is not a separate status or a timer that automatically kills work. An abandoned task can be cleared through cancellation even when its type normally cannot be stopped. Bulk admission can retire an abandoned conflicting operation before accepting a replacement. +A worker checks saved activity at the next batch boundary. Retirement therefore stops it from starting another batch even if it missed the admission decision while paused. This cannot undo side effects from a batch that was already in flight. + Cancelling a finished task is idempotent. A successful cancellation request can still return `Running` while the worker finishes its current batch. Continue observing the task until its state changes. The task snapshot does not expose a separate cancellation-pending field. ## HTTP and notification contracts @@ -110,7 +118,13 @@ All routes are under `/api/v2`. Task lookup and cancellation enforce organizatio The HTTP [view model](../src/Exceptionless.Web/Models/BackgroundTask/ViewBackgroundTask.cs) includes status, counts, message, creator, timestamps, and a selection summary for bulk work. That summary can contain the filter, original time expression, resolved UTC bounds, and explicit ID count. It omits the ID array, operation parameters, pinned membership, snapshot identifier, and resume cursor. The smaller [BackgroundTaskUpdated message](../src/Exceptionless.Core/Models/Messaging/BackgroundTaskUpdated.cs) also omits the selection summary. -Messages can be duplicated, delayed, or delivered out of order. Compare `UpdatedUtc` at .NET tick precision before applying them; JavaScript's `Date` alone discards submillisecond precision. Unknown task identifiers trigger rediscovery. The client polls while tasks are active and refetches when returning to an organization. A message is a hint to reconcile saved state, not an instruction to execute work. +Messages can be duplicated, delayed, or delivered out of order. Compare `UpdatedUtc` at .NET tick precision before applying them; JavaScript's `Date` alone discards submillisecond precision. Unknown task identifiers trigger rediscovery, including when the initial list request has not finished. Bursts share the required reconciliation request. A distinct task created after that request's snapshot can be discovered by the next poll. + +Known progress updates do not cancel list reads. At the cache commit, the client merges the HTTP result with changes received since the read began: newer task revisions win, newly inserted or changed tasks survive omission from an older response, and unchanged omitted records can disappear. This uses one baseline per in-flight query and releases it after the read; it does not buffer every progress message. + +The panel polls the current organization every three seconds while any task is active and once a minute while idle. Its fixed cadence changes only with the organization or active/idle state, so progress revisions cannot continually restart it. A tick skips hidden tabs and an already-running query; returning to an organization also refetches its list. Idle polling covers missed notifications, work started elsewhere, and ordinary queue retries after a failed attempt; it does not guess when a provider has exhausted retries. The idle cost is one task-list request per minute per visible tab. Notification delivery remains an optimization for latency. + +Dismissal and completion handling follow the task's outcome revision. A newer terminal outcome can appear and refresh affected data even when the client missed its intervening `Running` state. Repeated copies of the same outcome do not repeat that refresh. ## Bulk selection and execution @@ -118,9 +132,13 @@ Explicit ID selection accepts at most 1,000 distinct IDs. Larger requests use a For filter selections, current organization/project scope, search-plan rules, and retention can narrow the eligible set. Later arrivals or edits cannot expand it. Missing, expired, or legacy selection state fails safely and requires starting a new operation; the worker does not silently re-run a live filter. Empty selections have an explicit persisted representation because the application serializer omits empty collections. -Preparation runs before the organization admission lock. Destructive bulk operations that overlap the same project cannot be admitted together; an organization-wide operation overlaps every project. Each batch executes at most the configured number of IDs, with a default batch size of 500. Event deletion fetches only identity, daily-index routing, organization, project, and stack fields, then uses the same deletion service as individual deletes. +Preparation runs before the organization admission lock. Admission serializes destructive bulk operations that overlap the same project while its lease is valid; an organization-wide operation overlaps every project. Each batch executes at most the configured number of IDs, with a default batch size of 500. Event deletion fetches only identity, daily-index routing, organization, project, and stack fields, then uses the same deletion service as individual deletes. + +Admission renews its lease between bounded conflict-read batches, before stale-task retirement, and before indexing the candidate. Monotonic elapsed time detects a scan or renewal that outlived the prior lease; that request returns busy instead of admitting work from an old snapshot. Cache leases are not fencing tokens: a process suspended across lease expiry during the final store writes, or a cache that loses a live lock key, cannot provide a cross-store transactional exclusion guarantee. Handlers must still tolerate replay and already-running side effects. + +The snapshot has a two-hour idle lease, renewed while paging. The cached immutable selection also has its own seven-day retention, renewed as the active worker checkpoints so a task older than seven days can still resume. Renewing that expiry does not rewrite the selection payload. Terminal or rejected work releases the snapshot best effort; active retries and shutdown retain it. Expiry bounds the resource when cleanup cannot reach the backend. -The snapshot has a two-hour idle lease, renewed while paging. Terminal or rejected work releases it best effort; active retries and shutdown retain it. Expiry bounds the resource when cleanup cannot reach the backend. +The [pinned Foundatio.Repositories implementation](https://github.com/FoundatioFx/Foundatio.Repositories/blob/690e3f7d2c9a81cad2f658e10c29cc609f1e79f7/src/Foundatio.Repositories.Elasticsearch/Repositories/ElasticReadOnlyRepositoryBase.cs) can open a snapshot and then throw before returning its identifier during preparation. The caller cannot release an identifier it never received; that snapshot remains bounded by its idle lease. Fixing this boundary belongs in the provider's failed-query cleanup, not in a second Elasticsearch access path in the task service. Completion does not force an Elasticsearch index refresh. Deletion uses the repository's normal eventual search consistency, so search results can briefly retain deleted events until the index refreshes. @@ -141,6 +159,8 @@ For a bulk action, implement `IBulkOperationHandler` with a unique operation typ Keep immutable selection data separate from frequently updated progress where the existing format permits it. Process one batch at a time, project only required fields, and use the cache's bulk-read operation for groups of task records. Listing reads each index once, orders the IDs, and loads records in batches of at most 50. It has no silent history-page cutoff. Conflict detection uses the same bounded record reads while checking the active index. +ObjectId order determines the creation second, but IDs from different servers do not determine creation order inside that second. Listing reads the full cutoff second and selects by precise `CreatedUtc`, then ID. It retains only the requested finished history between batches, plus all active tasks. A burst in the cutoff second increases record reads without retaining the burst's entire task payload. + Returning all active tasks inherently requires work proportional to that active set. The generic cache-list API does not provide a portable ordered recent-history query, so listing still reads and orders the live history IDs. `GetAllAsync` lets providers combine reads; it does not promise one network round trip on every provider or Redis Cluster topology. Active-index renewal adds writes to a progress update, while the history index is renewed only on terminal updates. Measure these operations separately when changing retention or storage. | Reusable responsibility | Keep in the application adapter | @@ -156,6 +176,8 @@ The current `BackgroundTask` includes organization/project/user fields and `Bulk Service tests use real Foundatio in-memory infrastructure and controlled time. Regression coverage belongs at the failure boundary: factory/index/queue failures, concurrent updates, delayed notifications, cancellation, retention, full selection paging, and checkpoint recovery. Integration tests exercise real serialization, Elasticsearch routing, authorization, usage accounting, and selection mutation. Browser tests cover discovery, concurrent task completion, and cancellation access. +Provider behavior remains part of this contract. Foundatio 13.0.5's [in-memory expiry cleanup](https://github.com/FoundatioFx/Foundatio/blob/be10785cfed4e1850350dfaa024e1c3f25b7fac7/src/Foundatio/Caching/InMemoryCacheClient.cs) can observe an expired entry and then remove a concurrently renewed replacement by key. The controlled-time retention regression exposed its `Removed item was not expired` diagnostic. This provider race is not repaired by task-service locking; the cache-loss recovery described above still applies. It is distinct from the Redis provider and should be addressed in the in-memory cache implementation before relying on stronger retention guarantees there. + Start with the focused classes under [BackgroundTasks](../tests/Exceptionless.Tests/Services/BackgroundTasks) and [BulkOperations](../tests/Exceptionless.Tests/Services/BulkOperations), then the [worker tests](../tests/Exceptionless.Tests/Jobs/WorkItemHandlers), endpoint tests, and the full CI run. Manual performance measurements live in [benchmarks/BackgroundTasks](../benchmarks/BackgroundTasks); they report service time, allocation, and cache calls separately from provider/network throughput. Deploy the updated job before or with the web application. Cache loss can remove progress and prepared selections; treat that as an interrupted operation requiring user-visible recovery. This feature adds no database migration or index. Preserve public DTOs and message shapes when changing internal storage or moving reusable components to Foundatio. From 2130ec1720cc265d23a355694c55d1064c0a4442 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 15 Nov 2026 12:08:46 -0600 Subject: [PATCH 39/42] Preserve worker outcomes through shutdown and accounting failures --- .../BackgroundTaskTrackingWorkItemHandler.cs | 10 +- .../GenerateSampleEventsWorkItemHandler.cs | 21 +++- .../Services/EventDeletionService.cs | 5 +- ...kgroundTaskTrackingWorkItemHandlerTests.cs | 42 +++++++ ...enerateSampleEventsWorkItemHandlerTests.cs | 108 ++++++++++++++++++ .../Services/EventDeletionServiceTests.cs | 108 ++++++++++++++++++ 6 files changed, 280 insertions(+), 14 deletions(-) create mode 100644 tests/Exceptionless.Tests/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandlerTests.cs create mode 100644 tests/Exceptionless.Tests/Services/EventDeletionServiceTests.cs diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs index 45459c3070..11185a1e64 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandler.cs @@ -10,9 +10,8 @@ namespace Exceptionless.Core.Jobs.WorkItemHandlers; /// /// Wraps the handler of a work item that is tracked by a background task. Progress, success and failure are written to the -/// task in the job process that runs the work (not from a bus subscriber, which every web instance would run), then the -/// original behavior happens: the wrapped handler runs and the job's own progress callback still renews the locks. Work items -/// without a task id pass straight through. +/// task in the job process that runs the work. Progress reaches the job's original callback before tracking I/O, preserving +/// lock renewal when enabled. Work items without a task id pass straight through. /// internal sealed class BackgroundTaskTrackingWorkItemHandler : IWorkItemHandler { @@ -65,6 +64,9 @@ public async Task HandleItemAsync(WorkItemContext context) string? lastMessage = null; var tracked = new WorkItemContext(context.Data, context.JobId, context.WorkItemLock, context.CancellationToken, async (progress, message) => { + // Give the job its renewal opportunity before tracking waits on cache or messaging. + await context.ReportProgressAsync(progress, message); + lastMessage = message ?? lastMessage; await TryUpdateAsync(taskId, task => { @@ -76,8 +78,6 @@ await TryUpdateAsync(taskId, task => task.Message = message; return true; }); - - await context.ReportProgressAsync(progress, message); }); if (!await StartAsync(taskId)) diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs index f88d91c33f..f8611080de 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs @@ -84,18 +84,26 @@ public override async Task HandleItemAsync(WorkItemContext context) for (int p = 0; p < projectList.Count; p++) { + int projectEventCount = eventsPerProject + (p < remainder ? 1 : 0); + if (projectEventCount == 0) + continue; + if (context.CancellationToken.IsCancellationRequested) - break; + { + context.Result = JobResult.CancelledWithMessage("Stopping sample event generation for shutdown"); + return; + } var project = projectList[p]; - int projectEventCount = eventsPerProject + (p < remainder ? 1 : 0); - var events = generator.Generate(organization.Id, project.Id, projectEventCount, minDate, utcNow); foreach (var batch in events.Chunk(batchSize)) { if (context.CancellationToken.IsCancellationRequested) - break; + { + context.Result = JobResult.CancelledWithMessage("Stopping sample event generation for shutdown"); + return; + } await _eventPipeline.RunAsync(batch, organization, project, allowExtendedEventDateRange: true); totalProcessed += batch.Length; @@ -138,7 +146,10 @@ private async Task GenerateProjectSampleEventsAsync(WorkItemContext context, Ran foreach (var batch in events.Chunk(batchSize)) { if (context.CancellationToken.IsCancellationRequested) - break; + { + context.Result = JobResult.CancelledWithMessage("Stopping sample event generation for shutdown"); + return; + } await _eventPipeline.RunAsync(batch, organization, project, allowExtendedEventDateRange: true); totalProcessed += batch.Length; diff --git a/src/Exceptionless.Core/Services/EventDeletionService.cs b/src/Exceptionless.Core/Services/EventDeletionService.cs index d104e78187..382cb91db9 100644 --- a/src/Exceptionless.Core/Services/EventDeletionService.cs +++ b/src/Exceptionless.Core/Services/EventDeletionService.cs @@ -36,12 +36,9 @@ public async Task DeleteAsync(IReadOnlyCollection events) { await _usageService.IncrementDeletedAsync(projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, projectGroup.Count()); } - catch (OperationCanceledException) - { - throw; - } catch (Exception ex) { + // Accounting has no caller cancellation token. A transport timeout must not replay a completed delete. _logger.LogWarning(ex, "Failed to increment deleted usage metrics for organization {OrganizationId} project {ProjectId}: {Message}", projectGroup.Key.OrganizationId, projectGroup.Key.ProjectId, ex.Message); } } diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs index 9ac6bc9a8d..945c67bcc2 100644 --- a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/BackgroundTaskTrackingWorkItemHandlerTests.cs @@ -61,6 +61,48 @@ await handler.HandleItemAsync(CreateContext(task.Id, (progress, message) => Assert.Contains(_harness.Publisher.Updates, u => u.Status == BackgroundTaskStatus.Running && u.Progress == 50); } + [Fact] + public async Task HandleItemAsync_ProgressPublicationWaits_ReportsToTheJobBeforeTrackingFinishes() + { + // Arrange + var task = await AddTaskAsync(); + var publishing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var releasePublication = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + bool reported = false; + var handler = CreateHandler(async context => + { + _harness.Publisher.BeforePublish = () => + { + _harness.Publisher.BeforePublish = null; + publishing.SetResult(); + return releasePublication.Task; + }; + await context.ReportProgressAsync(50, "Halfway"); + }); + var context = CreateContext(task.Id, (_, _) => + { + reported = true; + return Task.CompletedTask; + }); + + // Act + var handling = handler.HandleItemAsync(context); + bool reportedWhilePublishing; + try + { + await publishing.Task.WaitAsync(TestCancellationToken); + reportedWhilePublishing = reported; + } + finally + { + releasePublication.SetResult(); + await handling; + } + + // Assert + Assert.True(reportedWhilePublishing); + } + [Fact] public async Task HandleItemAsync_ProgressTrackingIsCancelled_StillReportsProgressAndCompletesTheWork() { diff --git a/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandlerTests.cs b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandlerTests.cs new file mode 100644 index 0000000000..a2fe5248a8 --- /dev/null +++ b/tests/Exceptionless.Tests/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandlerTests.cs @@ -0,0 +1,108 @@ +using Exceptionless.Core; +using Exceptionless.Core.Jobs.WorkItemHandlers; +using Exceptionless.Core.Models; +using Exceptionless.Core.Models.BackgroundTasks; +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Plugins.EventProcessor; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Utility; +using Exceptionless.Tests.Utility; +using Foundatio.Jobs; +using Foundatio.Lock; +using Foundatio.Repositories; +using Foundatio.Repositories.Models; +using Foundatio.Repositories.Options; +using Foundatio.Utility; +using Foundatio.Xunit; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Time.Testing; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Jobs.WorkItemHandlers; + +public class GenerateSampleEventsWorkItemHandlerTests(ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Theory] + [InlineData(true, 101, 1, 100, false, BackgroundTaskStatus.Running)] + [InlineData(true, 100, 1, 100, true, BackgroundTaskStatus.Completed)] + [InlineData(false, 101, 1, 100, false, BackgroundTaskStatus.Queued)] + [InlineData(false, 100, 1, 100, true, BackgroundTaskStatus.Queued)] + [InlineData(false, 200, 2, 100, false, BackgroundTaskStatus.Queued)] + [InlineData(false, 1, 2, 1, true, BackgroundTaskStatus.Queued)] + public async Task HandleItemAsync_HostStopsAfterBatch_OnlyCompletesWhenAllWorkFinished( + bool projectScoped, int eventCount, int projectCount, int expectedProcessed, bool expectSuccess, BackgroundTaskStatus expectedTaskStatus) + { + // Arrange + var time = new FakeTimeProvider(); + using var harness = new BackgroundTaskTestHarness(time, Log); + using var services = new ServiceCollection().BuildServiceProvider(); + using var shutdown = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + var organization = new Organization { Id = SampleDataService.TEST_ORG_ID }; + var project = new Project { Id = SampleDataService.TEST_PROJECT_ID, OrganizationId = organization.Id }; + var organizations = new Mock(); + organizations.Setup(repository => repository.GetByIdAsync(organization.Id, It.IsAny())) + .ReturnsAsync(organization); + var projects = new Mock(); + projects.Setup(repository => repository.GetByIdAsync(project.Id, It.IsAny())) + .ReturnsAsync(project); + var hits = Enumerable.Range(0, projectCount) + .Select(index => new Project { Id = index == 0 ? project.Id : SampleDataService.TEST_ROCKET_SHIP_PROJECT_ID, OrganizationId = organization.Id }) + .Select(item => new FindHit(item.Id, item, 0)); + projects.Setup(repository => repository.GetByOrganizationIdAsync(organization.Id, It.IsAny>())) + .ReturnsAsync(new FindResults(hits, projectCount)); + var pipeline = new RecordingEventPipeline(services, Log, shutdown.Cancel); + var inner = new GenerateSampleEventsWorkItemHandler(pipeline, organizations.Object, projects.Object, harness.LockProvider, time, Log); + var handler = new BackgroundTaskTrackingWorkItemHandler(inner, harness.Tasks, time); + var task = new BackgroundTask + { + OrganizationId = organization.Id, + ProjectId = project.Id, + Type = BackgroundTaskTypes.GenerateProjectSampleData, + Description = "Generate sample data", + CreatedByUserId = "537650f3b77efe23a47914f8" + }; + await harness.Tasks.AddAsync(task); + var workItem = new GenerateSampleEventsWorkItem + { + OrganizationId = projectScoped ? organization.Id : null, + ProjectId = projectScoped ? project.Id : null, + TaskId = projectScoped ? task.Id : null, + EventCount = eventCount + }; + var reported = new List(); + var context = new WorkItemContext(workItem, "test-job", EmptyLock.Empty, shutdown.Token, (progress, _) => + { + reported.Add(progress); + return Task.CompletedTask; + }); + + // Act + await handler.HandleItemAsync(context); + var stored = await harness.Tasks.GetAsync(task.Id); + + // Assert + Assert.Equal(expectedProcessed, pipeline.Processed); + Assert.Equal(expectSuccess, context.Result.IsSuccess); + Assert.Equal(expectSuccess, reported.Contains(100)); + Assert.Equal(expectedTaskStatus, stored!.Status); + Assert.Equal(expectedTaskStatus is BackgroundTaskStatus.Completed, stored.CompletedUtc.HasValue); + } + + private sealed class RecordingEventPipeline(IServiceProvider services, ILoggerFactory loggerFactory, Action afterBatch) + : EventPipeline(services, new AppOptions(), loggerFactory) + { + public int Processed { get; private set; } + + public override Task> RunAsync(ICollection contexts) + { + Processed += contexts.Count; + afterBatch(); + return Task.FromResult(contexts); + } + + protected override IList GetActionTypes() => []; + } +} diff --git a/tests/Exceptionless.Tests/Services/EventDeletionServiceTests.cs b/tests/Exceptionless.Tests/Services/EventDeletionServiceTests.cs new file mode 100644 index 0000000000..424d5fb9cf --- /dev/null +++ b/tests/Exceptionless.Tests/Services/EventDeletionServiceTests.cs @@ -0,0 +1,108 @@ +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; +using Foundatio.Caching; +using Foundatio.Messaging; +using Foundatio.Repositories; +using Moq; +using Xunit; + +namespace Exceptionless.Tests.Services; + +public class EventDeletionServiceTests : TestWithServices +{ + private const string OrganizationId = "537650f3b77efe23a47914f4"; + private const string FirstProjectId = "537650f3b77efe23a47914f6"; + private const string SecondProjectId = "537650f3b77efe23a47914f7"; + private readonly IOrganizationRepository _organizations; + private readonly IProjectRepository _projects; + private readonly IMessagePublisher _publisher; + private readonly NotificationService _notifications; + private readonly ILogger _deletionLogger; + + public EventDeletionServiceTests(ITestOutputHelper output) : base(output) + { + _organizations = GetService(); + _projects = GetService(); + _publisher = GetService(); + _notifications = GetService(); + _deletionLogger = GetService>(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DeleteAsync_UsageTransportFails_PreservesDeletionAndRecordsRemainingProjects(bool dependencyCancelled) + { + // Arrange + using var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { TimeProvider = TimeProvider, LoggerFactory = Log }); + var failure = dependencyCancelled + ? (Exception)new TaskCanceledException("Usage transport timed out") + : new TimeoutException("Usage transport timed out"); + var usageCache = new FaultingUsageCache(cache, failure); + var events = new Mock(); + events.Setup(repository => repository.RemoveAsync(It.IsAny>(), It.IsAny())) + .Returns(Task.CompletedTask); + var service = CreateService(events.Object, usageCache); + PersistentEvent[] selected = + [ + new() { Id = "first", OrganizationId = OrganizationId, ProjectId = FirstProjectId }, + new() { Id = "second", OrganizationId = OrganizationId, ProjectId = SecondProjectId } + ]; + + // Act + var thrown = await Record.ExceptionAsync(() => service.DeleteAsync(selected)); + var keys = cache.Keys.Where(key => key.EndsWith($":{OrganizationId}:{SecondProjectId}:deleted", StringComparison.Ordinal)); + var remainingProjectUsage = await cache.GetAllAsync(keys); + + // Assert + Assert.Null(thrown); + events.Verify(repository => repository.RemoveAsync(selected, It.IsAny()), Times.Once); + Assert.True(usageCache.Failed); + var recorded = Assert.Single(remainingProjectUsage).Value; + Assert.True(recorded.HasValue); + Assert.Equal(1, recorded.Value); + } + + [Fact] + public async Task DeleteAsync_DeletionIsCancelled_PropagatesWithoutRecordingUsage() + { + // Arrange + using var cache = new InMemoryCacheClient(new InMemoryCacheClientOptions { TimeProvider = TimeProvider, LoggerFactory = Log }); + var failure = new TaskCanceledException("Deletion transport timed out"); + var events = new Mock(); + events.Setup(repository => repository.RemoveAsync(It.IsAny>(), It.IsAny())) + .ThrowsAsync(failure); + var service = CreateService(events.Object, cache); + PersistentEvent[] selected = [new() { Id = "first", OrganizationId = OrganizationId, ProjectId = FirstProjectId }]; + + // Act + var thrown = await Record.ExceptionAsync(() => service.DeleteAsync(selected)); + + // Assert + Assert.Same(failure, thrown); + Assert.Empty(cache.Keys); + } + + private EventDeletionService CreateService(IEventRepository events, ICacheClient cache) + { + var usage = new UsageService(_organizations, _projects, cache, _publisher, _notifications, TimeProvider, Log); + return new EventDeletionService(events, usage, _deletionLogger); + } + + private sealed class FaultingUsageCache(ICacheClient inner, Exception failure) : ScopedCacheClient(inner, null), ICacheClient + { + public bool Failed { get; private set; } + + Task ICacheClient.IncrementAsync(string key, long amount, TimeSpan? expiresIn) + { + if (key.EndsWith($":{FirstProjectId}:deleted", StringComparison.Ordinal)) + { + Failed = true; + return Task.FromException(failure); + } + + return UnscopedCache.IncrementAsync(key, amount, expiresIn); + } + } +} From ec1125e4494fc8e48faac15238f9a2f2a23d7f6c Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 15 Nov 2026 12:08:49 -0600 Subject: [PATCH 40/42] Reconcile task outcomes across initial discovery and pending reads --- .../src/lib/features/tasks/api.svelte.ts | 23 +++-- .../src/lib/features/tasks/api.test.ts | 94 +++++++++++++++++-- .../tasks/components/task-panel.svelte | 6 +- .../components/task-panel.svelte.test.ts | 30 ++++-- 4 files changed, 126 insertions(+), 27 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts index daccfa36a7..06fcaa7b7d 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.svelte.ts @@ -108,14 +108,23 @@ export function getTasksQuery(request: GetTasksRequest) { /** Refreshes everything a finished bulk task may have changed. */ export async function invalidateTaskAffectedQueries(queryClient: QueryClient) { - await Promise.all([ - queryClient.invalidateQueries({ - queryKey: eventQueryKeys.type - }), - queryClient.invalidateQueries({ - queryKey: stackQueryKeys.type + await Promise.all( + [eventQueryKeys.type, stackQueryKeys.type].map(async (queryKey) => { + // Invalidation alone reuses an uncached initial fetch, which may predate the completed operation. + await queryClient.cancelQueries({ + queryKey + }); + // Simultaneous completions can share the replacement read; a later completion cancels it above. + await queryClient.invalidateQueries( + { + queryKey + }, + { + cancelRefetch: false + } + ); }) - ]); + ); } /** Refetches the task lists, e.g. after a request that starts a task. */ diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts index faa68ffbb2..fae2f0b23c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/api.test.ts @@ -169,16 +169,96 @@ describe('updateTaskFromMessage', () => { }); describe('invalidateTaskAffectedQueries', () => { - it('refreshes event and stack queries', async () => { + it.each([false, true])('replaces affected reads started before task completion with cached data: %s', async (cached) => { // Arrange - const queryClient = { invalidateQueries: vi.fn().mockResolvedValue(undefined) }; + const client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: 60_000 } } }); + const lists = [['PersistentEvent'], ['Stack']].map((queryKey) => { + const pending = Promise.withResolvers(); + const queryFn = vi.fn().mockReturnValueOnce(pending.promise).mockResolvedValue([]); + if (cached) { + client.setQueryData(queryKey, ['deleted-item']); + } + const observer = new QueryObserver(client, { queryFn, queryKey, refetchOnMount: false }); + const unsubscribe = observer.subscribe(() => {}); + const read = observer.refetch(); + return { observer, pending, queryFn, read, unsubscribe }; + }); - // Act - await invalidateTaskAffectedQueries(queryClient as never); + try { + // Act + // With removal pushes unavailable, the task panel learns the delete completed through polling. + const completion = invalidateTaskAffectedQueries(client); + for (const list of lists) { + list.pending.resolve(['deleted-item']); + } + await Promise.all([completion, ...lists.map((list) => list.read)]); - // Assert - expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: ['PersistentEvent'] }); - expect(queryClient.invalidateQueries).toHaveBeenCalledWith({ queryKey: ['Stack'] }); + // Assert + for (const list of lists) { + expect(list.observer.getCurrentResult().data).toEqual([]); + expect(list.queryFn).toHaveBeenCalledTimes(2); + expect(list.observer.getCurrentResult().isStale).toBe(false); + } + } finally { + for (const list of lists) { + list.unsubscribe(); + } + client.clear(); + } + }); + + it.each([false, true])('shares simultaneous completion refreshes and replaces them after a later completion with cached data: %s', async (cached) => { + // Arrange + const client = new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: 60_000 } } }); + const queryKey = ['PersistentEvent', 'organizations', 'organization-id', 'events']; + const beforeCompletion = Promise.withResolvers(); + const firstRefresh = Promise.withResolvers(); + const firstRefreshStarted = Promise.withResolvers(); + const lastRefresh = Promise.withResolvers(); + const lastRefreshStarted = Promise.withResolvers(); + const queryFn = vi + .fn() + .mockReturnValueOnce(beforeCompletion.promise) + .mockImplementationOnce(() => { + firstRefreshStarted.resolve(); + return firstRefresh.promise; + }) + .mockImplementationOnce(() => { + lastRefreshStarted.resolve(); + return lastRefresh.promise; + }); + if (cached) { + client.setQueryData(queryKey, ['first-event', 'second-event']); + } + const observer = new QueryObserver(client, { queryFn, queryKey, refetchOnMount: false }); + const unsubscribe = observer.subscribe(() => {}); + const initialRead = observer.refetch(); + + try { + // Act + const simultaneous = [invalidateTaskAffectedQueries(client), invalidateTaskAffectedQueries(client)]; + await firstRefreshStarted.promise; + + // Assert + expect(queryFn).toHaveBeenCalledTimes(2); + + // Act + // Another delete finishes after the shared refresh took its snapshot. + const laterCompletion = invalidateTaskAffectedQueries(client); + await lastRefreshStarted.promise; + beforeCompletion.resolve(['first-event', 'second-event']); + firstRefresh.resolve(['second-event']); + lastRefresh.resolve([]); + await Promise.all([initialRead, ...simultaneous, laterCompletion]); + + // Assert + expect(queryFn).toHaveBeenCalledTimes(3); + expect(observer.getCurrentResult().data).toEqual([]); + expect(observer.getCurrentResult().isError).toBe(false); + } finally { + unsubscribe(); + client.clear(); + } }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte index 81edaf4374..936e025d9c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte @@ -45,7 +45,7 @@ return () => window.clearInterval(timer); }); - // Reconcile each new outcome even when a retry's running update was missed. The first list establishes a baseline. + // Reconcile each new outcome, including the first list: other views may have loaded before a task finished. $effect(() => { const tasks = tasksQuery.data; const organizationId = organization.current; @@ -76,10 +76,6 @@ continue; } - if (!previousTasks) { - continue; - } - cancellingIds.delete(task.id); hasFinishedTasks = true; } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts index 54a80e3fa5..4185da09af 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts @@ -89,7 +89,7 @@ describe('TaskPanel', () => { vi.useRealTimers(); }); - it('hydrates running tasks from the list after a reload and ignores stale finished ones', async () => { + it('hydrates running tasks and reconciles older outcomes without displaying them', async () => { // Arrange state.set('tasks', [createTask(), createTask({ id: 'long-finished', status: BackgroundTaskStatus.Completed, updated_utc: '2020-01-01T00:00:00Z' })]); @@ -100,7 +100,17 @@ describe('TaskPanel', () => { // Assert expect(screen.getAllByText('Delete 50 events matching filter')).toHaveLength(1); expect(document.querySelector('[data-slot="task-progress"]')?.textContent?.replace(/\s+/g, ' ').trim()).toBe('20 of 50'); - expect(mocks.invalidateAffected).not.toHaveBeenCalled(); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + + // Act + state.set( + 'tasks', + state.get('tasks')!.map((task) => ({ ...task })) + ); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); }); it('keeps polling across progress and replaces it only when activity or organization changes', async () => { @@ -243,6 +253,7 @@ describe('TaskPanel', () => { // Assert expect(screen.queryByText('Delete 50 events matching filter')).toBeNull(); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); // Act state.set('tasks', [createTask()]); @@ -257,7 +268,7 @@ describe('TaskPanel', () => { // Assert expect(document.querySelector('[data-slot="task-progress"]')?.textContent).toContain('Completed'); - expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); }); it.each([BackgroundTaskStatus.Completed, BackgroundTaskStatus.Failed])( @@ -279,7 +290,7 @@ describe('TaskPanel', () => { // Assert expect(screen.getByRole('button', { name: /^Dismiss / })).toBeVisible(); expect(document.querySelector('[data-slot="task"]')).toHaveAttribute('data-status', status); - expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); // Act await fireEvent.click(screen.getByRole('button', { name: /^Dismiss / })); @@ -288,13 +299,16 @@ describe('TaskPanel', () => { // Assert expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); - expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); } ); - it('reconciles a newly discovered completed task even when it never saw the task running', async () => { + it.each(['unavailable', 'empty'])('reconciles a newly discovered completed task when its initial list was %s', async (initialList) => { // Arrange - state.set('tasks', []); + // Event and stack views can finish loading while the first task list is still unavailable. + if (initialList === 'empty') { + state.set('tasks', []); + } render(TaskPanel); await tick(); const completed = createTask({ status: BackgroundTaskStatus.Completed }); @@ -336,7 +350,7 @@ describe('TaskPanel', () => { // Assert expect(screen.getByRole('button', { name: /^Dismiss / })).toBeVisible(); - expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); }); it('renders nothing when there is nothing to show', () => { From b6b0779b32ebda23ce4cb5cfdc2c2c740bc22334 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 15 Nov 2026 12:08:51 -0600 Subject: [PATCH 41/42] Clarify task recovery and reconciliation guarantees --- docs/background-tasks.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/background-tasks.md b/docs/background-tasks.md index 70d4755aad..319093d571 100644 --- a/docs/background-tasks.md +++ b/docs/background-tasks.md @@ -19,6 +19,8 @@ Task records use the configured `ICacheClient` and expire seven days after their Task mutations require isolated snapshots from the cache: serialized providers provide this boundary, and the in-memory provider must use `CloneValues = true`. A callback must retain the task's identity and ownership. A future store adapter must preserve these requirements rather than exposing a shared mutable task instance. +Updates are serialized while their update lease remains valid. Like admission locks, these cache leases are not fencing tokens: a process or store call that outlives its lease can race a newer writer. A future durable store needs conditional revisions or equivalent fencing before promising stronger write isolation. + Record and index writes are separate operations. A cache failure can leave incomplete discovery even when a record was saved; a no-op mutation does not repair every prior index failure. History refresh precedes active removal so normal completion has overlapping membership and a failed refresh preserves the existing fallback. Treat this cache-backed history as best effort, and use a store with stronger indexing guarantees if a future consumer requires transactional discovery. ## Creating and updating a task @@ -89,11 +91,11 @@ Returning false discards the mutation and reloads the stored state for the resul | Bulk permanent error, lost selection, or exhausted checkpoint retries | `Failed`; the task does not reopen automatically | | Duplicate delivery of completed or cancelled tracked work | Skip execution | -A non-success ordinary work-item result, such as shutdown, leaves tracking active so queue delivery can resume it. The wrapper has no provider-neutral retry-exhaustion signal; an arbitrary handler that repeatedly returns non-success results could exhaust deliveries while its task remains active. The current tracked reset/sample handlers use success or exceptions. Bulk shutdown also retains its saved progress and selection. An ordinary work item with no `TaskId` runs without tracking. If an ordinary task record disappears, its existing queue payload can still run; a bulk task requires its saved record and selection and stops if either is lost. +A non-success ordinary work-item result, such as shutdown, leaves tracking active so queue delivery can resume it. Sample generation returns non-success when shutdown leaves a required batch unstarted; shutdown after its final required batch remains successful. Ordinary queue retries can replay earlier side effects. The wrapper has no provider-neutral retry-exhaustion signal; repeated non-success deliveries can therefore exhaust retries while their task remains active. Bulk shutdown also retains its saved progress and selection. An ordinary work item with no `TaskId` runs without tracking. If an ordinary task record disappears, its existing queue payload can still run; a bulk task requires its saved record and selection and stops if either is lost. `Failed` therefore describes a failed attempt for an ordinary tracked work item and a terminal outcome for a bulk operation. Consumers must allow a newer ordinary-work update to move `Failed` back to `Running`. Retry policy belongs to the queue or bulk worker, not to a status label alone. -The tracking wrapper keeps the initial start decision strict so a storage failure cannot accidentally run completed or cancelled work. Subsequent progress and outcome tracking is best effort; a tracking transport failure cannot replace the handler's original exception or prevent the job's progress callback from renewing execution locks. Cancellation of the actual work and its original progress callback still propagates normally. +The tracking wrapper keeps the initial start decision strict so a storage failure cannot accidentally run completed or cancelled work. Subsequent progress and outcome tracking is best effort. The wrapper forwards the job's original progress callback before tracking I/O, including execution-lock renewal when the handler enables it. A tracking transport failure cannot replace the handler's original exception. Cancellation of the actual work and its original progress callback still propagates normally. ## Cancellation and staleness @@ -126,6 +128,8 @@ The panel polls the current organization every three seconds while any task is a Dismissal and completion handling follow the task's outcome revision. A newer terminal outcome can appear and refresh affected data even when the client missed its intervening `Running` state. Repeated copies of the same outcome do not repeat that refresh. +The first successful task list also reconciles affected event and stack queries when it contains finished tasks. This covers completion during delayed initial discovery or missed notifications. Old finished history can cause one additional refresh on first discovery for that organization; later identical polls do not. Reconciliation supersedes reads started before the outcome, including uncached initial reads, and simultaneous completions share the replacement request. This guarantees a new query attempt, while normal Elasticsearch search visibility still applies. + ## Bulk selection and execution Explicit ID selection accepts at most 1,000 distinct IDs. Larger requests use a filter. The resolver captures filter membership during preparation: a complete small selection keeps fixed IDs, while a larger selection keeps an Elasticsearch point-in-time snapshot and search-after cursor. Relative dates use a fixed UTC anchor and invariant formatting. @@ -142,6 +146,8 @@ The [pinned Foundatio.Repositories implementation](https://github.com/FoundatioF Completion does not force an Elasticsearch index refresh. Deletion uses the repository's normal eventual search consistency, so search results can briefly retain deleted events until the index refreshes. +Usage accounting after successful deletion is best effort. Its transport failures, including cancellation without a caller token, do not turn a completed deletion into a failed batch or prevent accounting for other projects. A failure of the actual deletion still propagates. + Deletion and checkpoint persistence are separate writes. The worker retries saving a known batch result before considering another execution. A process can still stop after a side effect but before its checkpoint is saved, so handlers must tolerate replay. Missing items are reported as failed; counts are not an exactly-once ledger of external side effects. ## Adding work From e0320bb224004b42de242662dbd44b906b6bc8b2 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 15 Nov 2026 21:26:53 -0600 Subject: [PATCH 42/42] Preserve task discovery through retries and history cleanup --- docs/background-tasks.md | 5 +- .../BackgroundTasks/BackgroundTaskService.cs | 38 ++++++--- .../BulkOperations/BulkOperationService.cs | 9 +- .../tasks/components/task-panel.svelte | 17 ++-- .../components/task-panel.svelte.test.ts | 47 +++++++++++ .../BackgroundTaskServiceTests.cs | 84 +++++++++++++++++++ .../BulkOperationServiceTests.cs | 48 +++++++++++ 7 files changed, 224 insertions(+), 24 deletions(-) diff --git a/docs/background-tasks.md b/docs/background-tasks.md index 319093d571..7bcf5d09ff 100644 --- a/docs/background-tasks.md +++ b/docs/background-tasks.md @@ -17,6 +17,8 @@ The task record is the source of truth for saved progress. Counts and the bulk c Task records use the configured `ICacheClient` and expire seven days after their last save. They provide limited history rather than a durable audit log. Index entries have independent per-member expiration. Every active update renews active membership; a terminal update renews finished-history membership. This keeps long-running work discoverable without rewriting the larger history index on every progress report. The service also refreshes the index key's lifetime because renewing an existing member does not extend the key in every provider. +Index cleanup re-reads suspected stale entries under the same task-update lease. It skips busy tasks, retains tasks that became active, and removes history only when the record is still absent. This avoids waiting behind progress writes or deleting a retry's renewed membership. Readers clean up within their existing batches; normal active reads acquire no additional locks. + Task mutations require isolated snapshots from the cache: serialized providers provide this boundary, and the in-memory provider must use `CloneValues = true`. A callback must retain the task's identity and ownership. A future store adapter must preserve these requirements rather than exposing a shared mutable task instance. Updates are serialized while their update lease remains valid. Like admission locks, these cache leases are not fencing tokens: a process or store call that outlives its lease can race a newer writer. A future durable store needs conditional revisions or equivalent fencing before promising stronger write isolation. @@ -64,6 +66,7 @@ This uses the same work item as [SampleDataService](../src/Exceptionless.Core/Ut | `GetAsync` | Returns the stored record, or null when absent. Internal callers must enforce their own access policy. | | `GetByIdsAsync` | Returns existing tasks once per ID in input order, omitting missing records. Cache reads contain at most 50 records; callers enforce scope. | | `GetByOrganizationAsync` | Returns all discoverable active tasks plus up to `limit` finished tasks, newest first; the default finished limit is 50. | +| `PruneIndexesAsync` | Rechecks suspected stale IDs under their update leases. Removes finished tasks from the active index and missing tasks from both indexes; skips busy, active, or foreign tasks. | | `UpdateAsync` | Locks one task, reloads it, runs a synchronous mutation, and saves only when the callback returns true. Returns the stored state, including after a skipped mutation, or null when the record is absent. | | `CancelAsync` | Records a cancellation request and returns the current task outcome, including a distinct non-cancellable result. | | `IsCancelRequestedAsync` | Lets a cooperative worker check the persisted cancellation request at a safe boundary. | @@ -126,7 +129,7 @@ Known progress updates do not cancel list reads. At the cache commit, the client The panel polls the current organization every three seconds while any task is active and once a minute while idle. Its fixed cadence changes only with the organization or active/idle state, so progress revisions cannot continually restart it. A tick skips hidden tabs and an already-running query; returning to an organization also refetches its list. Idle polling covers missed notifications, work started elsewhere, and ordinary queue retries after a failed attempt; it does not guess when a provider has exhausted retries. The idle cost is one task-list request per minute per visible tab. Notification delivery remains an optimization for latency. -Dismissal and completion handling follow the task's outcome revision. A newer terminal outcome can appear and refresh affected data even when the client missed its intervening `Running` state. Repeated copies of the same outcome do not repeat that refresh. +Dismissal and completion handling follow the task's outcome revision. A newer terminal outcome can appear and refresh affected data even when the client missed its intervening `Running` state. Repeated copies of the same outcome do not repeat that refresh. If a previously active task disappears from a later list, the panel also refreshes affected data once: the task may have finished outside the 50-task history window or its record may have expired. Omission alone does not establish its final status. The first successful task list also reconciles affected event and stack queries when it contains finished tasks. This covers completion during delayed initial discovery or missed notifications. Old finished history can cause one additional refresh on first discovery for that organization; later identical polls do not. Reconciliation supersedes reads started before the outcome, including uncached initial reads, and simultaneous completions share the replacement request. This guarantees a new query attempt, while normal Elasticsearch search visibility still applies. diff --git a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs index 2224e29333..8c86a76275 100644 --- a/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs +++ b/src/Exceptionless.Core/Services/BackgroundTasks/BackgroundTaskService.cs @@ -36,6 +36,7 @@ public class BackgroundTaskService /// Bounds task payloads loaded together, including selections attached to bulk tasks. internal const int ReadBatchSize = 50; + private static readonly TimeSpan UpdateLockLifetime = TimeSpan.FromSeconds(30); private readonly ICacheClient _cache; private readonly ILockProvider _lockProvider; private readonly IMessagePublisher _publisher; @@ -113,14 +114,13 @@ public async Task> GetByOrganizationAsync(string o var tasks = new List(); var finished = new List(); - var expired = new List(); string? cutoffSecond = null; // Read the entire cutoff second before choosing its newest tasks by CreatedUtc, including across cache batches. foreach (var batch in newestIds.Where(id => finished.Count < limit || activeIds.Contains(id) || (cutoffSecond is not null && id.StartsWith(cutoffSecond, StringComparison.Ordinal))).Chunk(ReadBatchSize)) { var found = await GetByIdsAsync(batch); - expired.AddRange(batch.Except(found.Select(task => task.Id), StringComparer.Ordinal)); + var expired = batch.Except(found.Select(task => task.Id), StringComparer.Ordinal).ToArray(); foreach (var task in found) { if (!String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal)) @@ -142,12 +142,9 @@ public async Task> GetByOrganizationAsync(string o finished.Sort(CompareNewestFirst); finished.RemoveRange(limit, finished.Count - limit); } - } - if (expired.Count > 0) - { - await _cache.ListRemoveAsync(GetOrganizationIndexCacheKey(organizationId), expired); - await RemoveFromActiveIndexAsync(organizationId, expired); + if (expired.Length > 0) + await PruneIndexesAsync(organizationId, expired); } tasks.AddRange(finished); @@ -163,10 +160,27 @@ public async Task> GetActiveIdsAsync(string organizationId return ids.HasValue ? ids.Value.Distinct(StringComparer.Ordinal).ToList() : []; } - /// Removes finished (or expired) ids from the active index of the organization. - public Task RemoveFromActiveIndexAsync(string organizationId, IReadOnlyCollection ids) + /// + /// Rechecks suspected stale ids under their update locks before removing index entries. Busy tasks are skipped for + /// a later read; active tasks and tasks owned by another organization are retained. Only missing tasks lose history. + /// + public async Task PruneIndexesAsync(string organizationId, IReadOnlyCollection ids) { - return _cache.ListRemoveAsync(GetActiveIndexCacheKey(organizationId), ids); + foreach (string id in ids.Distinct(StringComparer.Ordinal)) + { + await using var updateLock = await _lockProvider.TryAcquireAsync(GetUpdateLockResource(id), UpdateLockLifetime, TimeSpan.Zero); + if (updateLock is null) + continue; + + var task = await GetAsync(id); + if (task is not null && (task.IsActive || !String.Equals(task.OrganizationId, organizationId, StringComparison.Ordinal))) + continue; + + if (task is null) + await _cache.ListRemoveAsync(GetOrganizationIndexCacheKey(organizationId), [id]); + + await _cache.ListRemoveAsync(GetActiveIndexCacheKey(organizationId), [id]); + } } /// @@ -265,7 +279,7 @@ public Task ContinueAsync(T workItem) where T : class { BackgroundTask? task; await using (var updateLock = await _lockPolicy.ExecuteAsync(async ct => - await _lockProvider.AcquireAsync(GetUpdateLockResource(id), TimeSpan.FromSeconds(30), _options.LockAcquireTimeout), CancellationToken.None)) + await _lockProvider.AcquireAsync(GetUpdateLockResource(id), UpdateLockLifetime, _options.LockAcquireTimeout), CancellationToken.None)) { task = await GetAsync(id); if (task is null) @@ -292,7 +306,7 @@ await _lockProvider.AcquireAsync(GetUpdateLockResource(id), TimeSpan.FromSeconds // Renew history on completion, rather than rewriting the large history index on every progress report. // Keep active membership until history is saved so a concurrent list cannot miss a long-running task. await RefreshIndexAsync(GetOrganizationIndexCacheKey(task.OrganizationId), task.Id); - await RemoveFromActiveIndexAsync(task.OrganizationId, [task.Id]); + await _cache.ListRemoveAsync(GetActiveIndexCacheKey(task.OrganizationId), [task.Id]); } } diff --git a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs index 8911aa4344..283b96b3e8 100644 --- a/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs +++ b/src/Exceptionless.Core/Services/BulkOperations/BulkOperationService.cs @@ -230,7 +230,6 @@ public Task PublishLostAsync(BulkOperationWorkItem workItem) // Only active tasks can conflict, and every one of them is in the active index no matter how much history // the organization has. var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - var finished = new List(); BackgroundTask? conflict = null; var activeIds = await _taskService.GetActiveIdsAsync(candidate.OrganizationId); foreach (var batch in activeIds.Chunk(BackgroundTaskService.ReadBatchSize)) @@ -239,7 +238,7 @@ public Task PublishLostAsync(BulkOperationWorkItem workItem) return BulkOperationCreateResult.Busy(); var tasks = await _taskService.GetByIdsAsync(batch); - finished.AddRange(batch.Except(tasks.Where(task => task.IsActive).Select(task => task.Id), StringComparer.Ordinal)); + var finished = batch.Except(tasks.Where(task => task.IsActive).Select(task => task.Id), StringComparer.Ordinal).ToArray(); foreach (var active in tasks.Where(task => task.IsActive)) { var existing = active; @@ -274,10 +273,10 @@ public Task PublishLostAsync(BulkOperationWorkItem workItem) conflict ??= existing; } - } - if (finished.Count > 0) - await _taskService.RemoveFromActiveIndexAsync(candidate.OrganizationId, finished); + if (finished.Length > 0) + await _taskService.PruneIndexesAsync(candidate.OrganizationId, finished); + } return conflict is null ? null : BulkOperationCreateResult.Conflicting(conflict); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte index 936e025d9c..9dc96ec236 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte @@ -53,11 +53,14 @@ return; } + // An empty panel has no expiry timer, so newly discovered tasks need the current time too. + now = Date.now(); + // The task list drives this effect; remembering its revisions must not schedule it again. - const hasFinishedTasks = untrack(() => { + const shouldReconcile = untrack(() => { const previousTasks = observedTasksByOrganization.get(organizationId); const nextTasks = new SvelteMap>(); - let hasFinishedTasks = false; + let shouldReconcile = false; for (const task of tasks) { const previous = previousTasks?.get(task.id); const revision = previous ? compareTaskUpdateTimes(task.updated_utc, previous.updated_utc) : 1; @@ -77,20 +80,22 @@ } cancellingIds.delete(task.id); - hasFinishedTasks = true; + shouldReconcile = true; } // Bound remembered revisions and dismissals to each organization's latest task list. - for (const id of previousTasks?.keys() ?? []) { + for (const [id, previous] of previousTasks ?? []) { if (!nextTasks.has(id)) { dismissedUpdates.delete(id); + // A finished task can leave the limited history before its terminal update reaches us. + shouldReconcile ||= isTaskActive(previous.status); } } observedTasksByOrganization.set(organizationId, nextTasks); - return hasFinishedTasks; + return shouldReconcile; }); - if (hasFinishedTasks) { + if (shouldReconcile) { void invalidateTaskAffectedQueries(queryClient); } }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts index 4185da09af..5b729874f9 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/tasks/components/task-panel.svelte.test.ts @@ -192,6 +192,35 @@ describe('TaskPanel', () => { expect(document.querySelector('[data-slot="task-progress"]')?.textContent).toContain('Completed'); }); + it('reconciles an active task omitted from the finished history only once', async () => { + // Arrange + const finished = createTask({ id: 'known-finished', status: BackgroundTaskStatus.Completed, updated_utc: '2020-01-01T00:00:00Z' }); + state.set('tasks', [createTask(), finished]); + + // Act + render(TaskPanel); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + + // Act + // A completed task can fall outside the API's finished-history limit before its terminal update arrives. + state.set('tasks', [finished]); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); + + // Act + state.set('tasks', [{ ...finished }]); + await tick(); + + // Assert + expect(mocks.invalidateAffected).toHaveBeenCalledTimes(2); + }); + it('requests cancellation of a running task', async () => { // Arrange state.set('tasks', [createTask()]); @@ -364,6 +393,24 @@ describe('TaskPanel', () => { expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); }); + it('does not show expired outcomes discovered after the panel was idle', async () => { + // Arrange + vi.useFakeTimers(); + vi.setSystemTime('2026-01-01T10:00:00Z'); + state.set('tasks', []); + render(TaskPanel); + await tick(); + + // Act + await vi.advanceTimersByTimeAsync(10 * 60 * 1000); + state.set('tasks', [createTask({ status: BackgroundTaskStatus.Completed, updated_utc: '2026-01-01T10:05:00Z' })]); + await tick(); + + // Assert + expect(screen.queryByRole('region', { name: 'Background tasks' })).toBeNull(); + expect(mocks.invalidateAffected).toHaveBeenCalledOnce(); + }); + it('expires a finished task while another task keeps reporting progress', async () => { // Arrange vi.useFakeTimers(); diff --git a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs index 397c707123..77c5eb3606 100644 --- a/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BackgroundTasks/BackgroundTaskServiceTests.cs @@ -288,6 +288,70 @@ public async Task GetByOrganizationAsync_ExpiredAndForeignIds_AreSkippedAndExpir Assert.Contains(kept.Id, indexed.Value); } + [Theory] + [InlineData(OrganizationId, BackgroundTaskStatus.Running)] + [InlineData(OtherOrganizationId, BackgroundTaskStatus.Failed)] + public async Task PruneIndexesAsync_ActiveOrForeignTask_ReadsUnderLockWithoutRemovingIndexes(string taskOrganizationId, BackgroundTaskStatus status) + { + // Arrange + ObservingPruneCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingPruneCacheClient(inner)); + var task = NewTask(taskOrganizationId); + await harness.Tasks.AddAsync(task); + task.Status = status; + await harness.Cache.SetAsync(BackgroundTaskService.GetTaskCacheKey(task.Id), task, BackgroundTaskService.TaskLifetime); + await harness.Cache.ListAddAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId), [task.Id], BackgroundTaskService.TaskLifetime); + await harness.Cache.ListAddAsync(BackgroundTaskService.GetActiveIndexCacheKey(OrganizationId), [task.Id], BackgroundTaskService.TaskLifetime); + var readsUnderLock = new List(); + cache!.BeforeTaskRead = async () => + { + readsUnderLock.Add(await harness.LockProvider.IsLockedAsync(BackgroundTaskService.GetUpdateLockResource(task.Id))); + }; + + // Act + await harness.Tasks.PruneIndexesAsync(OrganizationId, [task.Id]); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var history = await harness.Cache.GetListAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId)); + + // Assert + Assert.Equal([true], readsUnderLock); + Assert.Equal(0, cache.IndexRemovals); + Assert.Contains(task.Id, active); + Assert.Contains(task.Id, history.Value); + } + + [Fact] + public async Task PruneIndexesAsync_TaskUpdateLockIsBusy_SkipsItUntilALaterRead() + { + // Arrange + using var harness = new BackgroundTaskTestHarness(_time, Log, + new BackgroundTaskOptions { LockAcquireTimeout = TimeSpan.Zero }, lockAttempts: 1); + var task = NewTask(); + await harness.Tasks.AddAsync(task); + await harness.Cache.RemoveAsync(BackgroundTaskService.GetTaskCacheKey(task.Id)); + await using (var held = await harness.LockProvider.AcquireAsync(BackgroundTaskService.GetUpdateLockResource(task.Id), TimeSpan.FromSeconds(30), TestCancellationToken)) + { + // Act + await harness.Tasks.PruneIndexesAsync(OrganizationId, [task.Id]).WaitAsync(TestCancellationToken); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var history = await harness.Cache.GetListAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId)); + + // Assert + Assert.Contains(task.Id, active); + Assert.Contains(task.Id, history.Value); + } + + // Act + await harness.Tasks.PruneIndexesAsync(OrganizationId, [task.Id]); + var remainingActive = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var remainingHistory = await harness.Cache.GetListAsync(BackgroundTaskService.GetOrganizationIndexCacheKey(OrganizationId)); + + // Assert + Assert.Empty(remainingActive); + Assert.False(remainingHistory.HasValue && remainingHistory.Value.Contains(task.Id)); + } + [Theory] [InlineData(0)] [InlineData(-1)] @@ -959,6 +1023,26 @@ private sealed class ObservingHistoryCacheClient(ICacheClient inner) : ScopedCac } } + private sealed class ObservingPruneCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient + { + public Func? BeforeTaskRead { get; set; } + public int IndexRemovals { get; private set; } + + async Task> ICacheClient.GetAsync(string key) + { + if (typeof(T) == typeof(BackgroundTask) && BeforeTaskRead is { } observe) + await observe(); + + return await UnscopedCache.GetAsync(key); + } + + Task ICacheClient.ListRemoveAsync(string key, IEnumerable values) + { + IndexRemovals++; + return UnscopedCache.ListRemoveAsync(key, values); + } + } + private sealed class RecordingReadCacheClient(ICacheClient inner) : ScopedCacheClient(inner, null), ICacheClient { public List ReadBatches { get; } = []; diff --git a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs index 1c3274e2e3..910feae9d9 100644 --- a/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/BulkOperations/BulkOperationServiceTests.cs @@ -126,6 +126,54 @@ await harness.Tasks.UpdateAsync(completing.Id, current => Assert.Equal(BackgroundTaskStatus.Completed, (await harness.Tasks.GetAsync(completing.Id))!.Status); } + [Fact] + public async Task CreateAsync_OrdinaryTaskRetriesAfterItsFailedSnapshot_KeepsItDiscoverable() + { + // Arrange + ObservingTaskReadsCacheClient? cache = null; + using var harness = new BackgroundTaskTestHarness(_time, Log, + decorateCache: inner => cache = new ObservingTaskReadsCacheClient(inner)); + var service = CreateService(harness); + var retrying = NewTask(BackgroundTaskTypes.ResetProjectData, ProjectId); + retrying.Bulk = null; + await harness.Tasks.AddAsync(retrying); + await harness.Tasks.UpdateAsync(retrying.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + return true; + }); + cache!.AfterRead = async () => + { + await harness.Tasks.UpdateAsync(retrying.Id, current => + { + current.Finish(BackgroundTaskStatus.Failed, "The attempt failed", _time.GetUtcNow().UtcDateTime); + return true; + }); + }; + cache.AfterBatchRead = async () => + { + cache.AfterBatchRead = null; + await harness.Tasks.UpdateAsync(retrying.Id, current => + { + current.Start(_time.GetUtcNow().UtcDateTime); + current.CompletedUtc = null; + return true; + }); + }; + + // Act + var created = await service.CreateAsync(NewTask("test.destructive", ProjectId)); + var stored = await harness.Tasks.GetAsync(retrying.Id); + var active = await harness.Tasks.GetActiveIdsAsync(OrganizationId); + var listed = await harness.Tasks.GetByOrganizationAsync(OrganizationId, limit: 0); + + // Assert + Assert.NotNull(created.Task); + Assert.Equal(BackgroundTaskStatus.Running, stored!.Status); + Assert.Contains(retrying.Id, active); + Assert.Contains(listed, task => task.Id == retrying.Id); + } + [Fact] public async Task CreateAsync_LongConflictScan_KeepsTheAdmissionLeaseUntilTheTaskIsIndexed() {