diff --git a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs index 6a869d0425..c2c1ab1132 100644 --- a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs +++ b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs @@ -108,7 +108,7 @@ public static void SetEventReference(this PersistentEvent ev, string name, strin public static string? GetSessionId(this PersistentEvent ev) { - return ev.IsSessionStart() ? ev.ReferenceId : ev.GetEventReference("session"); + return ev.IsSessionStart() ? ev.ReferenceId : ev.GetEventReference(Event.KnownReferenceNames.Session); } public static void SetSessionId(this PersistentEvent ev, string sessionId) @@ -119,7 +119,7 @@ public static void SetSessionId(this PersistentEvent ev, string sessionId) if (ev.IsSessionStart()) ev.ReferenceId = sessionId; else - ev.SetEventReference("session", sessionId); + ev.SetEventReference(Event.KnownReferenceNames.Session, sessionId); } public static bool HasSessionEndTime(this PersistentEvent ev) diff --git a/src/Exceptionless.Core/Extensions/StringExtensions.cs b/src/Exceptionless.Core/Extensions/StringExtensions.cs index e974c209f4..b80cbfd1f2 100644 --- a/src/Exceptionless.Core/Extensions/StringExtensions.cs +++ b/src/Exceptionless.Core/Extensions/StringExtensions.cs @@ -131,7 +131,7 @@ public static bool IsNumeric(this string? value) public static bool IsValidFieldName(this string? value) { - if (value is null || value.Length > 25) + if (String.IsNullOrEmpty(value) || value.Length > 25) return false; return IsValidIdentifier(value); diff --git a/src/Exceptionless.Core/Migrations/003_BackfillParentReferences.cs b/src/Exceptionless.Core/Migrations/003_BackfillParentReferences.cs new file mode 100644 index 0000000000..d8e5db59b1 --- /dev/null +++ b/src/Exceptionless.Core/Migrations/003_BackfillParentReferences.cs @@ -0,0 +1,113 @@ +using System.Diagnostics; +using System.Text.Json; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Tasks; +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories.Configuration; +using Foundatio.Repositories.Elasticsearch.Extensions; +using Foundatio.Repositories.Migrations; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Migrations; + +public sealed class BackfillParentReferences : MigrationBase +{ + private readonly ElasticsearchClient _client; + private readonly ExceptionlessElasticConfiguration _config; + private readonly TimeProvider _timeProvider; + + public BackfillParentReferences(ExceptionlessElasticConfiguration configuration, TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(loggerFactory) + { + _config = configuration; + _client = configuration.Client; + _timeProvider = timeProvider; + + MigrationType = MigrationType.VersionedAndResumable; + Version = 3; + } + + public override async Task RunAsync(MigrationContext context) + { + string referenceKey = $"@ref:{Event.KnownReferenceNames.Parent}"; + string indexKey = $"{Event.KnownReferenceNames.Parent}-r"; + string script = $$""" + def parentReference = null; + if (ctx._source.data != null) { + parentReference = ctx._source.data['{{referenceKey}}']; + if (parentReference == null) { + for (def entry : ctx._source.data.entrySet()) { + if (entry.getKey().equalsIgnoreCase('{{referenceKey}}')) { + parentReference = entry.getValue(); + break; + } + } + } + } + if (parentReference != null) { + if (ctx._source.idx == null) ctx._source.idx = [:]; + ctx._source.idx['{{indexKey}}'] = parentReference.toString(); + } else { + ctx.op = 'noop'; + } + """; + + _logger.LogInformation("Backfilling retained event parent references"); + var stopwatch = Stopwatch.StartNew(); + var response = await _client.UpdateByQueryAsync(request => request + .Indices($"{_config.Events.VersionedName}-*") + .Query(query => query.Bool(filter => filter.MustNot(mustNot => mustNot.Exists(exists => exists.Field($"idx.{indexKey}"))))) + .Script(value => value.Source(script).Lang(ScriptLanguage.Painless)) + .WaitForCompletion(false)); + _logger.LogRequest(response, LogLevel.Information); + + if (!response.IsValidResponse || response.Task is null) + throw new ApplicationException($"Unable to start parent-reference backfill: {response.DebugInformation}"); + + int attempts = 0; + while (!context.CancellationToken.IsCancellationRequested) + { + var taskStatus = await _client.Tasks.GetAsync(response.Task.FullyQualifiedId, context.CancellationToken); + if (!taskStatus.IsValidResponse) + throw new ApplicationException($"Unable to monitor parent-reference backfill: {taskStatus.DebugInformation}"); + + if (taskStatus.Completed) + { + EnsureTaskSucceeded(taskStatus); + _logger.LogInformation("Finished parent-reference backfill: Duration={Duration}", stopwatch.Elapsed); + return; + } + + attempts++; + await context.Lock.RenewAsync(); + await Task.Delay(TimeSpan.FromSeconds(attempts <= 5 ? 1 : 5), _timeProvider, context.CancellationToken); + } + + context.CancellationToken.ThrowIfCancellationRequested(); + } + + internal static void EnsureTaskSucceeded(GetTasksResponse taskStatus) + { + if (taskStatus.Error is not null) + throw new ApplicationException($"Parent-reference backfill failed: {JsonSerializer.Serialize(taskStatus.Error)}"); + + if (taskStatus.Response is null) + return; + + JsonElement response = JsonSerializer.SerializeToElement(taskStatus.Response); + if (response.ValueKind == JsonValueKind.Object + && response.TryGetProperty("version_conflicts", out var versionConflicts) + && versionConflicts.TryGetInt64(out long conflictCount) + && conflictCount > 0) + { + throw new ApplicationException($"Parent-reference backfill failed with {conflictCount} version conflicts."); + } + + if (response.ValueKind == JsonValueKind.Object + && response.TryGetProperty("failures", out var failures) + && failures.ValueKind == JsonValueKind.Array + && failures.GetArrayLength() > 0) + { + throw new ApplicationException($"Parent-reference backfill failed: {failures}"); + } + } +} diff --git a/src/Exceptionless.Core/Models/Event.cs b/src/Exceptionless.Core/Models/Event.cs index 84a96c7acd..429aa4ff45 100644 --- a/src/Exceptionless.Core/Models/Event.cs +++ b/src/Exceptionless.Core/Models/Event.cs @@ -159,6 +159,12 @@ public static class KnownTags public const string Internal = "Internal"; } + public static class KnownReferenceNames + { + public const string Parent = "parent"; + public const string Session = "session"; + } + public static class KnownDataKeys { public const string Error = "@error"; diff --git a/src/Exceptionless.Core/Pipeline/035_CopySimpleDataToIdxAction.cs b/src/Exceptionless.Core/Pipeline/035_CopySimpleDataToIdxAction.cs index b85723efa2..c4588fe5e4 100644 --- a/src/Exceptionless.Core/Pipeline/035_CopySimpleDataToIdxAction.cs +++ b/src/Exceptionless.Core/Pipeline/035_CopySimpleDataToIdxAction.cs @@ -1,4 +1,5 @@ -using Exceptionless.Core.Plugins.EventProcessor; +using Exceptionless.Core.Models; +using Exceptionless.Core.Plugins.EventProcessor; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Pipeline; @@ -11,7 +12,12 @@ public CopySimpleDataToIdxAction(AppOptions options, ILoggerFactory loggerFactor public override Task ProcessAsync(EventContext ctx) { if (!ctx.Organization.HasPremiumFeatures) + { + if (ctx.Event.GetEventReference(Event.KnownReferenceNames.Parent) is not null) + ctx.Event.CopyDataToIndex([$"@ref:{Event.KnownReferenceNames.Parent}"]); + return Task.CompletedTask; + } // TODO: Do we need a pipeline action to trim keys and remove null values that may be sent by other native clients. ctx.Event.CopyDataToIndex([]); diff --git a/src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs b/src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs index 5ba177186a..d80784baa0 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Validation/PersistentEventQueryValidator.cs @@ -1,4 +1,5 @@ -using Exceptionless.Core.Repositories.Configuration; +using Exceptionless.Core.Models; +using Exceptionless.Core.Repositories.Configuration; using Foundatio.Parsers.LuceneQueries; using Foundatio.Parsers.LuceneQueries.Visitors; using Microsoft.Extensions.Logging; @@ -11,6 +12,7 @@ public sealed class PersistentEventQueryValidator : AppQueryValidator "date", "type", EventIndex.Alias.ReferenceId, + $"idx.{Event.KnownReferenceNames.Parent}-r", "reference_id", EventIndex.Alias.OrganizationId, "organization_id", @@ -93,9 +95,11 @@ public PersistentEventQueryValidator(ExceptionlessElasticConfiguration configura protected override QueryProcessResult ApplyQueryRules(QueryValidationResult result) { + bool hasInvalidReferenceField = result.ReferencedFields.Any(field => field.StartsWith("ref.", StringComparison.OrdinalIgnoreCase)); return new QueryProcessResult { - IsValid = result.IsValid, + IsValid = result.IsValid && !hasInvalidReferenceField, + Message = hasInvalidReferenceField ? "Invalid reference field name" : null, UsesPremiumFeatures = !result.ReferencedFields.All(_freeQueryFields.Contains) }; } diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs index 43065cc189..5bc4774572 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/EventFieldsQueryVisitor.cs @@ -73,7 +73,7 @@ public override Task VisitAsync(MissingNode node, IQueryVisitorContext context) return null; string[] parts = field.Split('.'); - if (parts.Length != 2 || (parts.Length == 2 && parts[1].StartsWith("@"))) + if (parts.Length != 2 || parts[1].StartsWith("@") || !parts[1].IsValidFieldName()) return field; if (String.Equals(parts[0], "data", StringComparison.OrdinalIgnoreCase)) diff --git a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs index 6238b95681..2563f38606 100644 --- a/src/Exceptionless.Web/Api/Handlers/EventHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/EventHandler.cs @@ -16,19 +16,19 @@ using Exceptionless.Core.Validation; using Exceptionless.DateTimeExtensions; using Exceptionless.Web.Api.Infrastructure; -using Exceptionless.Web.Api.Results; using Exceptionless.Web.Api.Messages; +using Exceptionless.Web.Api.Results; using Exceptionless.Web.Extensions; using Exceptionless.Web.Models; using Exceptionless.Web.Utility; using Foundatio.Caching; using Foundatio.Mediator; using Foundatio.Queues; -using Foundatio.Serializer; using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using Microsoft.Net.Http.Headers; namespace Exceptionless.Web.Api.Handlers; @@ -230,7 +230,7 @@ public async Task>> Handle(GetEventsByReferenceId mes var ti = TimeRangeParser.GetTimeInfo(null, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organizations.GetRetentionUtcCutoff(appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(organizations) { IsUserOrganizationsFilter = true }; - return await GetInternalAsync(sf, ti, httpContext, String.Concat("reference:", message.ReferenceId), null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.ReferenceId} OR ref.{Event.KnownReferenceNames.Parent}:{message.ReferenceId})", null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); } public async Task>> Handle(GetEventsByReferenceIdAndProject message) @@ -249,7 +249,7 @@ public async Task>> Handle(GetEventsByReferenceIdAndP var ti = TimeRangeParser.GetTimeInfo(null, message.Offset, timeProvider, _allowedDateFields, DefaultDateField, organization.GetRetentionUtcCutoff(project, appOptions.MaximumRetentionDays, timeProvider)); var sf = new AppFilter(project, organization); - return await GetInternalAsync(sf, ti, httpContext, String.Concat("reference:", message.ReferenceId), null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); + return await GetInternalAsync(sf, ti, httpContext, $"(reference:{message.ReferenceId} OR ref.{Event.KnownReferenceNames.Parent}:{message.ReferenceId})", null, message.Mode, message.Page, message.Limit, message.Before, message.After, includeTotal: ShouldIncludeTotal(message.Include)); } public async Task>> Handle(GetEventsBySessionId message) diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts index fd54c1e078..35a64bd214 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/api.svelte.ts @@ -4,7 +4,7 @@ import type { CountResult, WorkInProgressResult } from '$shared/models'; import { accessToken } from '$features/auth/index.svelte'; import { queryKeys as stackQueryKeys } from '$features/stacks/api.svelte'; import { DEFAULT_OFFSET } from '$shared/api/api.svelte'; -import { type ProblemDetails, useFetchClient } from '@exceptionless/fetchclient'; +import { type FetchClientResponse, type ProblemDetails, useFetchClient } from '@exceptionless/fetchclient'; import { createMutation, createQuery, keepPreviousData, QueryClient, useQueryClient } from '@tanstack/svelte-query'; import type { EventSummaryModel, SummaryTemplateKeys } from './components/summary/index'; @@ -240,7 +240,7 @@ export function getEventQuery(request: GetEventRequest) { } export function getEventsByReferenceQuery(request: GetEventsByReferenceRequest) { - return createQuery[], ProblemDetails>(() => ({ + return createQuery[]>, ProblemDetails>(() => ({ enabled: () => !!accessToken.current && !!request.route.referenceId, queryFn: async ({ signal }: { signal: AbortSignal }) => { const client = useFetchClient(); @@ -253,12 +253,13 @@ export function getEventsByReferenceQuery(request: GetEventsByReferenceRequest) limit: 20, mode: 'summary', page: 1, - ...request.params + ...request.params, + include: 'total' }, signal }); - return response.data!; + return response; }, queryKey: queryKeys.eventsByReference(request.route.referenceId, request.route.projectId, request.params) })); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.test.ts index 7d63ed90aa..8a9ed838c7 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/helpers.svelte.test.ts @@ -86,6 +86,16 @@ describe('TagFilter', () => { }); }); +describe('ReferenceFilter', () => { + it('matches direct and parent references', () => { + expect(new ReferenceFilter('ref-123').toFilter()).toBe('(reference:"ref-123" OR ref.parent:"ref-123")'); + }); + + it('quotes the reference in both clauses', () => { + expect(new ReferenceFilter('ref 123').toFilter()).toBe('(reference:"ref 123" OR ref.parent:"ref 123")'); + }); +}); + describe('applyTimeFilter', () => { it('removes an existing date filter when time is explicitly empty', () => { // Arrange diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts index ce9d78ac2b..6c15226e9f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/filters/models.svelte.ts @@ -245,7 +245,8 @@ export class ReferenceFilter implements IFilter { return ''; } - return `reference:${quoteIfSpecialCharacters(this.value)}`; + const reference = quoteIfSpecialCharacters(this.value); + return `(reference:${reference} OR ref.parent:${reference})`; } } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte index 17ca4ddd60..a8e9575d0c 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/components/views/overview.svelte @@ -52,6 +52,10 @@ return refs; }); + function isValidReferenceName(name: string): boolean { + return /^[\p{L}\p{Nd}-]{1,25}$/u.test(name); + } + let level = $derived(event.data?.['@level']?.toLowerCase()); let location = $derived(getLocation(event)); @@ -95,9 +99,17 @@ {#if reference.name === 'session'} Session - {:else} + {:else if reference.name === 'parent'} {reference.name} + {:else if isValidReferenceName(reference.name)} + {reference.name} + + {:else} + {reference.name} + {/if} {reference.id} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.test.ts new file mode 100644 index 0000000000..210883469f --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; + +import { filterUsesPremiumFeatures } from './premium-filter'; + +describe('filterUsesPremiumFeatures', () => { + it('keeps built-in parent reference navigation available without premium features', () => { + expect(filterUsesPremiumFeatures('(reference:"parent-id" OR ref.parent:"parent-id")')).toBe(false); + }); + + it('still requires premium features for custom references', () => { + expect(filterUsesPremiumFeatures('ref.custom:"reference-id"')).toBe(true); + }); + + it.each(['ref.order-id:"reference-id"', 'ref.订单-1:"reference-id"'])('recognizes backend-valid custom reference field %s', (filter) => { + expect(filterUsesPremiumFeatures(filter)).toBe(true); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.ts index ccb9fbd3a4..2b5f35b345 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/events/premium-filter.ts @@ -9,6 +9,7 @@ const FREE_QUERY_FIELDS = new Set([ 'organization_id', 'project', 'project_id', + 'ref.parent', 'reference', 'reference_id', 'stack', @@ -35,7 +36,7 @@ export function filterUsesPremiumFeatures(filter: null | string | undefined): bo * Matches patterns like `field:value` or `field:(value1 OR value2)`. */ function extractFilterFields(filter: string): string[] { - const fieldPattern = /(?:^|\s|[(!])(\w[\w.]*):/g; + const fieldPattern = /(?:^|\s|[(!])([^\s:()]+):/g; const fields: string[] = []; let match: null | RegExpExecArray; diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/by-ref/[referenceId]/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/by-ref/[referenceId]/+page.svelte index 8225be7f1a..48093a0230 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/by-ref/[referenceId]/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/event/by-ref/[referenceId]/+page.svelte @@ -1,3 +1,9 @@ + +