From af21cb2dcb3b5739473cbfb211f28478cca639a1 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 6 Mar 2025 13:01:11 -0800 Subject: [PATCH 01/60] added distributed tracing for entities --- .../Command/CreateSubOrchestrationAction.cs | 21 ++++++++++++ src/DurableTask.Core/OrchestrationContext.cs | 18 ++++++++++ .../TaskOrchestrationContext.cs | 34 +++++++++++++++---- .../TaskOrchestrationDispatcher.cs | 16 ++++++--- src/DurableTask.Core/Tracing/TraceHelper.cs | 15 ++++++++ 5 files changed, 93 insertions(+), 11 deletions(-) diff --git a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs index 67a9e1013..388f73c84 100644 --- a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs +++ b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs @@ -14,6 +14,7 @@ namespace DurableTask.Core.Command { using System.Collections.Generic; + using System.Diagnostics; /// /// Orchestrator action for creating sub-orchestrations. @@ -50,5 +51,25 @@ public class CreateSubOrchestrationAction : OrchestratorAction /// Tags to be applied to the sub-orchestration. /// public IDictionary? Tags { get; set; } + + /// + /// Parent trace ID of the sub-orchestration. + /// + public string? ParentTraceId { get; set; } + + /// + /// Parent span ID of the sub-orchestration. + /// + public string? ParentSpanId { get; set; } + + /// + /// Parent trace flags of the sub-orchestration. + /// + public ActivityTraceFlags ParentTraceFlags { get; set; } + + /// + /// Parent trace state of the sub-orchestration. + /// + public string? ParentTraceState { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 4b2d8c7fe..5cbd22b3a 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -15,6 +15,7 @@ namespace DurableTask.Core { using System; using System.Collections.Generic; + using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Castle.DynamicProxy; @@ -377,6 +378,23 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, object input, IDictionary tags); + /// + /// Create a sub-orchestration of the specified name and version with the specific instance id + /// + /// Return Type of the TaskOrchestration.RunTask method + /// Name of the orchestration as specified by the ObjectCreator + /// Name of the orchestration as specified by the ObjectCreator + /// InstanceId of the sub-orchestration to create + /// Input for the TaskOrchestration.RunTask method + /// Dictionary of key/value tags associated with this instance + /// Trace ID of the parent instance creating this suborchestration + /// Span ID of the parent instance creating this suborchestration + /// Trace flags of the parent instance creating this suborchestration + /// Trace state of the parent instance creating this suborchestration + /// Task that represents the execution of the specified sub-orchestration + public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, + object input, IDictionary tags, string parentTraceId, string parentSpanId, ActivityTraceFlags parentTraceFlags, string parentTraceState); + /// /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 8a48cbe93..0e4083799 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -129,7 +129,7 @@ public override Task CreateSubOrchestrationInstance( string instanceId, object input) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null, null, null, ActivityTraceFlags.None, null); } public override Task CreateSubOrchestrationInstance( @@ -139,7 +139,7 @@ public override Task CreateSubOrchestrationInstance( object input, IDictionary tags) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, null, null, ActivityTraceFlags.None, null); } public override Task CreateSubOrchestrationInstance( @@ -147,7 +147,21 @@ public override Task CreateSubOrchestrationInstance( string version, object input) { - return CreateSubOrchestrationInstanceCore(name, version, null, input, null); + return CreateSubOrchestrationInstanceCore(name, version, null, input, null, null, null, ActivityTraceFlags.None, null); + } + + public override Task CreateSubOrchestrationInstance( + string name, + string version, + string instanceId, + object input, + IDictionary tags, + string parentTraceId, + string parentSpanId, + ActivityTraceFlags parentTraceFlags, + string parentTraceState) + { + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, parentTraceId, parentSpanId, parentTraceFlags, parentTraceState); } async Task CreateSubOrchestrationInstanceCore( @@ -155,7 +169,11 @@ async Task CreateSubOrchestrationInstanceCore( string version, string instanceId, object input, - IDictionary tags) + IDictionary tags, + string parentTraceId, + string parentSpanId, + ActivityTraceFlags parentTraceFlags, + string parentTraceState) { int id = this.idCounter++; string serializedInput = this.MessageDataConverter.SerializeInternal(input); @@ -173,9 +191,13 @@ async Task CreateSubOrchestrationInstanceCore( Name = name, Version = version, Input = serializedInput, - Tags = tags + Tags = tags, + ParentTraceId = parentTraceId, + ParentSpanId = parentSpanId, + ParentTraceFlags = parentTraceFlags, + ParentTraceState = parentTraceState }; - + this.orchestratorActionsMap.Add(id, action); if (OrchestrationTags.IsTaggedAsFireAndForget(tags)) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 61864a1a5..6892f8a56 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -430,7 +430,10 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work createSubOrchestrationAction, runtimeState, this.IncludeParameters, - traceActivity)); + createSubOrchestrationAction.ParentTraceId ?? traceActivity?.TraceId.ToString(), + createSubOrchestrationAction.ParentSpanId ?? traceActivity?.SpanId.ToString(), + createSubOrchestrationAction.ParentTraceId != null ? createSubOrchestrationAction.ParentTraceFlags : traceActivity?.ActivityTraceFlags, + createSubOrchestrationAction.ParentTraceState ?? traceActivity?.TraceStateString)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1083,7 +1086,10 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( CreateSubOrchestrationAction createSubOrchestrationAction, OrchestrationRuntimeState runtimeState, bool includeParameters, - Activity? parentTraceActivity) + string? parentTraceId, + string? parentSpanId, + ActivityTraceFlags? parentTraceFlags, + string? parentTraceState) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1096,7 +1102,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); + ActivitySpanId clientSpanId = parentSpanId != null ? ActivitySpanId.CreateFromString(parentSpanId.AsSpan()) : ActivitySpanId.CreateRandom(); historyEvent.ClientSpanId = clientSpanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1122,9 +1128,9 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - if (parentTraceActivity != null) + if (parentTraceId != null) { - ActivityContext activityContext = new ActivityContext(parentTraceActivity.TraceId, clientSpanId, parentTraceActivity.ActivityTraceFlags, parentTraceActivity.TraceStateString); + ActivityContext activityContext = new ActivityContext(ActivityTraceId.CreateFromString(parentTraceId.AsSpan()), clientSpanId, (ActivityTraceFlags)parentTraceFlags!, parentTraceState); startedEvent.SetParentTraceContext(activityContext); } diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 37f09c9d5..0e64ac001 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -357,6 +357,14 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { + + // If either the target or source of this event is an entity, we don't want to start a trace activity for it because entity tracing + // is handled separately in the WebJobs extension + if ((targetInstanceId != null && targetInstanceId.StartsWith("@")) || (instance != null && instance.InstanceId.StartsWith("@"))) + { + return null; + } + Activity? newActivity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaisedEvent.Name, null), kind: ActivityKind.Producer, @@ -390,6 +398,13 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { + // If either the target of this event is an entity, we don't want to start a trace activity for it because entity tracing + // is handled separately in the WebJobs extension + if (instance.InstanceId.StartsWith("@")) + { + return null; + } + Activity? newActivity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaised.Name, null), kind: ActivityKind.Producer, From 8846d7046b4bd2c6a49a023610ecd757049fb402 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 10 Mar 2025 13:10:16 -0700 Subject: [PATCH 02/60] change in how the distributed trace context is propagated in the case that an entity in WebJobs starts an orchestration --- .../Command/CreateSubOrchestrationAction.cs | 20 ++----------- src/DurableTask.Core/OrchestrationContext.cs | 8 ++--- .../TaskOrchestrationContext.cs | 23 +++++--------- .../TaskOrchestrationDispatcher.cs | 30 +++++++++---------- .../Tracing/DistributedTraceContext.cs | 12 ++++++++ 5 files changed, 40 insertions(+), 53 deletions(-) diff --git a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs index 388f73c84..fd118ece0 100644 --- a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs +++ b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs @@ -13,6 +13,7 @@ #nullable enable namespace DurableTask.Core.Command { + using DurableTask.Core.Tracing; using System.Collections.Generic; using System.Diagnostics; @@ -53,23 +54,8 @@ public class CreateSubOrchestrationAction : OrchestratorAction public IDictionary? Tags { get; set; } /// - /// Parent trace ID of the sub-orchestration. + /// Parent trace of the sub-orchestration. /// - public string? ParentTraceId { get; set; } - - /// - /// Parent span ID of the sub-orchestration. - /// - public string? ParentSpanId { get; set; } - - /// - /// Parent trace flags of the sub-orchestration. - /// - public ActivityTraceFlags ParentTraceFlags { get; set; } - - /// - /// Parent trace state of the sub-orchestration. - /// - public string? ParentTraceState { get; set; } + public DistributedTraceContext? ParentTraceContext { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 5cbd22b3a..7de1dcb58 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -21,6 +21,7 @@ namespace DurableTask.Core using Castle.DynamicProxy; using DurableTask.Core.Entities; using DurableTask.Core.Serializing; + using DurableTask.Core.Tracing; /// /// Context for an orchestration containing the instance, replay status, orchestration methods and proxy methods @@ -387,13 +388,10 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// InstanceId of the sub-orchestration to create /// Input for the TaskOrchestration.RunTask method /// Dictionary of key/value tags associated with this instance - /// Trace ID of the parent instance creating this suborchestration - /// Span ID of the parent instance creating this suborchestration - /// Trace flags of the parent instance creating this suborchestration - /// Trace state of the parent instance creating this suborchestration + /// Trace context of the parent instance creating this suborchestration /// Task that represents the execution of the specified sub-orchestration public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, - object input, IDictionary tags, string parentTraceId, string parentSpanId, ActivityTraceFlags parentTraceFlags, string parentTraceState); + object input, IDictionary tags, DistributedTraceContext parentTraceContext); /// diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 0e4083799..73cbda09c 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -129,7 +129,7 @@ public override Task CreateSubOrchestrationInstance( string instanceId, object input) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null, null, null, ActivityTraceFlags.None, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null, null); } public override Task CreateSubOrchestrationInstance( @@ -139,7 +139,7 @@ public override Task CreateSubOrchestrationInstance( object input, IDictionary tags) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, null, null, ActivityTraceFlags.None, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, null); } public override Task CreateSubOrchestrationInstance( @@ -147,7 +147,7 @@ public override Task CreateSubOrchestrationInstance( string version, object input) { - return CreateSubOrchestrationInstanceCore(name, version, null, input, null, null, null, ActivityTraceFlags.None, null); + return CreateSubOrchestrationInstanceCore(name, version, null, input, null, null); } public override Task CreateSubOrchestrationInstance( @@ -156,12 +156,9 @@ public override Task CreateSubOrchestrationInstance( string instanceId, object input, IDictionary tags, - string parentTraceId, - string parentSpanId, - ActivityTraceFlags parentTraceFlags, - string parentTraceState) + DistributedTraceContext parentTraceContext) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, parentTraceId, parentSpanId, parentTraceFlags, parentTraceState); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, parentTraceContext); } async Task CreateSubOrchestrationInstanceCore( @@ -170,10 +167,7 @@ async Task CreateSubOrchestrationInstanceCore( string instanceId, object input, IDictionary tags, - string parentTraceId, - string parentSpanId, - ActivityTraceFlags parentTraceFlags, - string parentTraceState) + DistributedTraceContext parentTraceContext) { int id = this.idCounter++; string serializedInput = this.MessageDataConverter.SerializeInternal(input); @@ -192,10 +186,7 @@ async Task CreateSubOrchestrationInstanceCore( Version = version, Input = serializedInput, Tags = tags, - ParentTraceId = parentTraceId, - ParentSpanId = parentSpanId, - ParentTraceFlags = parentTraceFlags, - ParentTraceState = parentTraceState + ParentTraceContext = parentTraceContext }; this.orchestratorActionsMap.Add(id, action); diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 6892f8a56..fd28c6d66 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -425,15 +425,18 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; + DistributedTraceContext? traceContext = null; + if (traceActivity != null) + { + traceContext = new DistributedTraceContext(traceActivity.TraceId.ToString(), traceActivity.TraceStateString); + traceContext.ParentTraceFlags = traceActivity.ActivityTraceFlags; + } orchestratorMessages.Add( - this.ProcessCreateSubOrchestrationInstanceDecision( - createSubOrchestrationAction, - runtimeState, - this.IncludeParameters, - createSubOrchestrationAction.ParentTraceId ?? traceActivity?.TraceId.ToString(), - createSubOrchestrationAction.ParentSpanId ?? traceActivity?.SpanId.ToString(), - createSubOrchestrationAction.ParentTraceId != null ? createSubOrchestrationAction.ParentTraceFlags : traceActivity?.ActivityTraceFlags, - createSubOrchestrationAction.ParentTraceState ?? traceActivity?.TraceStateString)); + this.ProcessCreateSubOrchestrationInstanceDecision( + createSubOrchestrationAction, + runtimeState, + this.IncludeParameters, + createSubOrchestrationAction.ParentTraceContext ?? traceContext)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1086,10 +1089,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( CreateSubOrchestrationAction createSubOrchestrationAction, OrchestrationRuntimeState runtimeState, bool includeParameters, - string? parentTraceId, - string? parentSpanId, - ActivityTraceFlags? parentTraceFlags, - string? parentTraceState) + DistributedTraceContext? parentTraceContext) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1102,7 +1102,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = parentSpanId != null ? ActivitySpanId.CreateFromString(parentSpanId.AsSpan()) : ActivitySpanId.CreateRandom(); + ActivitySpanId clientSpanId = parentTraceContext?.ParentSpanId != null ? ActivitySpanId.CreateFromString(parentTraceContext.ParentSpanId.AsSpan()) : ActivitySpanId.CreateRandom(); historyEvent.ClientSpanId = clientSpanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1128,9 +1128,9 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - if (parentTraceId != null) + if (parentTraceContext != null) { - ActivityContext activityContext = new ActivityContext(ActivityTraceId.CreateFromString(parentTraceId.AsSpan()), clientSpanId, (ActivityTraceFlags)parentTraceFlags!, parentTraceState); + ActivityContext activityContext = new ActivityContext(ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), clientSpanId, (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, parentTraceContext.TraceState); startedEvent.SetParentTraceContext(activityContext); } diff --git a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs index b69b1f5f6..39ce0c88e 100644 --- a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs +++ b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs @@ -14,6 +14,7 @@ namespace DurableTask.Core.Tracing { using System; + using System.Diagnostics; using System.Runtime.Serialization; /// @@ -80,5 +81,16 @@ public string? TraceState /// [DataMember] public DateTimeOffset? ActivityStartTime { get; set; } + + /// + /// The span ID of the parent trace + /// + [DataMember] + public string? ParentSpanId { get; set; } + + /// + /// The activity trace flags of the parent trace + /// + public ActivityTraceFlags? ParentTraceFlags { get; set; } } } From d3f25f3a8d4d1e9cae7348494c4d259e6f3890eb Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 11 Mar 2025 00:20:16 -0700 Subject: [PATCH 03/60] slight style update --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index fd28c6d66..302fb4df3 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1130,7 +1130,10 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( if (parentTraceContext != null) { - ActivityContext activityContext = new ActivityContext(ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), clientSpanId, (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, parentTraceContext.TraceState); + ActivityContext activityContext = new ActivityContext( + ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), + clientSpanId, (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, + parentTraceContext.TraceState); startedEvent.SetParentTraceContext(activityContext); } From ed72bf70edf46f9935d351c0091c921ea9223b86 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 11 Mar 2025 19:37:25 -0700 Subject: [PATCH 04/60] some more tiny style things --- src/DurableTask.Core/TaskOrchestrationContext.cs | 2 +- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 73cbda09c..656d852bd 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -188,7 +188,7 @@ async Task CreateSubOrchestrationInstanceCore( Tags = tags, ParentTraceContext = parentTraceContext }; - + this.orchestratorActionsMap.Add(id, action); if (OrchestrationTags.IsTaggedAsFireAndForget(tags)) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 302fb4df3..119f7f115 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1132,7 +1132,8 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( { ActivityContext activityContext = new ActivityContext( ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), - clientSpanId, (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, + clientSpanId, + (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, parentTraceContext.TraceState); startedEvent.SetParentTraceContext(activityContext); } From 1c271de54aed7ba589f863d961e90b44bd1c997b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 12 Mar 2025 12:24:19 -0700 Subject: [PATCH 05/60] addressing two more PR comments --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 8 ++++---- src/DurableTask.Core/Tracing/TraceHelper.cs | 6 ++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 119f7f115..eb506dc87 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -432,10 +432,10 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work traceContext.ParentTraceFlags = traceActivity.ActivityTraceFlags; } orchestratorMessages.Add( - this.ProcessCreateSubOrchestrationInstanceDecision( - createSubOrchestrationAction, - runtimeState, - this.IncludeParameters, + this.ProcessCreateSubOrchestrationInstanceDecision( + createSubOrchestrationAction, + runtimeState, + this.IncludeParameters, createSubOrchestrationAction.ParentTraceContext ?? traceContext)); break; case OrchestratorActionType.SendEvent: diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 0e64ac001..10231b7df 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -358,8 +358,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( string? targetInstanceId) { - // If either the target or source of this event is an entity, we don't want to start a trace activity for it because entity tracing - // is handled separately in the WebJobs extension + // We don't want to emit tracing for external events when they are related to entities if ((targetInstanceId != null && targetInstanceId.StartsWith("@")) || (instance != null && instance.InstanceId.StartsWith("@"))) { return null; @@ -398,8 +397,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - // If either the target of this event is an entity, we don't want to start a trace activity for it because entity tracing - // is handled separately in the WebJobs extension + // We don't want to emit tracing for external events when they are related to entities if (instance.InstanceId.StartsWith("@")) { return null; From ffe061ee6b4c67b47183c3bac71ed3e450985789 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 12 Mar 2025 17:44:28 -0700 Subject: [PATCH 06/60] forgot to add one annotation --- src/DurableTask.Core/Tracing/DistributedTraceContext.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs index 39ce0c88e..15062c919 100644 --- a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs +++ b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs @@ -91,6 +91,7 @@ public string? TraceState /// /// The activity trace flags of the parent trace /// + [DataMember] public ActivityTraceFlags? ParentTraceFlags { get; set; } } } From 7462c40c4ee821b98b6f1cc5d24d975e6d26b79f Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 13 Mar 2025 12:45:34 -0700 Subject: [PATCH 07/60] yet another small style change --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index eb506dc87..2634275a8 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -428,8 +428,12 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work DistributedTraceContext? traceContext = null; if (traceActivity != null) { - traceContext = new DistributedTraceContext(traceActivity.TraceId.ToString(), traceActivity.TraceStateString); - traceContext.ParentTraceFlags = traceActivity.ActivityTraceFlags; + traceContext = new DistributedTraceContext( + traceActivity.TraceId.ToString(), + traceActivity.TraceStateString) + { + ParentTraceFlags = traceActivity.ActivityTraceFlags + }; } orchestratorMessages.Add( this.ProcessCreateSubOrchestrationInstanceDecision( From 88718730fa556e30cf80be59e00ae79ae880e732 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 18 Mar 2025 11:32:04 -0700 Subject: [PATCH 08/60] addressing PR comments --- .../Command/CreateSubOrchestrationAction.cs | 5 --- src/DurableTask.Core/OrchestrationContext.cs | 15 ------- src/DurableTask.Core/OrchestrationTags.cs | 10 +++++ .../TaskOrchestrationContext.cs | 23 +++-------- .../TaskOrchestrationDispatcher.cs | 40 ++++++++++++------- 5 files changed, 40 insertions(+), 53 deletions(-) diff --git a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs index fd118ece0..7b55011f8 100644 --- a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs +++ b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs @@ -52,10 +52,5 @@ public class CreateSubOrchestrationAction : OrchestratorAction /// Tags to be applied to the sub-orchestration. /// public IDictionary? Tags { get; set; } - - /// - /// Parent trace of the sub-orchestration. - /// - public DistributedTraceContext? ParentTraceContext { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 7de1dcb58..dae4d592a 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -379,21 +379,6 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, object input, IDictionary tags); - /// - /// Create a sub-orchestration of the specified name and version with the specific instance id - /// - /// Return Type of the TaskOrchestration.RunTask method - /// Name of the orchestration as specified by the ObjectCreator - /// Name of the orchestration as specified by the ObjectCreator - /// InstanceId of the sub-orchestration to create - /// Input for the TaskOrchestration.RunTask method - /// Dictionary of key/value tags associated with this instance - /// Trace context of the parent instance creating this suborchestration - /// Task that represents the execution of the specified sub-orchestration - public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, - object input, IDictionary tags, DistributedTraceContext parentTraceContext); - - /// /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the /// orchestration to fire. diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index 960297143..974406418 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -36,6 +36,16 @@ public static class OrchestrationTags /// public const string FireAndForget = "FireAndForget"; + /// + /// The ID of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#traceparent-header) + /// + public const string TraceParent = "TraceParent"; + + /// + /// The trace state of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#tracestate-header) + /// + public const string TraceState = "TraceState"; + /// /// Check whether the given tags contain the fire and forget tag /// diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 656d852bd..8a48cbe93 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -129,7 +129,7 @@ public override Task CreateSubOrchestrationInstance( string instanceId, object input) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null); } public override Task CreateSubOrchestrationInstance( @@ -139,7 +139,7 @@ public override Task CreateSubOrchestrationInstance( object input, IDictionary tags) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags); } public override Task CreateSubOrchestrationInstance( @@ -147,18 +147,7 @@ public override Task CreateSubOrchestrationInstance( string version, object input) { - return CreateSubOrchestrationInstanceCore(name, version, null, input, null, null); - } - - public override Task CreateSubOrchestrationInstance( - string name, - string version, - string instanceId, - object input, - IDictionary tags, - DistributedTraceContext parentTraceContext) - { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, parentTraceContext); + return CreateSubOrchestrationInstanceCore(name, version, null, input, null); } async Task CreateSubOrchestrationInstanceCore( @@ -166,8 +155,7 @@ async Task CreateSubOrchestrationInstanceCore( string version, string instanceId, object input, - IDictionary tags, - DistributedTraceContext parentTraceContext) + IDictionary tags) { int id = this.idCounter++; string serializedInput = this.MessageDataConverter.SerializeInternal(input); @@ -185,8 +173,7 @@ async Task CreateSubOrchestrationInstanceCore( Name = name, Version = version, Input = serializedInput, - Tags = tags, - ParentTraceContext = parentTraceContext + Tags = tags }; this.orchestratorActionsMap.Add(id, action); diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 2634275a8..2d198d44c 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -425,22 +425,31 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; - DistributedTraceContext? traceContext = null; - if (traceActivity != null) + ActivityContext? parentTraceContext = traceActivity?.Context; + var createNewSpanId = true; + if (createSubOrchestrationAction.Tags != null + && createSubOrchestrationAction.Tags.ContainsKey(OrchestrationTags.TraceParent) + && createSubOrchestrationAction.Tags.ContainsKey(OrchestrationTags.TraceState)) { - traceContext = new DistributedTraceContext( - traceActivity.TraceId.ToString(), - traceActivity.TraceStateString) + try { - ParentTraceFlags = traceActivity.ActivityTraceFlags - }; + parentTraceContext = ActivityContext.Parse( + createSubOrchestrationAction.Tags[OrchestrationTags.TraceParent], + createSubOrchestrationAction.Tags[OrchestrationTags.TraceState]); + createNewSpanId = false; + } + catch (Exception e) + { + TraceHelper.TraceException(TraceEventType.Error, "TaskOrchestrationDispatcher-CreateSubOrchestration-MalformedParentTrace", e); + } } orchestratorMessages.Add( this.ProcessCreateSubOrchestrationInstanceDecision( createSubOrchestrationAction, runtimeState, this.IncludeParameters, - createSubOrchestrationAction.ParentTraceContext ?? traceContext)); + parentTraceContext, + createNewSpanId)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1093,7 +1102,8 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( CreateSubOrchestrationAction createSubOrchestrationAction, OrchestrationRuntimeState runtimeState, bool includeParameters, - DistributedTraceContext? parentTraceContext) + ActivityContext? parentTraceContext, + bool createNewSpanId) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1106,7 +1116,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = parentTraceContext?.ParentSpanId != null ? ActivitySpanId.CreateFromString(parentTraceContext.ParentSpanId.AsSpan()) : ActivitySpanId.CreateRandom(); + ActivitySpanId clientSpanId = createNewSpanId ? ActivitySpanId.CreateRandom() : ((ActivityContext)parentTraceContext!).SpanId; historyEvent.ClientSpanId = clientSpanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1132,13 +1142,13 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - if (parentTraceContext != null) + if (parentTraceContext is ActivityContext parentContext) { ActivityContext activityContext = new ActivityContext( - ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), - clientSpanId, - (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, - parentTraceContext.TraceState); + parentContext.TraceId, + clientSpanId, + parentContext.TraceFlags, + parentContext.TraceState); startedEvent.SetParentTraceContext(activityContext); } From d49c005ae5f745fd339dd1ab892d7cfc84fbb1f5 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 18 Mar 2025 11:33:42 -0700 Subject: [PATCH 09/60] forgot one file --- .../Tracing/DistributedTraceContext.cs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs index 15062c919..a6e3611f4 100644 --- a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs +++ b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs @@ -14,7 +14,6 @@ namespace DurableTask.Core.Tracing { using System; - using System.Diagnostics; using System.Runtime.Serialization; /// @@ -81,17 +80,5 @@ public string? TraceState /// [DataMember] public DateTimeOffset? ActivityStartTime { get; set; } - - /// - /// The span ID of the parent trace - /// - [DataMember] - public string? ParentSpanId { get; set; } - - /// - /// The activity trace flags of the parent trace - /// - [DataMember] - public ActivityTraceFlags? ParentTraceFlags { get; set; } } -} +} \ No newline at end of file From 717573743dc88eac394222c428600f6bee1cf8eb Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 25 Mar 2025 15:29:43 -0700 Subject: [PATCH 10/60] addressing more PR comments --- .../Command/CreateSubOrchestrationAction.cs | 2 -- .../Entities/EventFormat/RequestMessage.cs | 6 ++++++ src/DurableTask.Core/OrchestrationContext.cs | 3 +-- .../TaskOrchestrationDispatcher.cs | 21 ++++++++----------- .../Tracing/DistributedTraceContext.cs | 2 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 11 ++++------ 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs index 7b55011f8..67a9e1013 100644 --- a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs +++ b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs @@ -13,9 +13,7 @@ #nullable enable namespace DurableTask.Core.Command { - using DurableTask.Core.Tracing; using System.Collections.Generic; - using System.Diagnostics; /// /// Orchestrator action for creating sub-orchestrations. diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index e46d1759f..3043a0680 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -98,6 +98,12 @@ internal class RequestMessage : EntityMessage [DataMember] public bool IsLockRequest => LockSet != null; + /// + /// Whether this message already has a trace created for it + /// + [DataMember(Name = "traceCreated")] + public bool TraceCreated { get; set; } + /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index dae4d592a..4b2d8c7fe 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -15,13 +15,11 @@ namespace DurableTask.Core { using System; using System.Collections.Generic; - using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Castle.DynamicProxy; using DurableTask.Core.Entities; using DurableTask.Core.Serializing; - using DurableTask.Core.Tracing; /// /// Context for an orchestration containing the instance, replay status, orchestration methods and proxy methods @@ -379,6 +377,7 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, object input, IDictionary tags); + /// /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the /// orchestration to fire. diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 2d198d44c..9f511a0a4 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -426,17 +426,15 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; ActivityContext? parentTraceContext = traceActivity?.Context; - var createNewSpanId = true; + var spanId = ActivitySpanId.CreateRandom(); if (createSubOrchestrationAction.Tags != null - && createSubOrchestrationAction.Tags.ContainsKey(OrchestrationTags.TraceParent) - && createSubOrchestrationAction.Tags.ContainsKey(OrchestrationTags.TraceState)) + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent) + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState)) { try { - parentTraceContext = ActivityContext.Parse( - createSubOrchestrationAction.Tags[OrchestrationTags.TraceParent], - createSubOrchestrationAction.Tags[OrchestrationTags.TraceState]); - createNewSpanId = false; + parentTraceContext = ActivityContext.Parse(traceParent, traceState); + spanId = parentTraceContext.Value.SpanId; } catch (Exception e) { @@ -449,7 +447,7 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work runtimeState, this.IncludeParameters, parentTraceContext, - createNewSpanId)); + spanId)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1103,7 +1101,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( OrchestrationRuntimeState runtimeState, bool includeParameters, ActivityContext? parentTraceContext, - bool createNewSpanId) + ActivitySpanId spanId) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1116,8 +1114,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = createNewSpanId ? ActivitySpanId.CreateRandom() : ((ActivityContext)parentTraceContext!).SpanId; - historyEvent.ClientSpanId = clientSpanId.ToString(); + historyEvent.ClientSpanId = spanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1146,7 +1143,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( { ActivityContext activityContext = new ActivityContext( parentContext.TraceId, - clientSpanId, + spanId, parentContext.TraceFlags, parentContext.TraceState); startedEvent.SetParentTraceContext(activityContext); diff --git a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs index a6e3611f4..b69b1f5f6 100644 --- a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs +++ b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs @@ -81,4 +81,4 @@ public string? TraceState [DataMember] public DateTimeOffset? ActivityStartTime { get; set; } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 10231b7df..d51be3009 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -54,7 +54,7 @@ public class TraceHelper { newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - + startEvent.SetParentTraceContext(newActivity); } @@ -357,10 +357,8 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { - - // We don't want to emit tracing for external events when they are related to entities - if ((targetInstanceId != null && targetInstanceId.StartsWith("@")) || (instance != null && instance.InstanceId.StartsWith("@"))) - { + if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + { return null; } @@ -397,8 +395,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - // We don't want to emit tracing for external events when they are related to entities - if (instance.InstanceId.StartsWith("@")) + if (Entities.IsEntityInstance(instance.InstanceId)) { return null; } From b2f859f168641e737a740e83e628a25ab248fa78 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 25 Mar 2025 15:34:57 -0700 Subject: [PATCH 11/60] forgot two comments --- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index d51be3009..01148ed50 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -357,6 +357,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { + // There is a possibility that we mislabel the event as an entity event if entities are not enabled if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) { return null; @@ -395,6 +396,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { + // There is a possibility that we mislabel the event as an entity event if entities are not enabled if (Entities.IsEntityInstance(instance.InstanceId)) { return null; From a904e3e6d02aba75f51c6a72bcc8ba67bc6a9dbc Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 27 Mar 2025 19:51:55 -0700 Subject: [PATCH 12/60] first commit! --- .../Entities/ClientEntityHelpers.cs | 5 +- .../Entities/EventFormat/RequestMessage.cs | 19 ++ .../OperationFormat/OperationRequest.cs | 6 + .../OperationFormat/OperationResult.cs | 7 + .../SendSignalOperationAction.cs | 11 ++ .../StartNewOrchestrationOperationAction.cs | 11 ++ .../Entities/OrchestrationEntityContext.cs | 8 +- src/DurableTask.Core/TaskEntityDispatcher.cs | 56 +++++- src/DurableTask.Core/Tracing/Schema.cs | 11 ++ .../Tracing/TraceActivityConstants.cs | 4 + src/DurableTask.Core/Tracing/TraceHelper.cs | 165 +++++++++++++++++- 11 files changed, 295 insertions(+), 8 deletions(-) diff --git a/src/DurableTask.Core/Entities/ClientEntityHelpers.cs b/src/DurableTask.Core/Entities/ClientEntityHelpers.cs index 94bc0512a..507a92430 100644 --- a/src/DurableTask.Core/Entities/ClientEntityHelpers.cs +++ b/src/DurableTask.Core/Entities/ClientEntityHelpers.cs @@ -18,6 +18,7 @@ namespace DurableTask.Core.Entities using Newtonsoft.Json.Linq; using Newtonsoft.Json; using System; + using DurableTask.Core.Tracing; /// /// Utility functions for clients that interact with entities, either by sending events or by accessing the entity state directly in storage @@ -32,8 +33,9 @@ public static class ClientEntityHelpers /// The name of the operation. /// The serialized input for the operation. /// The time to schedule this signal, or null if not a scheduled signal + /// The parent trace context for this operation. /// The event to send. - public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string? input, (DateTime Original, DateTime Capped)? scheduledTimeUtc) + public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string? input, (DateTime Original, DateTime Capped)? scheduledTimeUtc, DistributedTraceContext? parentTraceContext = null) { var request = new RequestMessage() { @@ -44,6 +46,7 @@ public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targe Operation = operationName, ScheduledTime = scheduledTimeUtc?.Original, Input = input, + ParentTraceContext = parentTraceContext }; var eventName = scheduledTimeUtc.HasValue diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index 3043a0680..0d0dd43ae 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -13,6 +13,7 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { + using DurableTask.Core.Tracing; using System; using System.Runtime.Serialization; @@ -104,6 +105,24 @@ internal class RequestMessage : EntityMessage [DataMember(Name = "traceCreated")] public bool TraceCreated { get; set; } + /// + /// Parent trace of this request message + /// + [DataMember(Name = "parentTrace", EmitDefaultValue = false)] + public DistributedTraceContext? ParentTraceContext { get; set; } + + /// + /// Whether or not to create an entity-specific trace for this request message + /// + [DataMember(Name = "createTrace")] + public bool CreateTrace { get; set; } + + /// + /// The time the request was generated. + /// + [DataMember(Name = "requestTime", EmitDefaultValue = false)] + public DateTime? RequestTime { get; set; } + /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs index ab249f88f..6b75c65c0 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs @@ -13,6 +13,7 @@ #nullable enable namespace DurableTask.Core.Entities.OperationFormat { + using DurableTask.Core.Tracing; using System; /// @@ -37,5 +38,10 @@ public class OperationRequest /// The input for the operation. Can be null if no input was given. /// public string? Input { get; set; } + + /// + /// The trace context for the operation, if any. + /// + public DistributedTraceContext? TraceContext { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs index 01cc41b11..9e0a6a398 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs @@ -11,6 +11,8 @@ // limitations under the License. // ---------------------------------------------------------------------------------- #nullable enable +using System; + namespace DurableTask.Core.Entities.OperationFormat { /// @@ -46,5 +48,10 @@ public bool IsError /// this field exclusively when collecting error information. /// public FailureDetails? FailureDetails { get; set; } + + /// + /// The completion time of the operation. + /// + public DateTime? EndTime { get; set; } } } diff --git a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs index 04531ac31..eb5446d1e 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs @@ -13,6 +13,7 @@ #nullable enable namespace DurableTask.Core.Entities.OperationFormat { + using DurableTask.Core.Tracing; using System; /// @@ -45,5 +46,15 @@ public class SendSignalOperationAction : OperationAction /// Optionally, a scheduled delivery time for the signal. /// public DateTime? ScheduledTime { get; set; } + + /// + /// The time the signal request was generated. + /// + public DateTime? RequestTime { get; set; } + + /// + /// The parent trace context for the signal, if any. + /// + public DistributedTraceContext? ParentTraceContext { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs index 4c06f80cd..d44040deb 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs @@ -13,6 +13,7 @@ #nullable enable namespace DurableTask.Core.Entities.OperationFormat { + using DurableTask.Core.Tracing; using System; using System.Collections.Generic; @@ -52,5 +53,15 @@ public class StartNewOrchestrationOperationAction : OperationAction /// public DateTime? ScheduledStartTime { get; set; } + /// + /// The time of the new orchestration request creation. + /// + public DateTime? RequestTime { get; set; } + + /// + /// The parent trace context for the operation, if any. + /// + public DistributedTraceContext? ParentTraceContext { get; set; } + } } \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs index 3e21f31a6..9aa2308c3 100644 --- a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs +++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs @@ -228,6 +228,8 @@ public IEnumerable EmitLockReleaseMessages() /// A unique identifier for this request. /// A time for which to schedule the delivery, or null if this is not a scheduled message /// The operation input + /// Whether or not to create an entity-specific trace for this event + /// The time the request was created /// The event to send. public EntityMessageEvent EmitRequestMessage( OrchestrationInstance target, @@ -235,7 +237,9 @@ public EntityMessageEvent EmitRequestMessage( bool oneWay, Guid operationId, (DateTime Original, DateTime Capped)? scheduledTimeUtc, - string? input) + string? input, + bool? createTrace, + DateTime? requestTime) { this.CheckEntitySupport(); @@ -248,6 +252,8 @@ public EntityMessageEvent EmitRequestMessage( Operation = operationName, ScheduledTime = scheduledTimeUtc?.Original, Input = input, + CreateTrace = createTrace ?? false, + RequestTime = requestTime, }; this.AdjustOutgoingMessage(target.InstanceId, request, scheduledTimeUtc?.Capped, out string eventName); diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index aad78675d..82eaf8032 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -25,6 +25,7 @@ namespace DurableTask.Core using System; using System.Collections.Generic; using System.Diagnostics; + using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -647,20 +648,32 @@ public void ToBeContinued(SchedulerState schedulerState) } } - public List GetOperationRequests() + public (List, List) GetOperationRequestsAndTraceActivities(string instanceId) { var operations = new List(this.operationBatch.Count); + var traceActivities = new List(this.operationBatch.Count); for (int i = 0; i < this.operationBatch.Count; i++) { var request = this.operationBatch[i]; + + bool successfullyParsed = ActivityContext.TryParse(request.ParentTraceContext?.TraceParent, request.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext); + var traceActivity = TraceHelper.StartActivityForProcessingEntityInvocation( + instanceId, + EntityId.FromString(instanceId).Name, + request.Operation, + request.IsSignal, + successfullyParsed ? parentTraceContext : Activity.Current?.Context); + traceActivities.Add(traceActivity); + operations.Add(new OperationRequest() { Operation = request.Operation, Id = request.Id, Input = request.Input, + TraceContext = traceActivity != null ? new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString) : null, }); } - return operations; + return (operations, traceActivities); } public Queue RemoveDeferredWork(int index) @@ -728,6 +741,24 @@ void SendSignalMessage(WorkItemEffects effects, SchedulerState schedulerState, S eventName = EntityMessageEventNames.RequestMessageEventName; schedulerState.MessageSorter.LabelOutgoingMessage(message, action.InstanceId, DateTime.UtcNow, this.entityBackendProperties.EntityMessageReorderWindow); } + + bool successfullyParsed = ActivityContext.TryParse( + action.ParentTraceContext?.TraceParent, + action.ParentTraceContext?.TraceState, + out ActivityContext parentTraceContext); + using Activity traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( + destination.InstanceId, + EntityId.FromString(destination.InstanceId).Name, + action.Name, + true, + successfullyParsed ? parentTraceContext : Activity.Current?.Context, + entityId: effects.InstanceId, + scheduledTime: action.ScheduledTime, + startTime: action.RequestTime); + if (traceActivity?.Id != null) + { + message.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); + } this.ProcessSendEventMessage(effects, destination, eventName, message); } @@ -816,6 +847,21 @@ internal void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRunt Name = action.Name, Version = action.Version, }; + + bool successfullyParsed = ActivityContext.TryParse( + action.ParentTraceContext?.TraceParent, + action.ParentTraceContext?.TraceState, + out ActivityContext parentTraceContext); + using Activity traceActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( + runtimeState.OrchestrationInstance.InstanceId, + destination.InstanceId, + successfullyParsed ? parentTraceContext : Activity.Current?.Context, + scheduledTime: action.ScheduledStartTime, + startTime: action.RequestTime); + if (traceActivity?.Id != null) + { + executionStartedEvent.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); + } this.logHelper.SchedulingOrchestration(executionStartedEvent); effects.InstanceMessages.Add(new TaskMessage @@ -829,12 +875,13 @@ internal void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRunt async Task ExecuteViaMiddlewareAsync(Work workToDoNow, OrchestrationInstance instance, string serializedEntityState) { + var (operations, traceActivities) = workToDoNow.GetOperationRequestsAndTraceActivities(instance.InstanceId); // the request object that will be passed to the worker var request = new EntityBatchRequest() { InstanceId = instance.InstanceId, EntityState = serializedEntityState, - Operations = workToDoNow.GetOperationRequests(), + Operations = operations, }; this.logHelper.EntityBatchExecuting(request); @@ -871,11 +918,12 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => } var result = await taskEntity.ExecuteOperationBatchAsync(request); - + dispatchContext.SetProperty(result); }); var result = dispatchContext.GetProperty(); + TraceHelper.EndActivitiesForProcessingEntityInvocation(traceActivities, result.Results, result.FailureDetails); this.logHelper.EntityBatchExecuted(request, result); diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 8a91e3fb1..2317a8c79 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -33,5 +33,16 @@ internal static class Status internal const string Code = "otel.status_code"; internal const string Description = "otel.status_description"; } + + internal static class Entity + { + internal const string Type = Task.Type; + internal const string EntityId = "durabletask.entity.entity_id"; + internal const string TargetInstanceId = "durabletask.entity.target_instance_id"; + internal const string TargetEntityId = "durabletask.entity.target_entity_id"; + internal const string EntityOperation = "durabletask.entity.entity_operation"; + internal const string ErrorMessage = "durabletask.entity.error_message"; + internal const string ScheduledTime = "durabletask.entity.scheduled_time"; + } } } diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index 927d1bc93..ef56f52d8 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -20,8 +20,12 @@ internal class TraceActivityConstants public const string Activity = "activity"; public const string Event = "event"; public const string Timer = "timer"; + public const string Entity = "entity"; public const string CreateOrchestration = "create_orchestration"; public const string OrchestrationEvent = "orchestration_event"; + + public const string CallEntity = "call_entity"; + public const string SignalEntity = "signal_entity"; } } diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 01148ed50..861350b42 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -19,7 +19,12 @@ namespace DurableTask.Core.Tracing using System.Globalization; using System.Runtime.ExceptionServices; using DurableTask.Core.Common; + using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.History; + using System.Linq; + using Newtonsoft.Json; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities; /// /// Helper class for logging/tracing @@ -359,8 +364,8 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( { // There is a possibility that we mislabel the event as an entity event if entities are not enabled if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) - { - return null; + { + return TryParseEntityRequest(eventRaisedEvent, targetInstanceId); } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -447,6 +452,114 @@ internal static void EmitTraceActivityForTimer( } } + internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, ActivityContext? parentTraceContext, string? entityId = null, DateTime? scheduledTime = null, DateTime? startTime = null) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateEntitySpanName(entityName, operationName), + kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client, + parentContext: parentTraceContext ?? default, + startTime: startTime ?? DateTime.UtcNow); + + if (newActivity == null) + { + return null; + } + + newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Entity.EntityOperation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); + newActivity.SetTag(Schema.Entity.TargetEntityId, targetEntityId); + + if (!string.IsNullOrEmpty(entityId)) + { + newActivity.SetTag(Schema.Entity.EntityId, entityId); + } + + if (scheduledTime != null) + { + newActivity.SetTag(Schema.Entity.ScheduledTime, scheduledTime.Value.ToString()); + } + + return newActivity; + } + + internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string targetInstanceId, ActivityContext? parentTraceContext, DateTime? scheduledTime = null, DateTime? startTime = null) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.Entity, TraceActivityConstants.CreateOrchestration, null), + kind: ActivityKind.Producer, + parentContext: parentTraceContext ?? default, + startTime: startTime ?? DateTime.UtcNow); + + if (newActivity == null) + { + return null; + } + + newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Entity.TargetInstanceId, targetInstanceId); + newActivity.SetTag(Schema.Entity.EntityId, entityId); + + if (scheduledTime != null) + { + newActivity.SetTag(Schema.Entity.ScheduledTime, scheduledTime.Value.ToString()); + } + + return newActivity; + } + + internal static Activity? StartActivityForProcessingEntityInvocation(string entityId, string entityName, string operationName, bool signalEntity, ActivityContext? parentTraceContext) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateEntitySpanName(entityName, operationName), + kind: signalEntity ? ActivityKind.Consumer : ActivityKind.Server, + parentContext: parentTraceContext ?? default); + + if (newActivity == null) + { + return null; + } + + newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Entity.EntityOperation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); + newActivity.SetTag(Schema.Entity.EntityId, entityId); + + return newActivity; + } + + internal static void EndActivitiesForProcessingEntityInvocation(List traceActivities, List results, FailureDetails? batchFailureDetails) + { + if (results.Count == traceActivities.Count) + { + foreach (var (activity, result) in traceActivities.Zip(results, (activity, result) => (activity, result))) + { + if (result.ErrorMessage != null || result.FailureDetails != null) + { + activity.SetTag(Schema.Entity.ErrorMessage, result.ErrorMessage ?? result.FailureDetails!.ErrorMessage); + } + if (result.EndTime is DateTime endTime) + { + activity.SetEndTime(endTime); + } + activity.Dispose(); + } + } + // This can happen if some of the operations failed and have no corresponding OperationResult + // There is no way to map the successful operation results to the corresponding operation requests or trace activities, so we will just "fail" the trace activities in this case and dispose them + else + { + string errorMessage = "Unable to generate a trace activity for the entity invocation even though it may have succeeded."; + if (batchFailureDetails is FailureDetails failureDetails) + { + errorMessage += $" If it failed, it may be due to {failureDetails.ErrorMessage}"; + } + foreach (var activity in traceActivities) + { + activity.SetTag(Schema.Entity.ErrorMessage, errorMessage); + activity.Dispose(); + } + } + } + internal static void SetRuntimeStatusTag(string runtimeStatus) { DistributedTraceActivity.Current?.SetTag(Schema.Task.Status, runtimeStatus); @@ -474,6 +587,11 @@ static string CreateTimerSpanName(string orchestrationName) return $"{TraceActivityConstants.Orchestration}:{orchestrationName}:{TraceActivityConstants.Timer}"; } + static string CreateEntitySpanName(string entityName, string operationName) + { + return $"{TraceActivityConstants.Entity}:{entityName}:{operationName}"; + } + /// /// Simple trace with no iid or eid /// @@ -709,5 +827,48 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } + + private static Activity? TryParseEntityRequest(EventRaisedEvent raisedEvent, string? targetInstanceId) + { + if (string.IsNullOrEmpty(targetInstanceId)) + { + return null; + } + + try + { + var requestMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); + if (requestMessage == null || !requestMessage.CreateTrace) + { + return null; + } + + bool successfullyParsed = ActivityContext.TryParse( + requestMessage.ParentTraceContext?.TraceParent, + requestMessage.ParentTraceContext?.TraceState, + out ActivityContext parentTraceContext); + + Activity? newActivity = StartActivityForCallingOrSignalingEntity( + targetInstanceId!, + EntityId.FromString(targetInstanceId!).Name, + requestMessage.Operation!, + requestMessage.IsSignal, + successfullyParsed ? parentTraceContext : Activity.Current?.Context, + scheduledTime: requestMessage.ScheduledTime, + startTime: requestMessage.RequestTime); + + if (!string.IsNullOrEmpty(newActivity?.Id)) + { + requestMessage.ParentTraceContext = new DistributedTraceContext(newActivity!.Id!, newActivity.TraceStateString); + raisedEvent.Input = JsonConvert.SerializeObject(requestMessage, Serializer.InternalSerializerSettings); + } + + return newActivity; + } + catch (Exception) + { + return null; + } + } } } From 869d1a24e7665559d9fbae20b1c0ff8c59215549 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 31 Mar 2025 15:51:38 -0700 Subject: [PATCH 13/60] addressing a few small comments --- src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs | 6 ------ src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index 3043a0680..e46d1759f 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -98,12 +98,6 @@ internal class RequestMessage : EntityMessage [DataMember] public bool IsLockRequest => LockSet != null; - /// - /// Whether this message already has a trace created for it - /// - [DataMember(Name = "traceCreated")] - public bool TraceCreated { get; set; } - /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 01148ed50..a8b0c73c0 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -54,7 +54,7 @@ public class TraceHelper { newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - + startEvent.SetParentTraceContext(newActivity); } From 25b87ad84e7df2d33ca47d7bde716b75d883b87a Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 31 Mar 2025 16:04:32 -0700 Subject: [PATCH 14/60] added some new lines back --- src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs | 6 ------ .../Entities/OperationFormat/OperationRequest.cs | 2 +- .../Entities/OperationFormat/SendSignalOperationAction.cs | 2 +- .../OperationFormat/StartNewOrchestrationOperationAction.cs | 2 +- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index 0d0dd43ae..b4dbf994c 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -99,12 +99,6 @@ internal class RequestMessage : EntityMessage [DataMember] public bool IsLockRequest => LockSet != null; - /// - /// Whether this message already has a trace created for it - /// - [DataMember(Name = "traceCreated")] - public bool TraceCreated { get; set; } - /// /// Parent trace of this request message /// diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs index 6b75c65c0..bdeee2076 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs @@ -44,4 +44,4 @@ public class OperationRequest /// public DistributedTraceContext? TraceContext { get; set; } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs index eb5446d1e..12833dcde 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs @@ -57,4 +57,4 @@ public class SendSignalOperationAction : OperationAction /// public DistributedTraceContext? ParentTraceContext { get; set; } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs index d44040deb..35fedf81d 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs @@ -64,4 +64,4 @@ public class StartNewOrchestrationOperationAction : OperationAction public DistributedTraceContext? ParentTraceContext { get; set; } } -} \ No newline at end of file +} From 34a09f12fb63e3898e9c7dfa94563f5ae0da76e8 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 3 Apr 2025 12:49:42 -0700 Subject: [PATCH 15/60] added comments, fixed up some incorrect logic regarding linking trace activities to their parents --- src/DurableTask.Core/TaskEntityDispatcher.cs | 41 ++++++++++++++------ src/DurableTask.Core/Tracing/TraceHelper.cs | 32 ++++++++------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 82eaf8032..7a8aaa1ba 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -656,15 +656,26 @@ public void ToBeContinued(SchedulerState schedulerState) { var request = this.operationBatch[i]; + Activity traceActivity = null; bool successfullyParsed = ActivityContext.TryParse(request.ParentTraceContext?.TraceParent, request.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext); - var traceActivity = TraceHelper.StartActivityForProcessingEntityInvocation( - instanceId, - EntityId.FromString(instanceId).Name, - request.Operation, - request.IsSignal, - successfullyParsed ? parentTraceContext : Activity.Current?.Context); + + // We only want to create a trace activity for processing the entity invocation in the case that we can successfully parse the trace context of the request that led to this entity invocation. + // Otherwise, we will create an unlinked trace activity with no parent + if (successfullyParsed) + { + traceActivity = TraceHelper.StartActivityForProcessingEntityInvocation( + instanceId, + EntityId.FromString(instanceId).Name, + request.Operation, + request.IsSignal, + parentTraceContext); + } + + // We still want to add the trace activity to the list even if it was not successfully created and is null. This is because otherwise we have no easy way of mapping OperationResults to Activities otherwise if the lists + // do not have the same length in TraceHelper.EndActivitiesForProcessingEntityInvocation. We will simply skip ending the Activity if it is null in this method traceActivities.Add(traceActivity); + // The trace context of the operation request will be the Activity just created - this can become the parent of future operations started by the entity once it processes the OperationRequest operations.Add(new OperationRequest() { Operation = request.Operation, @@ -746,15 +757,18 @@ void SendSignalMessage(WorkItemEffects effects, SchedulerState schedulerState, S action.ParentTraceContext?.TraceParent, action.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext); - using Activity traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( + + // We only want to create a trace activity for signaling the entity in the case that we can successfully parse the parent trace context of the signal request. + // Otherwise, we will create an unlinked trace activity with no parent + using Activity traceActivity = successfullyParsed ? TraceHelper.StartActivityForCallingOrSignalingEntity( destination.InstanceId, EntityId.FromString(destination.InstanceId).Name, action.Name, true, - successfullyParsed ? parentTraceContext : Activity.Current?.Context, + parentTraceContext, entityId: effects.InstanceId, scheduledTime: action.ScheduledTime, - startTime: action.RequestTime); + startTime: action.RequestTime) : null; if (traceActivity?.Id != null) { message.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); @@ -852,12 +866,15 @@ internal void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRunt action.ParentTraceContext?.TraceParent, action.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext); - using Activity traceActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( + + // We only want to create a trace activity for an entity starting an orchestration in the case that we can successfully parse the parent trace context of the start orchestration request. + // Otherwise, we will create an unlinked trace activity with no parent + using Activity traceActivity = successfullyParsed ? TraceHelper.StartActivityForEntityStartingAnOrchestration( runtimeState.OrchestrationInstance.InstanceId, destination.InstanceId, - successfullyParsed ? parentTraceContext : Activity.Current?.Context, + parentTraceContext, scheduledTime: action.ScheduledStartTime, - startTime: action.RequestTime); + startTime: action.RequestTime) : null; if (traceActivity?.Id != null) { executionStartedEvent.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 861350b42..03824d57a 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -365,6 +365,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( // There is a possibility that we mislabel the event as an entity event if entities are not enabled if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) { + // In the case that this an event corresponding to an orchestrationg invoking an entity, we may want to create an entity-specific trace activity for the event return TryParseEntityRequest(eventRaisedEvent, targetInstanceId); } @@ -532,15 +533,18 @@ internal static void EndActivitiesForProcessingEntityInvocation(List t { foreach (var (activity, result) in traceActivities.Zip(results, (activity, result) => (activity, result))) { - if (result.ErrorMessage != null || result.FailureDetails != null) + if (activity != null) { - activity.SetTag(Schema.Entity.ErrorMessage, result.ErrorMessage ?? result.FailureDetails!.ErrorMessage); + if (result.ErrorMessage != null || result.FailureDetails != null) + { + activity.SetTag(Schema.Entity.ErrorMessage, result.ErrorMessage ?? result.FailureDetails!.ErrorMessage); + } + if (result.EndTime is DateTime endTime) + { + activity.SetEndTime(endTime); + } + activity.Dispose(); } - if (result.EndTime is DateTime endTime) - { - activity.SetEndTime(endTime); - } - activity.Dispose(); } } // This can happen if some of the operations failed and have no corresponding OperationResult @@ -554,8 +558,11 @@ internal static void EndActivitiesForProcessingEntityInvocation(List t } foreach (var activity in traceActivities) { - activity.SetTag(Schema.Entity.ErrorMessage, errorMessage); - activity.Dispose(); + if (activity != null) + { + activity.SetTag(Schema.Entity.ErrorMessage, errorMessage); + activity.Dispose(); + } } } } @@ -843,17 +850,12 @@ static void ExceptionHandlingWrapper(Action innerFunc) return null; } - bool successfullyParsed = ActivityContext.TryParse( - requestMessage.ParentTraceContext?.TraceParent, - requestMessage.ParentTraceContext?.TraceState, - out ActivityContext parentTraceContext); - Activity? newActivity = StartActivityForCallingOrSignalingEntity( targetInstanceId!, EntityId.FromString(targetInstanceId!).Name, requestMessage.Operation!, requestMessage.IsSignal, - successfullyParsed ? parentTraceContext : Activity.Current?.Context, + Activity.Current?.Context, scheduledTime: requestMessage.ScheduledTime, startTime: requestMessage.RequestTime); From 129a2e6e9421e6f3322378baa761a129ac11db41 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 8 Apr 2025 12:10:58 -0700 Subject: [PATCH 16/60] trying to add support for tracing when ShimDurableTaskClient is used to create in orchestration in the dotnet package --- .../Entities/ClientEntityHelpers.cs | 6 +++-- .../Entities/OrchestrationEntityContext.cs | 2 +- src/DurableTask.Core/OrchestrationTags.cs | 10 +++++++++ src/DurableTask.Core/Tracing/TraceHelper.cs | 22 +++++++++++++++++-- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/DurableTask.Core/Entities/ClientEntityHelpers.cs b/src/DurableTask.Core/Entities/ClientEntityHelpers.cs index 507a92430..65c22c7a5 100644 --- a/src/DurableTask.Core/Entities/ClientEntityHelpers.cs +++ b/src/DurableTask.Core/Entities/ClientEntityHelpers.cs @@ -34,8 +34,9 @@ public static class ClientEntityHelpers /// The serialized input for the operation. /// The time to schedule this signal, or null if not a scheduled signal /// The parent trace context for this operation. + /// The time at which the request was made. /// The event to send. - public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string? input, (DateTime Original, DateTime Capped)? scheduledTimeUtc, DistributedTraceContext? parentTraceContext = null) + public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string? input, (DateTime Original, DateTime Capped)? scheduledTimeUtc, DistributedTraceContext? parentTraceContext = null, DateTime? requestTime = null) { var request = new RequestMessage() { @@ -46,7 +47,8 @@ public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targe Operation = operationName, ScheduledTime = scheduledTimeUtc?.Original, Input = input, - ParentTraceContext = parentTraceContext + ParentTraceContext = parentTraceContext, + RequestTime = requestTime }; var eventName = scheduledTimeUtc.HasValue diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs index 9aa2308c3..754bef343 100644 --- a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs +++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs @@ -229,7 +229,7 @@ public IEnumerable EmitLockReleaseMessages() /// A time for which to schedule the delivery, or null if this is not a scheduled message /// The operation input /// Whether or not to create an entity-specific trace for this event - /// The time the request was created + /// The time at which the request was made. /// The event to send. public EntityMessageEvent EmitRequestMessage( OrchestrationInstance target, diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index 974406418..9b296b059 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -46,6 +46,16 @@ public static class OrchestrationTags /// public const string TraceState = "TraceState"; + /// + /// The time that the request for a new orchestration was issued + /// + public const string RequestTime = "RequestTime"; + + /// + /// Whether or not to create a trace for the of the orchestration + /// + public const string CreateTraceForNewOrchestration = "CreateTrace"; + /// /// Check whether the given tags contain the fire and forget tag /// diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 03824d57a..d5089d3f7 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -44,9 +44,19 @@ public class TraceHelper /// internal static Activity? StartActivityForNewOrchestration(ExecutionStartedEvent startEvent) { + startEvent.TryGetParentTraceContext(out ActivityContext activityContext); + DateTime? startTime = null; + if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.RequestTime) && + DateTime.TryParse(startEvent.Tags[OrchestrationTags.RequestTime], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTime requestTime)) + { + startTime = requestTime; + } + Activity? newActivity = ActivityTraceSource.StartActivity( - name: CreateSpanName(TraceActivityConstants.CreateOrchestration, startEvent.Name, startEvent.Version), - kind: ActivityKind.Producer); + CreateSpanName(TraceActivityConstants.CreateOrchestration, startEvent.Name, startEvent.Version), + kind: ActivityKind.Producer, + parentContext: activityContext, + startTime: startTime ?? DateTime.UtcNow); if (newActivity != null) { @@ -80,6 +90,14 @@ public class TraceHelper return null; } + if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.CreateTraceForNewOrchestration)) + { + startEvent.Tags.Remove(OrchestrationTags.CreateTraceForNewOrchestration); + // This immediately disposes of and ends the activity for the new orchestration. Note that in this case since the start time of activity is set to the time the + // the request for a new orchestration was made, but the Activity is only disposed of now, the duration of the Activity will be longer than if it was created and stopped immediately after the request creation. + using var activityForNewOrchestration = StartActivityForNewOrchestration(startEvent); + } + if (!startEvent.TryGetParentTraceContext(out ActivityContext activityContext)) { return null; From 31164634ce9808db8bf4e0b8a7eae62e4db35631 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 10 Apr 2025 22:27:03 -0700 Subject: [PATCH 17/60] refactored most of the tracing into this repo to more accurately reflect the end time of activites (like, when the call/signal request to an entity is actually sent, or when an orchestration is actually created by an entity) --- .../Entities/EventFormat/RequestMessage.cs | 19 ++ .../Entities/EventFormat/ResponseMessage.cs | 22 +++ src/DurableTask.Core/OrchestrationTags.cs | 5 + .../TaskOrchestrationDispatcher.cs | 57 +++--- src/DurableTask.Core/Tracing/Schema.cs | 11 ++ .../Tracing/TraceActivityConstants.cs | 4 + src/DurableTask.Core/Tracing/TraceHelper.cs | 164 +++++++++++++++++- 7 files changed, 249 insertions(+), 33 deletions(-) diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index e46d1759f..d9ed3460d 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -13,6 +13,7 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { + using DurableTask.Core.Tracing; using System; using System.Runtime.Serialization; @@ -98,6 +99,24 @@ internal class RequestMessage : EntityMessage [DataMember] public bool IsLockRequest => LockSet != null; + /// + /// Parent trace context of this request message. + /// + [DataMember(Name = "parentTraceContext", EmitDefaultValue = false)] + public DistributedTraceContext? ParentTraceContext { get; set; } + + /// + /// Whether or not to create an entity-specific trace for this request message + /// + [DataMember(Name = "createTrace")] + public bool CreateTrace { get; set; } + + /// + /// The time the request was generated. + /// + [DataMember(Name = "requestTime", EmitDefaultValue = false)] + public DateTimeOffset? RequestTime { get; set; } + /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs index 8f78516b5..eb06a76ea 100644 --- a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs @@ -13,6 +13,8 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { + using DurableTask.Core.Tracing; + using System; using System.Runtime.Serialization; [DataContract] @@ -32,6 +34,9 @@ internal class ResponseMessage : EntityMessage [IgnoreDataMember] public bool IsErrorResult => this.ErrorMessage != null || this.FailureDetails != null; + [DataMember(Name = "requestInfo", EmitDefaultValue = false)] + public RequestInformation? RequestInfo { get; set; } + public override string GetShortDescription() { if (this.IsErrorResult) @@ -47,5 +52,22 @@ public override string GetShortDescription() return $"[OperationSuccessful ({Result?.Length ?? 0} chars)]"; } } + internal class RequestInformation + { + [DataMember] + public string? Operation { get; set; } + + [DataMember] + public DateTime? ScheduledTime { get; set; } + + [DataMember] + public DateTimeOffset? RequestTime { get; set; } + + [DataMember] + public string? ClientSpanId { get; set; } + + [DataMember] + public DistributedTraceContext? ParentTraceContext { get; set; } + } } } diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index 974406418..f8c7492a2 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -46,6 +46,11 @@ public static class OrchestrationTags /// public const string TraceState = "TraceState"; + /// + /// The time the request for a new orchestration was created. + /// + public const string RequestTime = "RequestTime"; + /// /// Check whether the given tags contain the fire and forget tag /// diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 9f511a0a4..a58cabd71 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -425,29 +425,12 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; - ActivityContext? parentTraceContext = traceActivity?.Context; - var spanId = ActivitySpanId.CreateRandom(); - if (createSubOrchestrationAction.Tags != null - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent) - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState)) - { - try - { - parentTraceContext = ActivityContext.Parse(traceParent, traceState); - spanId = parentTraceContext.Value.SpanId; - } - catch (Exception e) - { - TraceHelper.TraceException(TraceEventType.Error, "TaskOrchestrationDispatcher-CreateSubOrchestration-MalformedParentTrace", e); - } - } orchestratorMessages.Add( this.ProcessCreateSubOrchestrationInstanceDecision( createSubOrchestrationAction, runtimeState, this.IncludeParameters, - parentTraceContext, - spanId)); + traceActivity)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1100,8 +1083,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( CreateSubOrchestrationAction createSubOrchestrationAction, OrchestrationRuntimeState runtimeState, bool includeParameters, - ActivityContext? parentTraceContext, - ActivitySpanId spanId) + Activity? parentTraceActivity) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1114,7 +1096,8 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - historyEvent.ClientSpanId = spanId.ToString(); + ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); + historyEvent.ClientSpanId = clientSpanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1139,13 +1122,33 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - if (parentTraceContext is ActivityContext parentContext) + // If a trace parent was provided, then we will attempt to parse the parent trace context provided in the orchestration tags and use that as the + // parent trace context of the call to create a suborchestration rather than the current Activity's context. + if (createSubOrchestrationAction.Tags != null + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) + { + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) + && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) + { + var requestTime = DateTimeOffset.UtcNow; + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.RequestTime, out string requestTimeString)) + { + DateTimeOffset.TryParse(requestTimeString, out requestTime); + } + using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( + runtimeState.OrchestrationInstance!.InstanceId, + createSubOrchestrationAction.InstanceId!, + parentTraceContext, + requestTime); + if (createOrchestrationActivity != null) + { + startedEvent.SetParentTraceContext(createOrchestrationActivity); + } + } + } + else if (parentTraceActivity != null) { - ActivityContext activityContext = new ActivityContext( - parentContext.TraceId, - spanId, - parentContext.TraceFlags, - parentContext.TraceState); + ActivityContext activityContext = new ActivityContext(parentTraceActivity.TraceId, clientSpanId, parentTraceActivity.ActivityTraceFlags, parentTraceActivity.TraceStateString); startedEvent.SetParentTraceContext(activityContext); } diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 8a91e3fb1..2317a8c79 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -33,5 +33,16 @@ internal static class Status internal const string Code = "otel.status_code"; internal const string Description = "otel.status_description"; } + + internal static class Entity + { + internal const string Type = Task.Type; + internal const string EntityId = "durabletask.entity.entity_id"; + internal const string TargetInstanceId = "durabletask.entity.target_instance_id"; + internal const string TargetEntityId = "durabletask.entity.target_entity_id"; + internal const string EntityOperation = "durabletask.entity.entity_operation"; + internal const string ErrorMessage = "durabletask.entity.error_message"; + internal const string ScheduledTime = "durabletask.entity.scheduled_time"; + } } } diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index 927d1bc93..ef56f52d8 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -20,8 +20,12 @@ internal class TraceActivityConstants public const string Activity = "activity"; public const string Event = "event"; public const string Timer = "timer"; + public const string Entity = "entity"; public const string CreateOrchestration = "create_orchestration"; public const string OrchestrationEvent = "orchestration_event"; + + public const string CallEntity = "call_entity"; + public const string SignalEntity = "signal_entity"; } } diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index a8b0c73c0..4fd15dec3 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -20,6 +20,9 @@ namespace DurableTask.Core.Tracing using System.Runtime.ExceptionServices; using DurableTask.Core.Common; using DurableTask.Core.History; + using Newtonsoft.Json; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities; /// /// Helper class for logging/tracing @@ -54,7 +57,7 @@ public class TraceHelper { newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - + startEvent.SetParentTraceContext(newActivity); } @@ -358,9 +361,13 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( string? targetInstanceId) { // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) - { - return null; + if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty)) + { + return TryParseEntityRequest(eventRaisedEvent, targetInstanceId!); + } + else if (Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + { + return TryParseEntityResponse(eventRaisedEvent, instance?.InstanceId!); } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -399,7 +406,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( // There is a possibility that we mislabel the event as an entity event if entities are not enabled if (Entities.IsEntityInstance(instance.InstanceId)) { - return null; + return TryParseEntityRequest(eventRaised, instance.InstanceId); } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -447,6 +454,56 @@ internal static void EmitTraceActivityForTimer( } } + internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, DateTime? scheduledTime, ActivityContext parentTraceContext, DateTimeOffset? startTime, string? entityId = null) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateEntitySpanName(entityName, operationName), + kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client, + parentContext: parentTraceContext, + startTime: startTime ?? DateTimeOffset.UtcNow); + + if (newActivity == null) + { + return null; + } + + newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Entity.EntityOperation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); + newActivity.SetTag(Schema.Entity.TargetEntityId, targetEntityId); + + if (!string.IsNullOrEmpty(entityId)) + { + newActivity.SetTag(Schema.Entity.EntityId, entityId); + } + + if (scheduledTime != null) + { + newActivity.SetTag(Schema.Entity.ScheduledTime, scheduledTime.Value.ToString()); + } + + return newActivity; + } + + internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset startTime) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.Entity, TraceActivityConstants.CreateOrchestration, null), + kind: ActivityKind.Producer, + parentContext: parentTraceContext, + startTime: startTime); + + if (newActivity == null) + { + return null; + } + + newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Entity.TargetInstanceId, targetInstanceId); + newActivity.SetTag(Schema.Entity.EntityId, entityId); + + return newActivity; + } + internal static void SetRuntimeStatusTag(string runtimeStatus) { DistributedTraceActivity.Current?.SetTag(Schema.Task.Status, runtimeStatus); @@ -474,6 +531,11 @@ static string CreateTimerSpanName(string orchestrationName) return $"{TraceActivityConstants.Orchestration}:{orchestrationName}:{TraceActivityConstants.Timer}"; } + static string CreateEntitySpanName(string entityName, string operationName) + { + return $"{TraceActivityConstants.Entity}:{entityName}:{operationName}"; + } + /// /// Simple trace with no iid or eid /// @@ -709,5 +771,95 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } + + private static Activity? TryParseEntityRequest(EventRaisedEvent raisedEvent, string targetInstanceId) + { + try + { + var requestMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); + if (requestMessage == null || !requestMessage.CreateTrace) + { + return null; + } + + var parentTraceContext = Activity.Current?.Context; + if (requestMessage.ParentTraceContext != null) + { + if (ActivityContext.TryParse(requestMessage.ParentTraceContext.TraceParent, requestMessage.ParentTraceContext.TraceState, out ActivityContext activityContext)) + { + parentTraceContext = activityContext; + } + // If a parent trace context was passed with the request message, this should be the parent of the current Activity, so if we cannot parse it we should not create the Activity + // or else it will be incorrectly linked. + else + { + parentTraceContext = null; + } + } + + // We only want to create a trace activity for calling/signaling an entity in the case that we can successfully get the parent trace context of the request. + // Otherwise, we will create an unlinked trace activity with no parent. + if (parentTraceContext == null) + { + return null; + } + + Activity? newActivity = StartActivityForCallingOrSignalingEntity( + targetInstanceId!, + EntityId.FromString(targetInstanceId!).Name, + requestMessage.Operation!, + requestMessage.IsSignal, + requestMessage.ScheduledTime, + parentTraceContext.Value, + entityId: requestMessage.ParentInstanceId != null && Entities.IsEntityInstance(requestMessage.ParentInstanceId) ? requestMessage.ParentInstanceId : null, + startTime: requestMessage.RequestTime); + + if (!string.IsNullOrEmpty(newActivity?.Id)) + { + requestMessage.ParentTraceContext = new DistributedTraceContext(newActivity!.Id!, newActivity.TraceStateString); + raisedEvent.Input = JsonConvert.SerializeObject(requestMessage, Serializer.InternalSerializerSettings); + } + + return newActivity; + } + catch (Exception) + { + return null; + } + } + + private static Activity? TryParseEntityResponse(EventRaisedEvent raisedEvent, string instanceId) + { + try + { + var responseMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); + if (responseMessage == null || responseMessage.RequestInfo == null) + { + return null; + } + + if (!ActivityContext.TryParse(responseMessage.RequestInfo.ParentTraceContext?.TraceParent, responseMessage.RequestInfo.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) + { + return null; + } + + Activity? newActivity = StartActivityForCallingOrSignalingEntity( + instanceId, + EntityId.FromString(instanceId).Name, + responseMessage.RequestInfo.Operation!, + signalEntity: false, + responseMessage.RequestInfo.ScheduledTime, + parentTraceContext, + startTime: responseMessage.RequestInfo.RequestTime); + + newActivity.SetSpanId(responseMessage.RequestInfo.ClientSpanId); + + return newActivity; + } + catch (Exception) + { + return null; + } + } } -} +} \ No newline at end of file From 87776bc77e3250df74157bc9bfceb14356825392 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 11 Apr 2025 01:07:49 -0700 Subject: [PATCH 18/60] fixing spacing --- src/DurableTask.Core/Tracing/TraceHelper.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 4fd15dec3..e3f75ea7a 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -58,6 +58,7 @@ public class TraceHelper newActivity.SetTag(Schema.Task.Version, startEvent.Version); } + startEvent.SetParentTraceContext(newActivity); } @@ -862,4 +863,4 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } -} \ No newline at end of file +} From 97cd8e6703cc97a1dd1af665cd3ebb26eb2943ae Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 11 Apr 2025 12:04:47 -0700 Subject: [PATCH 19/60] slight change in formatting of the create orchestration trace --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 1 + src/DurableTask.Core/Tracing/TraceHelper.cs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index a58cabd71..a3903f481 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1137,6 +1137,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( } using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( runtimeState.OrchestrationInstance!.InstanceId, + EntityId.FromString(runtimeState.OrchestrationInstance!.InstanceId).Name, createSubOrchestrationAction.InstanceId!, parentTraceContext, requestTime); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index e3f75ea7a..3e6400ed5 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -485,10 +485,10 @@ internal static void EmitTraceActivityForTimer( return newActivity; } - internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset startTime) + internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset startTime) { Activity? newActivity = ActivityTraceSource.StartActivity( - CreateSpanName(TraceActivityConstants.Entity, TraceActivityConstants.CreateOrchestration, null), + CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null), kind: ActivityKind.Producer, parentContext: parentTraceContext, startTime: startTime); From 9ed4bf6f0a77716140545aedea04da6a54d9e1de Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 11 Apr 2025 12:34:29 -0700 Subject: [PATCH 20/60] aligning the isolated case with the new in-process changes --- .../Entities/EventFormat/RequestMessage.cs | 8 +- .../Entities/EventFormat/ResponseMessage.cs | 24 +++- src/DurableTask.Core/OrchestrationTags.cs | 9 +- src/DurableTask.Core/TaskEntityDispatcher.cs | 13 +- .../TaskOrchestrationDispatcher.cs | 76 +++++------ src/DurableTask.Core/Tracing/Schema.cs | 2 +- .../Tracing/TraceActivityConstants.cs | 2 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 122 ++++++++++++------ 8 files changed, 158 insertions(+), 98 deletions(-) diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index b4dbf994c..85580b586 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -100,9 +100,9 @@ internal class RequestMessage : EntityMessage public bool IsLockRequest => LockSet != null; /// - /// Parent trace of this request message + /// Parent trace context of this request message. /// - [DataMember(Name = "parentTrace", EmitDefaultValue = false)] + [DataMember(Name = "parentTraceContext", EmitDefaultValue = false)] public DistributedTraceContext? ParentTraceContext { get; set; } /// @@ -115,7 +115,7 @@ internal class RequestMessage : EntityMessage /// The time the request was generated. /// [DataMember(Name = "requestTime", EmitDefaultValue = false)] - public DateTime? RequestTime { get; set; } + public DateTimeOffset? RequestTime { get; set; } /// public override string GetShortDescription() @@ -130,4 +130,4 @@ public override string GetShortDescription() } } } -} +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs index 8f78516b5..e86236792 100644 --- a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs @@ -13,6 +13,8 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { + using DurableTask.Core.Tracing; + using System; using System.Runtime.Serialization; [DataContract] @@ -32,6 +34,9 @@ internal class ResponseMessage : EntityMessage [IgnoreDataMember] public bool IsErrorResult => this.ErrorMessage != null || this.FailureDetails != null; + [DataMember(Name = "requestInfo", EmitDefaultValue = false)] + public RequestInformation? RequestInfo { get; set; } + public override string GetShortDescription() { if (this.IsErrorResult) @@ -47,5 +52,22 @@ public override string GetShortDescription() return $"[OperationSuccessful ({Result?.Length ?? 0} chars)]"; } } + internal class RequestInformation + { + [DataMember] + public string? Operation { get; set; } + + [DataMember] + public DateTime? ScheduledTime { get; set; } + + [DataMember] + public DateTimeOffset? RequestTime { get; set; } + + [DataMember] + public string? ClientSpanId { get; set; } + + [DataMember] + public DistributedTraceContext? ParentTraceContext { get; set; } + } } -} +} \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index 9b296b059..9b3f018c4 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -47,15 +47,10 @@ public static class OrchestrationTags public const string TraceState = "TraceState"; /// - /// The time that the request for a new orchestration was issued + /// The time the request for a new orchestration was created. /// public const string RequestTime = "RequestTime"; - /// - /// Whether or not to create a trace for the of the orchestration - /// - public const string CreateTraceForNewOrchestration = "CreateTrace"; - /// /// Check whether the given tags contain the fire and forget tag /// @@ -93,4 +88,4 @@ internal static IDictionary MergeTags( } } } -} +} \ No newline at end of file diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 7a8aaa1ba..ddb84ba11 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -764,11 +764,11 @@ void SendSignalMessage(WorkItemEffects effects, SchedulerState schedulerState, S destination.InstanceId, EntityId.FromString(destination.InstanceId).Name, action.Name, - true, + signalEntity: true, + action.ScheduledTime, parentTraceContext, - entityId: effects.InstanceId, - scheduledTime: action.ScheduledTime, - startTime: action.RequestTime) : null; + action.RequestTime, + entityId: effects.InstanceId) : null; if (traceActivity?.Id != null) { message.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); @@ -871,10 +871,11 @@ internal void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRunt // Otherwise, we will create an unlinked trace activity with no parent using Activity traceActivity = successfullyParsed ? TraceHelper.StartActivityForEntityStartingAnOrchestration( runtimeState.OrchestrationInstance.InstanceId, + EntityId.FromString(runtimeState.OrchestrationInstance.InstanceId).Name, destination.InstanceId, parentTraceContext, - scheduledTime: action.ScheduledStartTime, - startTime: action.RequestTime) : null; + action.RequestTime, + scheduledTime: action.ScheduledStartTime) : null; if (traceActivity?.Id != null) { executionStartedEvent.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 9f511a0a4..62df7aade 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -155,7 +155,7 @@ void EnsureExecutionStartedIsFirst(IList batch) { // Keep track of orchestrator generation changes, maybe update target position string executionId = message.OrchestrationInstance.ExecutionId; - if(previousExecutionId != executionId) + if (previousExecutionId != executionId) { // We want to re-position the ExecutionStarted event after the "right-most" // event with a non-null executionID that came before it. @@ -217,7 +217,7 @@ async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem) CorrelationTraceClient.Propagate( () => - { + { // Check if it is extended session. // TODO: Remove this code - it looks incorrect and dangerous isExtendedSession = this.concurrentSessionLock.Acquire(); @@ -305,7 +305,7 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work var isCompleted = false; var continuedAsNew = false; var isInterrupted = false; - + // correlation CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext); @@ -425,29 +425,12 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; - ActivityContext? parentTraceContext = traceActivity?.Context; - var spanId = ActivitySpanId.CreateRandom(); - if (createSubOrchestrationAction.Tags != null - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent) - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState)) - { - try - { - parentTraceContext = ActivityContext.Parse(traceParent, traceState); - spanId = parentTraceContext.Value.SpanId; - } - catch (Exception e) - { - TraceHelper.TraceException(TraceEventType.Error, "TaskOrchestrationDispatcher-CreateSubOrchestration-MalformedParentTrace", e); - } - } orchestratorMessages.Add( this.ProcessCreateSubOrchestrationInstanceDecision( createSubOrchestrationAction, runtimeState, this.IncludeParameters, - parentTraceContext, - spanId)); + traceActivity)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -635,7 +618,7 @@ await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( continuedAsNew ? null : timerMessages, continuedAsNewMessage, instanceState); - + if (workItem.RestoreOriginalRuntimeStateDuringCompletion) { workItem.OrchestrationRuntimeState = runtimeState; @@ -1100,8 +1083,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( CreateSubOrchestrationAction createSubOrchestrationAction, OrchestrationRuntimeState runtimeState, bool includeParameters, - ActivityContext? parentTraceContext, - ActivitySpanId spanId) + Activity? parentTraceActivity) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1114,7 +1096,8 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - historyEvent.ClientSpanId = spanId.ToString(); + ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); + historyEvent.ClientSpanId = clientSpanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1139,13 +1122,34 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - if (parentTraceContext is ActivityContext parentContext) + // If a trace parent was provided, then we will attempt to parse the parent trace context provided in the orchestration tags and use that as the + // parent trace context of the call to create a suborchestration rather than the current Activity's context. + if (createSubOrchestrationAction.Tags != null + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) { - ActivityContext activityContext = new ActivityContext( - parentContext.TraceId, - spanId, - parentContext.TraceFlags, - parentContext.TraceState); + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) + && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) + { + var requestTime = DateTimeOffset.UtcNow; + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.RequestTime, out string requestTimeString)) + { + DateTimeOffset.TryParse(requestTimeString, out requestTime); + } + using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( + runtimeState.OrchestrationInstance!.InstanceId, + EntityId.FromString(runtimeState.OrchestrationInstance!.InstanceId).Name, + createSubOrchestrationAction.InstanceId!, + parentTraceContext, + requestTime); + if (createOrchestrationActivity != null) + { + startedEvent.SetParentTraceContext(createOrchestrationActivity); + } + } + } + else if (parentTraceActivity != null) + { + ActivityContext activityContext = new ActivityContext(parentTraceActivity.TraceId, clientSpanId, parentTraceActivity.ActivityTraceFlags, parentTraceActivity.TraceStateString); startedEvent.SetParentTraceContext(activityContext); } @@ -1164,11 +1168,11 @@ TaskMessage ProcessSendEventDecision( { var historyEvent = new EventSentEvent(sendEventAction.Id) { - InstanceId = sendEventAction.Instance?.InstanceId, - Name = sendEventAction.EventName, - Input = sendEventAction.EventData + InstanceId = sendEventAction.Instance?.InstanceId, + Name = sendEventAction.EventName, + Input = sendEventAction.EventData }; - + runtimeState.AddEvent(historyEvent); EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, sendEventAction.EventData) @@ -1190,7 +1194,7 @@ TaskMessage ProcessSendEventDecision( Event = eventRaisedEvent }; } - + internal class NonBlockingCountdownLock { int available; diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 2317a8c79..8dff0303e 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -45,4 +45,4 @@ internal static class Entity internal const string ScheduledTime = "durabletask.entity.scheduled_time"; } } -} +} \ No newline at end of file diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index ef56f52d8..5caf06679 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -28,4 +28,4 @@ internal class TraceActivityConstants public const string CallEntity = "call_entity"; public const string SignalEntity = "signal_entity"; } -} +} \ No newline at end of file diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index d5089d3f7..52ee8f2f2 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -19,12 +19,12 @@ namespace DurableTask.Core.Tracing using System.Globalization; using System.Runtime.ExceptionServices; using DurableTask.Core.Common; - using DurableTask.Core.Entities.OperationFormat; using DurableTask.Core.History; - using System.Linq; using Newtonsoft.Json; using DurableTask.Core.Entities.EventFormat; using DurableTask.Core.Entities; + using DurableTask.Core.Entities.OperationFormat; + using System.Linq; /// /// Helper class for logging/tracing @@ -44,19 +44,9 @@ public class TraceHelper /// internal static Activity? StartActivityForNewOrchestration(ExecutionStartedEvent startEvent) { - startEvent.TryGetParentTraceContext(out ActivityContext activityContext); - DateTime? startTime = null; - if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.RequestTime) && - DateTime.TryParse(startEvent.Tags[OrchestrationTags.RequestTime], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTime requestTime)) - { - startTime = requestTime; - } - Activity? newActivity = ActivityTraceSource.StartActivity( - CreateSpanName(TraceActivityConstants.CreateOrchestration, startEvent.Name, startEvent.Version), - kind: ActivityKind.Producer, - parentContext: activityContext, - startTime: startTime ?? DateTime.UtcNow); + name: CreateSpanName(TraceActivityConstants.CreateOrchestration, startEvent.Name, startEvent.Version), + kind: ActivityKind.Producer); if (newActivity != null) { @@ -70,6 +60,7 @@ public class TraceHelper newActivity.SetTag(Schema.Task.Version, startEvent.Version); } + startEvent.SetParentTraceContext(newActivity); } @@ -90,14 +81,6 @@ public class TraceHelper return null; } - if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.CreateTraceForNewOrchestration)) - { - startEvent.Tags.Remove(OrchestrationTags.CreateTraceForNewOrchestration); - // This immediately disposes of and ends the activity for the new orchestration. Note that in this case since the start time of activity is set to the time the - // the request for a new orchestration was made, but the Activity is only disposed of now, the duration of the Activity will be longer than if it was created and stopped immediately after the request creation. - using var activityForNewOrchestration = StartActivityForNewOrchestration(startEvent); - } - if (!startEvent.TryGetParentTraceContext(out ActivityContext activityContext)) { return null; @@ -381,10 +364,13 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( string? targetInstanceId) { // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty)) { - // In the case that this an event corresponding to an orchestrationg invoking an entity, we may want to create an entity-specific trace activity for the event - return TryParseEntityRequest(eventRaisedEvent, targetInstanceId); + return TryParseEntityRequest(eventRaisedEvent, targetInstanceId!); + } + else if (Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + { + return TryParseEntityResponse(eventRaisedEvent, instance?.InstanceId!); } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -423,7 +409,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( // There is a possibility that we mislabel the event as an entity event if entities are not enabled if (Entities.IsEntityInstance(instance.InstanceId)) { - return null; + return TryParseEntityRequest(eventRaised, instance.InstanceId); } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -471,13 +457,13 @@ internal static void EmitTraceActivityForTimer( } } - internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, ActivityContext? parentTraceContext, string? entityId = null, DateTime? scheduledTime = null, DateTime? startTime = null) + internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, DateTime? scheduledTime, ActivityContext parentTraceContext, DateTimeOffset? startTime, string? entityId = null) { Activity? newActivity = ActivityTraceSource.StartActivity( CreateEntitySpanName(entityName, operationName), kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client, - parentContext: parentTraceContext ?? default, - startTime: startTime ?? DateTime.UtcNow); + parentContext: parentTraceContext, + startTime: startTime ?? DateTimeOffset.UtcNow); if (newActivity == null) { @@ -501,13 +487,13 @@ internal static void EmitTraceActivityForTimer( return newActivity; } - internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string targetInstanceId, ActivityContext? parentTraceContext, DateTime? scheduledTime = null, DateTime? startTime = null) + internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset? startTime, DateTime? scheduledTime = null) { Activity? newActivity = ActivityTraceSource.StartActivity( - CreateSpanName(TraceActivityConstants.Entity, TraceActivityConstants.CreateOrchestration, null), + CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null), kind: ActivityKind.Producer, - parentContext: parentTraceContext ?? default, - startTime: startTime ?? DateTime.UtcNow); + parentContext: parentTraceContext, + startTime: startTime ?? default); if (newActivity == null) { @@ -853,13 +839,8 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } - private static Activity? TryParseEntityRequest(EventRaisedEvent raisedEvent, string? targetInstanceId) + private static Activity? TryParseEntityRequest(EventRaisedEvent raisedEvent, string targetInstanceId) { - if (string.IsNullOrEmpty(targetInstanceId)) - { - return null; - } - try { var requestMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); @@ -868,13 +849,36 @@ static void ExceptionHandlingWrapper(Action innerFunc) return null; } + var parentTraceContext = Activity.Current?.Context; + if (requestMessage.ParentTraceContext != null) + { + if (ActivityContext.TryParse(requestMessage.ParentTraceContext.TraceParent, requestMessage.ParentTraceContext.TraceState, out ActivityContext activityContext)) + { + parentTraceContext = activityContext; + } + // If a parent trace context was passed with the request message, this should be the parent of the current Activity, so if we cannot parse it we should not create the Activity + // or else it will be incorrectly linked. + else + { + parentTraceContext = null; + } + } + + // We only want to create a trace activity for calling/signaling an entity in the case that we can successfully get the parent trace context of the request. + // Otherwise, we will create an unlinked trace activity with no parent. + if (parentTraceContext == null) + { + return null; + } + Activity? newActivity = StartActivityForCallingOrSignalingEntity( targetInstanceId!, EntityId.FromString(targetInstanceId!).Name, requestMessage.Operation!, requestMessage.IsSignal, - Activity.Current?.Context, - scheduledTime: requestMessage.ScheduledTime, + requestMessage.ScheduledTime, + parentTraceContext.Value, + entityId: requestMessage.ParentInstanceId != null && Entities.IsEntityInstance(requestMessage.ParentInstanceId) ? requestMessage.ParentInstanceId : null, startTime: requestMessage.RequestTime); if (!string.IsNullOrEmpty(newActivity?.Id)) @@ -890,5 +894,39 @@ static void ExceptionHandlingWrapper(Action innerFunc) return null; } } + + private static Activity? TryParseEntityResponse(EventRaisedEvent raisedEvent, string instanceId) + { + try + { + var responseMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); + if (responseMessage == null || responseMessage.RequestInfo == null) + { + return null; + } + + if (!ActivityContext.TryParse(responseMessage.RequestInfo.ParentTraceContext?.TraceParent, responseMessage.RequestInfo.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) + { + return null; + } + + Activity? newActivity = StartActivityForCallingOrSignalingEntity( + instanceId, + EntityId.FromString(instanceId).Name, + responseMessage.RequestInfo.Operation!, + signalEntity: false, + responseMessage.RequestInfo.ScheduledTime, + parentTraceContext, + startTime: responseMessage.RequestInfo.RequestTime); + + newActivity.SetSpanId(responseMessage.RequestInfo.ClientSpanId); + + return newActivity; + } + catch (Exception) + { + return null; + } + } } -} +} \ No newline at end of file From 864a06887ce2d8ad8cd4819e268dc908f7fa4e02 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 14 Apr 2025 00:18:28 -0700 Subject: [PATCH 21/60] pushing what i have so far --- .../Entities/ClientEntityHelpers.cs | 6 ++- .../Entities/EventFormat/RequestMessage.cs | 5 ++ src/DurableTask.Core/OrchestrationTags.cs | 5 ++ src/DurableTask.Core/TaskEntityDispatcher.cs | 49 +++++++++++++++++++ src/DurableTask.Core/Tracing/TraceHelper.cs | 22 ++++++++- 5 files changed, 83 insertions(+), 4 deletions(-) diff --git a/src/DurableTask.Core/Entities/ClientEntityHelpers.cs b/src/DurableTask.Core/Entities/ClientEntityHelpers.cs index 65c22c7a5..155eb1eca 100644 --- a/src/DurableTask.Core/Entities/ClientEntityHelpers.cs +++ b/src/DurableTask.Core/Entities/ClientEntityHelpers.cs @@ -35,8 +35,9 @@ public static class ClientEntityHelpers /// The time to schedule this signal, or null if not a scheduled signal /// The parent trace context for this operation. /// The time at which the request was made. + /// Whether to create a trace for this signal operation. /// The event to send. - public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string? input, (DateTime Original, DateTime Capped)? scheduledTimeUtc, DistributedTraceContext? parentTraceContext = null, DateTime? requestTime = null) + public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targetInstance, Guid requestId, string operationName, string? input, (DateTime Original, DateTime Capped)? scheduledTimeUtc, DistributedTraceContext? parentTraceContext = null, DateTimeOffset? requestTime = null, bool createTrace = false) { var request = new RequestMessage() { @@ -48,7 +49,8 @@ public static EntityMessageEvent EmitOperationSignal(OrchestrationInstance targe ScheduledTime = scheduledTimeUtc?.Original, Input = input, ParentTraceContext = parentTraceContext, - RequestTime = requestTime + RequestTime = requestTime, + CreateTrace = createTrace, }; var eventName = scheduledTimeUtc.HasValue diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index 85580b586..2907423af 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -117,6 +117,11 @@ internal class RequestMessage : EntityMessage [DataMember(Name = "requestTime", EmitDefaultValue = false)] public DateTimeOffset? RequestTime { get; set; } + /// + /// The client span ID of this request. + /// + public string? ClientSpanId { get; set; } + /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index 9b3f018c4..a3c459392 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -51,6 +51,11 @@ public static class OrchestrationTags /// public const string RequestTime = "RequestTime"; + /// + /// Whether or not to create a trace for the of the orchestration + /// + public const string CreateTraceForNewOrchestration = "CreateTrace"; + /// /// Check whether the given tags contain the fire and forget tag /// diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index ddb84ba11..14f4c203f 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -495,6 +495,38 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc throw new EntitySchedulerException("Failed to deserialize incoming request message - may be corrupted or wrong version.", exception); } + if (requestMessage.CreateTrace) + { + var parentTraceContext = Activity.Current?.Context; + if (requestMessage.ParentTraceContext != null) + { + if (ActivityContext.TryParse(requestMessage.ParentTraceContext.TraceParent, requestMessage.ParentTraceContext.TraceState, out ActivityContext activityContext)) + { + parentTraceContext = activityContext; + } + // If a parent trace context was passed with the request message, this should be the parent of the current Activity, so if we cannot parse it we should not create the Activity + // or else it will be incorrectly linked. + else + { + parentTraceContext = null; + } + } + + // We only want to create a trace activity for signaling an entity in the case that we can successfully get the parent trace context of the request. + // Otherwise, we will create an unlinked trace activity with no parent. + if (parentTraceContext != null) + { + using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( + instanceId, + EntityId.FromString(instanceId).Name, + requestMessage.Operation, + requestMessage.IsSignal, + requestMessage.ScheduledTime, + parentTraceContext.Value, + requestMessage.RequestTime); + } + } + IEnumerable deliverNow; if (requestMessage.ScheduledTime.HasValue) @@ -663,6 +695,15 @@ public void ToBeContinued(SchedulerState schedulerState) // Otherwise, we will create an unlinked trace activity with no parent if (successfullyParsed) { + if (!request.IsSignal) + { + var clientSpanId = ActivitySpanId.CreateRandom(); + + // In that case that we are processing a call request as a server, we want to generate a new span ID that will also be used by the Activity we create at the end corresponding to the client call request + // That way, this server Activity corresponding to processing the call request will be correctly linked as the child of the Activity for the client call request. + parentTraceContext = new ActivityContext(parentTraceContext.TraceId, clientSpanId, parentTraceContext.TraceFlags, parentTraceContext.TraceState); + request.ClientSpanId = clientSpanId.ToString(); + } traceActivity = TraceHelper.StartActivityForProcessingEntityInvocation( instanceId, EntityId.FromString(instanceId).Name, @@ -720,6 +761,14 @@ void SendResultMessage(WorkItemEffects effects, RequestMessage request, Operatio Result = result.Result, ErrorMessage = result.ErrorMessage, FailureDetails = result.FailureDetails, + RequestInfo = new ResponseMessage.RequestInformation() + { + Operation = request.Operation, + ScheduledTime = request.ScheduledTime, + RequestTime = request.RequestTime, + ClientSpanId = request.ClientSpanId, + ParentTraceContext = request.ParentTraceContext, + } }; this.ProcessSendEventMessage(effects, destination, EntityMessageEventNames.ResponseMessageEventName(request.Id), responseMessage); } diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 52ee8f2f2..8883d6048 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -44,9 +44,19 @@ public class TraceHelper /// internal static Activity? StartActivityForNewOrchestration(ExecutionStartedEvent startEvent) { + startEvent.TryGetParentTraceContext(out ActivityContext activityContext); + DateTimeOffset? startTime = null; + if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.RequestTime) && + DateTimeOffset.TryParse(startEvent.Tags[OrchestrationTags.RequestTime], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset requestTime)) + { + startTime = requestTime; + } + Activity? newActivity = ActivityTraceSource.StartActivity( - name: CreateSpanName(TraceActivityConstants.CreateOrchestration, startEvent.Name, startEvent.Version), - kind: ActivityKind.Producer); + CreateSpanName(TraceActivityConstants.CreateOrchestration, startEvent.Name, startEvent.Version), + kind: ActivityKind.Producer, + parentContext: activityContext, + startTime: startTime ?? DateTimeOffset.UtcNow); if (newActivity != null) { @@ -81,6 +91,14 @@ public class TraceHelper return null; } + if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.CreateTraceForNewOrchestration)) + { + startEvent.Tags.Remove(OrchestrationTags.CreateTraceForNewOrchestration); + // This immediately disposes of and ends the activity for the new orchestration. Note that in this case since the start time of activity is set to the time the + // the request for a new orchestration was made, but the Activity is only disposed of now, the duration of the Activity will be longer than if it was created and stopped immediately after the request creation. + using var activityForNewOrchestration = StartActivityForNewOrchestration(startEvent); + } + if (!startEvent.TryGetParentTraceContext(out ActivityContext activityContext)) { return null; From 8e1ff2ecb48afddbf6c7765e9e6e067ab1d5224c Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 14 Apr 2025 15:37:59 -0700 Subject: [PATCH 22/60] seems like everything is working and aligned with the new in-process changes --- .../Entities/OrchestrationEntityContext.cs | 2 +- src/DurableTask.Core/TaskEntityDispatcher.cs | 122 ++++++++---------- src/DurableTask.Core/Tracing/TraceHelper.cs | 21 ++- 3 files changed, 74 insertions(+), 71 deletions(-) diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs index 754bef343..4c764e06f 100644 --- a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs +++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs @@ -239,7 +239,7 @@ public EntityMessageEvent EmitRequestMessage( (DateTime Original, DateTime Capped)? scheduledTimeUtc, string? input, bool? createTrace, - DateTime? requestTime) + DateTimeOffset? requestTime) { this.CheckEntitySupport(); diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 14f4c203f..36be49824 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -495,35 +495,23 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc throw new EntitySchedulerException("Failed to deserialize incoming request message - may be corrupted or wrong version.", exception); } - if (requestMessage.CreateTrace) + // We only want to create a trace activity for signaling an entity in the case that we can successfully get the parent trace context of the request. + // Otherwise, we will create an unlinked trace activity with no parent. + // Note that if we create the trace activity for signaling an entity here, then its duration will be longer since its end time will be set to once we + // start processing the signal request rather than when the signal request is committed to storage. + if (requestMessage.CreateTrace && ActivityContext.TryParse(requestMessage.ParentTraceContext?.TraceParent, requestMessage.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) { - var parentTraceContext = Activity.Current?.Context; - if (requestMessage.ParentTraceContext != null) - { - if (ActivityContext.TryParse(requestMessage.ParentTraceContext.TraceParent, requestMessage.ParentTraceContext.TraceState, out ActivityContext activityContext)) - { - parentTraceContext = activityContext; - } - // If a parent trace context was passed with the request message, this should be the parent of the current Activity, so if we cannot parse it we should not create the Activity - // or else it will be incorrectly linked. - else - { - parentTraceContext = null; - } - } - - // We only want to create a trace activity for signaling an entity in the case that we can successfully get the parent trace context of the request. - // Otherwise, we will create an unlinked trace activity with no parent. - if (parentTraceContext != null) - { - using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( + using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( instanceId, EntityId.FromString(instanceId).Name, requestMessage.Operation, requestMessage.IsSignal, requestMessage.ScheduledTime, - parentTraceContext.Value, + parentTraceContext, requestMessage.RequestTime); + if (traceActivity?.Id != null) + { + requestMessage.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); } } @@ -689,11 +677,9 @@ public void ToBeContinued(SchedulerState schedulerState) var request = this.operationBatch[i]; Activity traceActivity = null; - bool successfullyParsed = ActivityContext.TryParse(request.ParentTraceContext?.TraceParent, request.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext); - // We only want to create a trace activity for processing the entity invocation in the case that we can successfully parse the trace context of the request that led to this entity invocation. // Otherwise, we will create an unlinked trace activity with no parent - if (successfullyParsed) + if (ActivityContext.TryParse(request.ParentTraceContext?.TraceParent, request.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) { if (!request.IsSignal) { @@ -751,6 +737,22 @@ public Queue RemoveDeferredWork(int index) void SendResultMessage(WorkItemEffects effects, RequestMessage request, OperationResult result) { + // We only want to create a trace activity for calling an entity in the case that we can successfully get the parent trace context of the request. + // Otherwise, we will create an unlinked trace activity with no parent. + // Note that we create the Activity once the result has been sent to capture the full length of calling the entity and receiving its response. + if (ActivityContext.TryParse(request.ParentTraceContext?.TraceParent, request.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) + { + using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( + effects.InstanceId, + EntityId.FromString(effects.InstanceId).Name, + request.Operation, + request.IsSignal, + request.ScheduledTime, + parentTraceContext, + request.RequestTime); + traceActivity?.SetSpanId(request.ClientSpanId); + } + var destination = new OrchestrationInstance() { InstanceId = request.ParentInstanceId, @@ -761,14 +763,6 @@ void SendResultMessage(WorkItemEffects effects, RequestMessage request, Operatio Result = result.Result, ErrorMessage = result.ErrorMessage, FailureDetails = result.FailureDetails, - RequestInfo = new ResponseMessage.RequestInformation() - { - Operation = request.Operation, - ScheduledTime = request.ScheduledTime, - RequestTime = request.RequestTime, - ClientSpanId = request.ClientSpanId, - ParentTraceContext = request.ParentTraceContext, - } }; this.ProcessSendEventMessage(effects, destination, EntityMessageEventNames.ResponseMessageEventName(request.Id), responseMessage); } @@ -802,25 +796,23 @@ void SendSignalMessage(WorkItemEffects effects, SchedulerState schedulerState, S schedulerState.MessageSorter.LabelOutgoingMessage(message, action.InstanceId, DateTime.UtcNow, this.entityBackendProperties.EntityMessageReorderWindow); } - bool successfullyParsed = ActivityContext.TryParse( - action.ParentTraceContext?.TraceParent, - action.ParentTraceContext?.TraceState, - out ActivityContext parentTraceContext); - // We only want to create a trace activity for signaling the entity in the case that we can successfully parse the parent trace context of the signal request. // Otherwise, we will create an unlinked trace activity with no parent - using Activity traceActivity = successfullyParsed ? TraceHelper.StartActivityForCallingOrSignalingEntity( - destination.InstanceId, - EntityId.FromString(destination.InstanceId).Name, - action.Name, - signalEntity: true, - action.ScheduledTime, - parentTraceContext, - action.RequestTime, - entityId: effects.InstanceId) : null; - if (traceActivity?.Id != null) - { - message.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); + if (ActivityContext.TryParse(action.ParentTraceContext?.TraceParent, action.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) + { + using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( + destination.InstanceId, + EntityId.FromString(destination.InstanceId).Name, + action.Name, + signalEntity: true, + action.ScheduledTime, + parentTraceContext, + action.RequestTime, + entityId: effects.InstanceId); + if (traceActivity?.Id != null) + { + message.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); + } } this.ProcessSendEventMessage(effects, destination, eventName, message); } @@ -911,23 +903,21 @@ internal void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRunt Version = action.Version, }; - bool successfullyParsed = ActivityContext.TryParse( - action.ParentTraceContext?.TraceParent, - action.ParentTraceContext?.TraceState, - out ActivityContext parentTraceContext); - // We only want to create a trace activity for an entity starting an orchestration in the case that we can successfully parse the parent trace context of the start orchestration request. - // Otherwise, we will create an unlinked trace activity with no parent - using Activity traceActivity = successfullyParsed ? TraceHelper.StartActivityForEntityStartingAnOrchestration( - runtimeState.OrchestrationInstance.InstanceId, - EntityId.FromString(runtimeState.OrchestrationInstance.InstanceId).Name, - destination.InstanceId, - parentTraceContext, - action.RequestTime, - scheduledTime: action.ScheduledStartTime) : null; - if (traceActivity?.Id != null) - { - executionStartedEvent.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); + // Otherwise, we will create an unlinked trace activity with no parent + if (ActivityContext.TryParse(action.ParentTraceContext?.TraceParent, action.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) + { + using var traceActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( + runtimeState.OrchestrationInstance.InstanceId, + EntityId.FromString(runtimeState.OrchestrationInstance.InstanceId).Name, + destination.InstanceId, + parentTraceContext, + action.RequestTime, + scheduledTime: action.ScheduledStartTime); + if (traceActivity?.Id != null) + { + executionStartedEvent.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); + } } this.logHelper.SchedulingOrchestration(executionStartedEvent); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 8883d6048..f7e3032f1 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -94,8 +94,8 @@ public class TraceHelper if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.CreateTraceForNewOrchestration)) { startEvent.Tags.Remove(OrchestrationTags.CreateTraceForNewOrchestration); - // This immediately disposes of and ends the activity for the new orchestration. Note that in this case since the start time of activity is set to the time the - // the request for a new orchestration was made, but the Activity is only disposed of now, the duration of the Activity will be longer than if it was created and stopped immediately after the request creation. + // Note that if we create the trace activity for starting a new orchestration here, then its duration will be longer since its end time will be set to once we + // start processing the orchestration rather than when the request for a new orchestration is committed to storage. using var activityForNewOrchestration = StartActivityForNewOrchestration(startEvent); } @@ -862,11 +862,23 @@ static void ExceptionHandlingWrapper(Action innerFunc) try { var requestMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); - if (requestMessage == null || !requestMessage.CreateTrace) + if (requestMessage == null) { return null; } + // This can be the case for an entity call request where we want to create the trace at the end after receiving the result of the call. + // We still want to attach the current activity context to the request so that it can be used when the trace is eventually created if there is not parent trace context attached already. + if (!requestMessage.CreateTrace) + { + if (requestMessage.ParentTraceContext == null && Activity.Current?.Id != null) + { + requestMessage.ParentTraceContext = new DistributedTraceContext(Activity.Current.Id, Activity.Current.TraceStateString); + raisedEvent.Input = JsonConvert.SerializeObject(requestMessage, Serializer.InternalSerializerSettings); + } + return null; + } + var parentTraceContext = Activity.Current?.Context; if (requestMessage.ParentTraceContext != null) { @@ -901,6 +913,7 @@ static void ExceptionHandlingWrapper(Action innerFunc) if (!string.IsNullOrEmpty(newActivity?.Id)) { + requestMessage.CreateTrace = false; requestMessage.ParentTraceContext = new DistributedTraceContext(newActivity!.Id!, newActivity.TraceStateString); raisedEvent.Input = JsonConvert.SerializeObject(requestMessage, Serializer.InternalSerializerSettings); } @@ -937,7 +950,7 @@ static void ExceptionHandlingWrapper(Action innerFunc) parentTraceContext, startTime: responseMessage.RequestInfo.RequestTime); - newActivity.SetSpanId(responseMessage.RequestInfo.ClientSpanId); + newActivity?.SetSpanId(responseMessage.RequestInfo.ClientSpanId); return newActivity; } From 900567e8eeec2b20e5a0f7034f93144a91f04453 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 14 Apr 2025 15:46:38 -0700 Subject: [PATCH 23/60] adding back new lines at the end of files --- src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs | 2 +- src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs | 2 +- .../Entities/OperationFormat/OperationRequest.cs | 1 + .../Entities/OperationFormat/SendSignalOperationAction.cs | 1 + .../OperationFormat/StartNewOrchestrationOperationAction.cs | 1 + src/DurableTask.Core/OrchestrationTags.cs | 2 +- src/DurableTask.Core/Tracing/Schema.cs | 2 +- src/DurableTask.Core/Tracing/TraceActivityConstants.cs | 2 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 9 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index 2907423af..ce355ab12 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -135,4 +135,4 @@ public override string GetShortDescription() } } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs index e86236792..eb06a76ea 100644 --- a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs @@ -70,4 +70,4 @@ internal class RequestInformation public DistributedTraceContext? ParentTraceContext { get; set; } } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs index bdeee2076..954e7e4d5 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs @@ -45,3 +45,4 @@ public class OperationRequest public DistributedTraceContext? TraceContext { get; set; } } } + diff --git a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs index 12833dcde..5283f3176 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs @@ -58,3 +58,4 @@ public class SendSignalOperationAction : OperationAction public DistributedTraceContext? ParentTraceContext { get; set; } } } + diff --git a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs index 35fedf81d..74e592633 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs @@ -65,3 +65,4 @@ public class StartNewOrchestrationOperationAction : OperationAction } } + diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index a3c459392..35e123bb3 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -93,4 +93,4 @@ internal static IDictionary MergeTags( } } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 8dff0303e..2317a8c79 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -45,4 +45,4 @@ internal static class Entity internal const string ScheduledTime = "durabletask.entity.scheduled_time"; } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index 5caf06679..ef56f52d8 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -28,4 +28,4 @@ internal class TraceActivityConstants public const string CallEntity = "call_entity"; public const string SignalEntity = "signal_entity"; } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index f7e3032f1..0d182ce2b 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -960,4 +960,4 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } -} \ No newline at end of file +} From f71e3004660ed9d5788a7c4902baa3db1e06b6f1 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 14 Apr 2025 15:50:40 -0700 Subject: [PATCH 24/60] missed some --- .../Entities/OperationFormat/OperationRequest.cs | 3 +-- .../Entities/OperationFormat/SendSignalOperationAction.cs | 3 +-- .../OperationFormat/StartNewOrchestrationOperationAction.cs | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs index 954e7e4d5..6b75c65c0 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationRequest.cs @@ -44,5 +44,4 @@ public class OperationRequest /// public DistributedTraceContext? TraceContext { get; set; } } -} - +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs index 5283f3176..eb5446d1e 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs @@ -57,5 +57,4 @@ public class SendSignalOperationAction : OperationAction /// public DistributedTraceContext? ParentTraceContext { get; set; } } -} - +} \ No newline at end of file diff --git a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs index 74e592633..d44040deb 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs @@ -64,5 +64,4 @@ public class StartNewOrchestrationOperationAction : OperationAction public DistributedTraceContext? ParentTraceContext { get; set; } } -} - +} \ No newline at end of file From d01179f9ae5c23c1c84e29ee32fb50131fa8161f Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 11:36:41 -0700 Subject: [PATCH 25/60] addressing PR comments --- .../Command/SendEventOrchestratorAction.cs | 7 ++ src/DurableTask.Core/EventTags.cs | 22 ++++++ .../History/EventRaisedEvent.cs | 8 ++- src/DurableTask.Core/OrchestrationContext.cs | 15 +++- src/DurableTask.Core/OrchestrationTags.cs | 6 +- src/DurableTask.Core/TaskHubClient.cs | 5 +- .../TaskOrchestrationContext.cs | 3 +- .../TaskOrchestrationDispatcher.cs | 68 +++++++++++-------- src/DurableTask.Core/Tracing/Schema.cs | 13 +--- src/DurableTask.Core/Tracing/TraceHelper.cs | 24 +++---- 10 files changed, 109 insertions(+), 62 deletions(-) create mode 100644 src/DurableTask.Core/EventTags.cs diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index fb922dc09..25f239f0d 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -11,6 +11,8 @@ // limitations under the License. // ---------------------------------------------------------------------------------- #nullable enable +using System.Collections.Generic; + namespace DurableTask.Core.Command { /// @@ -38,5 +40,10 @@ public class SendEventOrchestratorAction : OrchestratorAction /// The payload data of the external event. /// public string? EventData { get; set; } + + /// + /// The tags for the external event. + /// + public Dictionary? EventTags { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/EventTags.cs b/src/DurableTask.Core/EventTags.cs new file mode 100644 index 000000000..f0e348733 --- /dev/null +++ b/src/DurableTask.Core/EventTags.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DurableTask.Core +{ + /// + /// Tags to be used when sending external events. + /// + public static class EventTags + { + /// + /// Whether or not to create a trace for this entity request event + /// + public const string CreateEntityRequestEventTrace = "CreateSignalEntityEventTrace"; + + /// + /// Whether or not to create a trace for this entity response event + /// + public const string CreateEntityResponseEventTrace = "CreateEntityResponseEventTrace"; + } +} diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index 6da23f27c..e87227fba 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -14,6 +14,7 @@ namespace DurableTask.Core.History { using DurableTask.Core.Tracing; + using System.Collections.Generic; using System.Diagnostics; using System.Runtime.Serialization; @@ -52,9 +53,14 @@ public EventRaisedEvent(int eventId, string input) public string Input { get; set; } /// - /// The W3C trace context associated with this event. + /// Gets or sets the W3C trace context associated with this event. /// [DataMember] public DistributedTraceContext ParentTraceContext { get; set; } + + /// + /// Gets or sets any tags associated with the event + /// + public IDictionary Tags { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 4b2d8c7fe..6855bebad 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -385,7 +385,20 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// Instance in which to raise the event /// Name of the event /// Data for the event - public abstract void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData); + public void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + { + SendEvent(orchestrationInstance, eventName, eventData, null); + } + + /// + /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the + /// orchestration to fire. + /// + /// Instance in which to raise the event + /// Name of the event + /// Data for the event + /// Tags for the event + public abstract void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary eventTags); /// /// Checkpoint the orchestration instance by completing the current execution in the ContinueAsNew diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index f8c7492a2..3f680ca56 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -39,17 +39,17 @@ public static class OrchestrationTags /// /// The ID of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#traceparent-header) /// - public const string TraceParent = "TraceParent"; + public const string TraceParent = "MS_Entities_TraceParent"; /// /// The trace state of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#tracestate-header) /// - public const string TraceState = "TraceState"; + public const string TraceState = "MS_Entities_TraceState"; /// /// The time the request for a new orchestration was created. /// - public const string RequestTime = "RequestTime"; + public const string RequestTime = "MS_Entities_RequestTime"; /// /// Check whether the given tags contain the fire and forget tag diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index eb55d1936..f510796c6 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -745,7 +745,8 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) /// Instance in which to raise the event /// Name of the event /// Data for the event - public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + /// Tags for the event + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary? eventTags = null) { if (string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) @@ -756,7 +757,7 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s string serializedInput = this.defaultConverter.SerializeInternal(eventData); // Distributed Tracing - EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName }; + EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName, Tags = eventTags}; using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance); var taskMessage = new TaskMessage diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 8a48cbe93..bedb5c68f 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -194,7 +194,7 @@ async Task CreateSubOrchestrationInstanceCore( } } - public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary eventTags = null) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { @@ -211,6 +211,7 @@ public override void SendEvent(OrchestrationInstance orchestrationInstance, stri Instance = orchestrationInstance, EventName = eventName, EventData = serializedEventData, + EventTags = eventTags, }; this.orchestratorActionsMap.Add(id, action); diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index a3903f481..98341ca13 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1096,9 +1096,6 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); - historyEvent.ClientSpanId = clientSpanId.ToString(); - runtimeState.AddEvent(historyEvent); var taskMessage = new TaskMessage(); @@ -1122,33 +1119,11 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - // If a trace parent was provided, then we will attempt to parse the parent trace context provided in the orchestration tags and use that as the - // parent trace context of the call to create a suborchestration rather than the current Activity's context. - if (createSubOrchestrationAction.Tags != null - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) - { - if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) - && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) - { - var requestTime = DateTimeOffset.UtcNow; - if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.RequestTime, out string requestTimeString)) - { - DateTimeOffset.TryParse(requestTimeString, out requestTime); - } - using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( - runtimeState.OrchestrationInstance!.InstanceId, - EntityId.FromString(runtimeState.OrchestrationInstance!.InstanceId).Name, - createSubOrchestrationAction.InstanceId!, - parentTraceContext, - requestTime); - if (createOrchestrationActivity != null) - { - startedEvent.SetParentTraceContext(createOrchestrationActivity); - } - } - } - else if (parentTraceActivity != null) + // If this is an orchestration triggered by an entity, we will attempt to use the trace context provided in the CreateSubOrchestrationAction.Tags as the parent trace context rather than the current Activity. + if (!EntityTriggeredOrchestration(createSubOrchestrationAction, runtimeState, startedEvent) && parentTraceActivity != null) { + ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); + historyEvent.ClientSpanId = clientSpanId.ToString(); ActivityContext activityContext = new ActivityContext(parentTraceActivity.TraceId, clientSpanId, parentTraceActivity.ActivityTraceFlags, parentTraceActivity.TraceStateString); startedEvent.SetParentTraceContext(activityContext); } @@ -1177,7 +1152,8 @@ TaskMessage ProcessSendEventDecision( EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, sendEventAction.EventData) { - Name = sendEventAction.EventName + Name = sendEventAction.EventName, + Tags = sendEventAction.EventTags, }; // Distributed Tracing: start a new trace activity derived from the orchestration @@ -1194,6 +1170,38 @@ TaskMessage ProcessSendEventDecision( Event = eventRaisedEvent }; } + + private static bool EntityTriggeredOrchestration( + CreateSubOrchestrationAction createSubOrchestrationAction, + OrchestrationRuntimeState runtimeState, + ExecutionStartedEvent startedEvent) + { + if (createSubOrchestrationAction.Tags != null + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) + { + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) + && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) + { + var requestTime = DateTimeOffset.UtcNow; + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.RequestTime, out string requestTimeString)) + { + DateTimeOffset.TryParse(requestTimeString, out requestTime); + } + using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( + runtimeState.OrchestrationInstance!.InstanceId, + EntityId.FromString(runtimeState.OrchestrationInstance!.InstanceId).Name, + createSubOrchestrationAction.InstanceId!, + parentTraceContext, + requestTime); + if (createOrchestrationActivity != null) + { + startedEvent.SetParentTraceContext(createOrchestrationActivity); + } + } + return true; + } + return false; + } internal class NonBlockingCountdownLock { diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 2317a8c79..80872f52d 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -26,6 +26,8 @@ internal static class Task internal const string TaskId = "durabletask.task.task_id"; internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; internal const string FireAt = "durabletask.fire_at"; + internal const string Operation = "durabletask.task.operation"; + internal const string ScheduledTime = "durabletask.task.scheduled_time"; } internal static class Status @@ -33,16 +35,5 @@ internal static class Status internal const string Code = "otel.status_code"; internal const string Description = "otel.status_description"; } - - internal static class Entity - { - internal const string Type = Task.Type; - internal const string EntityId = "durabletask.entity.entity_id"; - internal const string TargetInstanceId = "durabletask.entity.target_instance_id"; - internal const string TargetEntityId = "durabletask.entity.target_entity_id"; - internal const string EntityOperation = "durabletask.entity.entity_operation"; - internal const string ErrorMessage = "durabletask.entity.error_message"; - internal const string ScheduledTime = "durabletask.entity.scheduled_time"; - } } } diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 3e6400ed5..345f6e287 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -361,12 +361,11 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { - // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty)) + if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) { return TryParseEntityRequest(eventRaisedEvent, targetInstanceId!); } - else if (Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + else if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityResponseEventTrace)) { return TryParseEntityResponse(eventRaisedEvent, instance?.InstanceId!); } @@ -404,8 +403,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(instance.InstanceId)) + if (eventRaised.Tags != null && eventRaised.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) { return TryParseEntityRequest(eventRaised, instance.InstanceId); } @@ -468,18 +466,18 @@ internal static void EmitTraceActivityForTimer( return null; } - newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Entity.EntityOperation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); - newActivity.SetTag(Schema.Entity.TargetEntityId, targetEntityId); + newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Task.Operation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); + newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetEntityId); if (!string.IsNullOrEmpty(entityId)) { - newActivity.SetTag(Schema.Entity.EntityId, entityId); + newActivity.SetTag(Schema.Task.InstanceId, entityId); } if (scheduledTime != null) { - newActivity.SetTag(Schema.Entity.ScheduledTime, scheduledTime.Value.ToString()); + newActivity.SetTag(Schema.Task.ScheduledTime, scheduledTime.Value.ToString()); } return newActivity; @@ -498,9 +496,9 @@ internal static void EmitTraceActivityForTimer( return null; } - newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Entity.TargetInstanceId, targetInstanceId); - newActivity.SetTag(Schema.Entity.EntityId, entityId); + newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetInstanceId); + newActivity.SetTag(Schema.Task.InstanceId, entityId); return newActivity; } From 30db2fb36466101bce22c4c923c87c1df9422938 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 11:38:13 -0700 Subject: [PATCH 26/60] adding back new lines at the ends of files --- src/DurableTask.Core/Command/SendEventOrchestratorAction.cs | 2 +- src/DurableTask.Core/History/EventRaisedEvent.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index 25f239f0d..f5e7d0af7 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -46,4 +46,4 @@ public class SendEventOrchestratorAction : OrchestratorAction /// public Dictionary? EventTags { get; set; } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index e87227fba..72ed160f3 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -63,4 +63,4 @@ public EventRaisedEvent(int eventId, string input) /// public IDictionary Tags { get; set; } } -} \ No newline at end of file +} From c4dc784786b6199daed7d46826b4463dc66cf1fa Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 11:49:18 -0700 Subject: [PATCH 27/60] trying to fix these line endings --- src/DurableTask.Core/Command/SendEventOrchestratorAction.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index f5e7d0af7..25f239f0d 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -46,4 +46,4 @@ public class SendEventOrchestratorAction : OrchestratorAction /// public Dictionary? EventTags { get; set; } } -} +} \ No newline at end of file From 41b0280cfc58d9b6248e3f66e0596cfd2f26a740 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 12:02:20 -0700 Subject: [PATCH 28/60] dealing with new lines again --- src/DurableTask.Core/History/EventRaisedEvent.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index 72ed160f3..e87227fba 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -63,4 +63,4 @@ public EventRaisedEvent(int eventId, string input) /// public IDictionary Tags { get; set; } } -} +} \ No newline at end of file From dcac8a8578c8faf2f212c780763addd1718508b0 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 12:27:45 -0700 Subject: [PATCH 29/60] tiny change --- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 345f6e287..8fe0bc182 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -459,7 +459,7 @@ internal static void EmitTraceActivityForTimer( CreateEntitySpanName(entityName, operationName), kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client, parentContext: parentTraceContext, - startTime: startTime ?? DateTimeOffset.UtcNow); + startTime: startTime ?? default); if (newActivity == null) { From 2bd49c751d092675b07170c38fc9e61fdac338c6 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 13:59:32 -0700 Subject: [PATCH 30/60] attempting to fix some whitespace issues --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index c406f71ad..6831f6196 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -155,7 +155,7 @@ void EnsureExecutionStartedIsFirst(IList batch) { // Keep track of orchestrator generation changes, maybe update target position string executionId = message.OrchestrationInstance.ExecutionId; - if (previousExecutionId != executionId) + if(previousExecutionId != executionId) { // We want to re-position the ExecutionStarted event after the "right-most" // event with a non-null executionID that came before it. @@ -217,7 +217,7 @@ async Task OnProcessWorkItemSessionAsync(TaskOrchestrationWorkItem workItem) CorrelationTraceClient.Propagate( () => - { + { // Check if it is extended session. // TODO: Remove this code - it looks incorrect and dangerous isExtendedSession = this.concurrentSessionLock.Acquire(); @@ -305,7 +305,7 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work var isCompleted = false; var continuedAsNew = false; var isInterrupted = false; - + // correlation CorrelationTraceClient.Propagate(() => CorrelationTraceContext.Current = workItem.TraceContext); @@ -618,7 +618,7 @@ await this.orchestrationService.CompleteTaskOrchestrationWorkItemAsync( continuedAsNew ? null : timerMessages, continuedAsNewMessage, instanceState); - + if (workItem.RestoreOriginalRuntimeStateDuringCompletion) { workItem.OrchestrationRuntimeState = runtimeState; @@ -1147,7 +1147,7 @@ TaskMessage ProcessSendEventDecision( Name = sendEventAction.EventName, Input = sendEventAction.EventData }; - + runtimeState.AddEvent(historyEvent); EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, sendEventAction.EventData) @@ -1202,7 +1202,7 @@ private static bool EntityTriggeredOrchestration( } return false; } - + internal class NonBlockingCountdownLock { int available; From d82f3a44df4eef90939d7e76a24b928727358549 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 14:00:03 -0700 Subject: [PATCH 31/60] attempting to fix some whitespace issues --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 6831f6196..98341ca13 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1143,9 +1143,9 @@ TaskMessage ProcessSendEventDecision( { var historyEvent = new EventSentEvent(sendEventAction.Id) { - InstanceId = sendEventAction.Instance?.InstanceId, - Name = sendEventAction.EventName, - Input = sendEventAction.EventData + InstanceId = sendEventAction.Instance?.InstanceId, + Name = sendEventAction.EventName, + Input = sendEventAction.EventData }; runtimeState.AddEvent(historyEvent); From 128970b053665619c16154df79ca51a0e1cd8b14 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 14:06:35 -0700 Subject: [PATCH 32/60] attempting to fix some whitespace issues --- src/DurableTask.Core/TaskEntityDispatcher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 36be49824..6dd414288 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -975,7 +975,7 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => } var result = await taskEntity.ExecuteOperationBatchAsync(request); - + dispatchContext.SetProperty(result); }); From 9a54fefd285ab18e2219eb193bcd00120003ca5b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 14:07:09 -0700 Subject: [PATCH 33/60] missed a null check --- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 8fe0bc182..b840395d8 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -851,7 +851,7 @@ static void ExceptionHandlingWrapper(Action innerFunc) parentTraceContext, startTime: responseMessage.RequestInfo.RequestTime); - newActivity.SetSpanId(responseMessage.RequestInfo.ClientSpanId); + newActivity?.SetSpanId(responseMessage.RequestInfo.ClientSpanId); return newActivity; } From 82bc84dceb059ef595423836c33aa85e354fca64 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 16:39:37 -0700 Subject: [PATCH 34/60] added a check for if we cannot successfully parse the parent trace context of the create orchestration action if one was provided --- src/DurableTask.Core/Tracing/TraceHelper.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 34dde6895..e9b775fd1 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -44,7 +44,11 @@ public class TraceHelper /// internal static Activity? StartActivityForNewOrchestration(ExecutionStartedEvent startEvent) { - startEvent.TryGetParentTraceContext(out ActivityContext activityContext); + if (!startEvent.TryGetParentTraceContext(out ActivityContext parentTraceContext) && startEvent.ParentTraceContext != null) + { + return null; + } + DateTimeOffset? startTime = null; if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.RequestTime) && DateTimeOffset.TryParse(startEvent.Tags[OrchestrationTags.RequestTime], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset requestTime)) @@ -55,8 +59,8 @@ public class TraceHelper Activity? newActivity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.CreateOrchestration, startEvent.Name, startEvent.Version), kind: ActivityKind.Producer, - parentContext: activityContext, - startTime: startTime ?? DateTimeOffset.UtcNow); + parentContext: parentTraceContext, + startTime: startTime ?? default); if (newActivity != null) { From 2be54cf9240b10dab0c452edd24f7e55ea6d9da4 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 17 Apr 2025 12:31:19 -0700 Subject: [PATCH 35/60] moving all trace activities into TaskEntityDispatcher to avoid any expensive deserializations in TraceHelper --- src/DurableTask.Core/Tracing/TraceHelper.cs | 118 +------------------- 1 file changed, 6 insertions(+), 112 deletions(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index e9b775fd1..93c9a3a98 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -385,13 +385,10 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { - if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) + // There is a possibility that we mislabel the event as an entity event if entities are not enabled + if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) { - return TryParseEntityRequest(eventRaisedEvent, targetInstanceId!); - } - else if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityResponseEventTrace)) - { - return TryParseEntityResponse(eventRaisedEvent, instance?.InstanceId!); + return null; } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -427,9 +424,10 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - if (eventRaised.Tags != null && eventRaised.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) + // There is a possibility that we mislabel the event as an entity event if entities are not enabled + if (Entities.IsEntityInstance(instance.InstanceId)) { - return TryParseEntityRequest(eventRaised, instance.InstanceId); + return null; } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -857,109 +855,5 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } - - private static Activity? TryParseEntityRequest(EventRaisedEvent raisedEvent, string targetInstanceId) - { - try - { - var requestMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); - - if (requestMessage == null) - { - return null; - } - - // This can be the case for an entity call request where we want to create the trace at the end after receiving the result of the call. - // We still want to attach the current activity context to the request so that it can be used when the trace is eventually created if there is not parent trace context attached already. - if (!requestMessage.CreateTrace) - { - if (requestMessage.ParentTraceContext == null && Activity.Current?.Id != null) - { - requestMessage.ParentTraceContext = new DistributedTraceContext(Activity.Current.Id, Activity.Current.TraceStateString); - raisedEvent.Input = JsonConvert.SerializeObject(requestMessage, Serializer.InternalSerializerSettings); - } - return null; - } - - var parentTraceContext = Activity.Current?.Context; - if (requestMessage.ParentTraceContext != null) - { - if (ActivityContext.TryParse(requestMessage.ParentTraceContext.TraceParent, requestMessage.ParentTraceContext.TraceState, out ActivityContext activityContext)) - { - parentTraceContext = activityContext; - } - // If a parent trace context was passed with the request message, this should be the parent of the current Activity, so if we cannot parse it we should not create the Activity - // or else it will be incorrectly linked. - else - { - parentTraceContext = null; - } - } - - // We only want to create a trace activity for calling/signaling an entity in the case that we can successfully get the parent trace context of the request. - // Otherwise, we will create an unlinked trace activity with no parent. - if (parentTraceContext == null) - { - return null; - } - - Activity? newActivity = StartActivityForCallingOrSignalingEntity( - targetInstanceId!, - EntityId.FromString(targetInstanceId!).Name, - requestMessage.Operation!, - requestMessage.IsSignal, - requestMessage.ScheduledTime, - parentTraceContext.Value, - entityId: requestMessage.ParentInstanceId != null && Entities.IsEntityInstance(requestMessage.ParentInstanceId) ? requestMessage.ParentInstanceId : null, - startTime: requestMessage.RequestTime); - - if (!string.IsNullOrEmpty(newActivity?.Id)) - { - requestMessage.CreateTrace = false; - requestMessage.ParentTraceContext = new DistributedTraceContext(newActivity!.Id!, newActivity.TraceStateString); - raisedEvent.Input = JsonConvert.SerializeObject(requestMessage, Serializer.InternalSerializerSettings); - } - - return newActivity; - } - catch (Exception) - { - return null; - } - } - - private static Activity? TryParseEntityResponse(EventRaisedEvent raisedEvent, string instanceId) - { - try - { - var responseMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); - if (responseMessage == null || responseMessage.RequestInfo == null) - { - return null; - } - - if (!ActivityContext.TryParse(responseMessage.RequestInfo.ParentTraceContext?.TraceParent, responseMessage.RequestInfo.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) - { - return null; - } - - Activity? newActivity = StartActivityForCallingOrSignalingEntity( - instanceId, - EntityId.FromString(instanceId).Name, - responseMessage.RequestInfo.Operation!, - signalEntity: false, - responseMessage.RequestInfo.ScheduledTime, - parentTraceContext, - startTime: responseMessage.RequestInfo.RequestTime); - - newActivity?.SetSpanId(responseMessage.RequestInfo.ClientSpanId); - - return newActivity; - } - catch (Exception) - { - return null; - } - } } } From 9aec0fc557a54895d7008394a5ea9fe0265dc637 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 11:57:11 -0700 Subject: [PATCH 36/60] reverting to old design --- .../Entities/EventFormat/RequestMessage.cs | 19 --- .../Entities/EventFormat/ResponseMessage.cs | 22 --- src/DurableTask.Core/TaskEntityDispatcher.cs | 8 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 144 ++---------------- 4 files changed, 16 insertions(+), 177 deletions(-) diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index d9ed3460d..e46d1759f 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -13,7 +13,6 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { - using DurableTask.Core.Tracing; using System; using System.Runtime.Serialization; @@ -99,24 +98,6 @@ internal class RequestMessage : EntityMessage [DataMember] public bool IsLockRequest => LockSet != null; - /// - /// Parent trace context of this request message. - /// - [DataMember(Name = "parentTraceContext", EmitDefaultValue = false)] - public DistributedTraceContext? ParentTraceContext { get; set; } - - /// - /// Whether or not to create an entity-specific trace for this request message - /// - [DataMember(Name = "createTrace")] - public bool CreateTrace { get; set; } - - /// - /// The time the request was generated. - /// - [DataMember(Name = "requestTime", EmitDefaultValue = false)] - public DateTimeOffset? RequestTime { get; set; } - /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs index eb06a76ea..8f78516b5 100644 --- a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs @@ -13,8 +13,6 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { - using DurableTask.Core.Tracing; - using System; using System.Runtime.Serialization; [DataContract] @@ -34,9 +32,6 @@ internal class ResponseMessage : EntityMessage [IgnoreDataMember] public bool IsErrorResult => this.ErrorMessage != null || this.FailureDetails != null; - [DataMember(Name = "requestInfo", EmitDefaultValue = false)] - public RequestInformation? RequestInfo { get; set; } - public override string GetShortDescription() { if (this.IsErrorResult) @@ -52,22 +47,5 @@ public override string GetShortDescription() return $"[OperationSuccessful ({Result?.Length ?? 0} chars)]"; } } - internal class RequestInformation - { - [DataMember] - public string? Operation { get; set; } - - [DataMember] - public DateTime? ScheduledTime { get; set; } - - [DataMember] - public DateTimeOffset? RequestTime { get; set; } - - [DataMember] - public string? ClientSpanId { get; set; } - - [DataMember] - public DistributedTraceContext? ParentTraceContext { get; set; } - } } } diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index aad78675d..8150b5a66 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -57,7 +57,7 @@ internal TaskEntityDispatcher( this.errorPropagationMode = errorPropagationMode; this.entityOrchestrationService = (orchestrationService as IEntityOrchestrationService)!; this.entityBackendProperties = entityOrchestrationService.EntityBackendProperties; - + this.dispatcher = new WorkItemDispatcher( "TaskEntityDispatcher", item => item == null ? string.Empty : item.InstanceId, @@ -379,7 +379,7 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work } OrchestrationState instanceState = (runtimeState.ExecutionStartedEvent != null) ? - instanceState = Utils.BuildOrchestrationState(runtimeState) : null; + instanceState = Utils.BuildOrchestrationState(runtimeState) : null; if (workItem.RestoreOriginalRuntimeStateDuringCompletion) { @@ -742,7 +742,7 @@ void SendLockResponseMessage(WorkItemEffects effects, OrchestrationInstance targ var message = new ResponseMessage() { // content is ignored by receiver but helps with tracing - Result = ResponseMessage.LockAcquisitionCompletion, + Result = ResponseMessage.LockAcquisitionCompletion, }; this.ProcessSendEventMessage(effects, target, EntityMessageEventNames.ResponseMessageEventName(requestId), message); } @@ -871,7 +871,7 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => } var result = await taskEntity.ExecuteOperationBatchAsync(request); - + dispatchContext.SetProperty(result); }); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index b840395d8..4818746b4 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -20,9 +20,6 @@ namespace DurableTask.Core.Tracing using System.Runtime.ExceptionServices; using DurableTask.Core.Common; using DurableTask.Core.History; - using Newtonsoft.Json; - using DurableTask.Core.Entities.EventFormat; - using DurableTask.Core.Entities; /// /// Helper class for logging/tracing @@ -361,13 +358,10 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { - if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) + // There is a possibility that we mislabel the event as an entity event if entities are not enabled + if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) { - return TryParseEntityRequest(eventRaisedEvent, targetInstanceId!); - } - else if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityResponseEventTrace)) - { - return TryParseEntityResponse(eventRaisedEvent, instance?.InstanceId!); + return null; } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -403,9 +397,10 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - if (eventRaised.Tags != null && eventRaised.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) + // There is a possibility that we mislabel the event as an entity event if entities are not enabled + if (Entities.IsEntityInstance(instance.InstanceId)) { - return TryParseEntityRequest(eventRaised, instance.InstanceId); + return null; } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -453,11 +448,11 @@ internal static void EmitTraceActivityForTimer( } } - internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, DateTime? scheduledTime, ActivityContext parentTraceContext, DateTimeOffset? startTime, string? entityId = null) + internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset? startTime, DateTime? scheduledTime = null) { Activity? newActivity = ActivityTraceSource.StartActivity( - CreateEntitySpanName(entityName, operationName), - kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client, + CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null), + kind: ActivityKind.Producer, parentContext: parentTraceContext, startTime: startTime ?? default); @@ -467,13 +462,8 @@ internal static void EmitTraceActivityForTimer( } newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Task.Operation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); - newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetEntityId); - - if (!string.IsNullOrEmpty(entityId)) - { - newActivity.SetTag(Schema.Task.InstanceId, entityId); - } + newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetInstanceId); + newActivity.SetTag(Schema.Task.InstanceId, entityId); if (scheduledTime != null) { @@ -483,26 +473,6 @@ internal static void EmitTraceActivityForTimer( return newActivity; } - internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset startTime) - { - Activity? newActivity = ActivityTraceSource.StartActivity( - CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null), - kind: ActivityKind.Producer, - parentContext: parentTraceContext, - startTime: startTime); - - if (newActivity == null) - { - return null; - } - - newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetInstanceId); - newActivity.SetTag(Schema.Task.InstanceId, entityId); - - return newActivity; - } - internal static void SetRuntimeStatusTag(string runtimeStatus) { DistributedTraceActivity.Current?.SetTag(Schema.Task.Status, runtimeStatus); @@ -770,95 +740,5 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } - - private static Activity? TryParseEntityRequest(EventRaisedEvent raisedEvent, string targetInstanceId) - { - try - { - var requestMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); - if (requestMessage == null || !requestMessage.CreateTrace) - { - return null; - } - - var parentTraceContext = Activity.Current?.Context; - if (requestMessage.ParentTraceContext != null) - { - if (ActivityContext.TryParse(requestMessage.ParentTraceContext.TraceParent, requestMessage.ParentTraceContext.TraceState, out ActivityContext activityContext)) - { - parentTraceContext = activityContext; - } - // If a parent trace context was passed with the request message, this should be the parent of the current Activity, so if we cannot parse it we should not create the Activity - // or else it will be incorrectly linked. - else - { - parentTraceContext = null; - } - } - - // We only want to create a trace activity for calling/signaling an entity in the case that we can successfully get the parent trace context of the request. - // Otherwise, we will create an unlinked trace activity with no parent. - if (parentTraceContext == null) - { - return null; - } - - Activity? newActivity = StartActivityForCallingOrSignalingEntity( - targetInstanceId!, - EntityId.FromString(targetInstanceId!).Name, - requestMessage.Operation!, - requestMessage.IsSignal, - requestMessage.ScheduledTime, - parentTraceContext.Value, - entityId: requestMessage.ParentInstanceId != null && Entities.IsEntityInstance(requestMessage.ParentInstanceId) ? requestMessage.ParentInstanceId : null, - startTime: requestMessage.RequestTime); - - if (!string.IsNullOrEmpty(newActivity?.Id)) - { - requestMessage.ParentTraceContext = new DistributedTraceContext(newActivity!.Id!, newActivity.TraceStateString); - raisedEvent.Input = JsonConvert.SerializeObject(requestMessage, Serializer.InternalSerializerSettings); - } - - return newActivity; - } - catch (Exception) - { - return null; - } - } - - private static Activity? TryParseEntityResponse(EventRaisedEvent raisedEvent, string instanceId) - { - try - { - var responseMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); - if (responseMessage == null || responseMessage.RequestInfo == null) - { - return null; - } - - if (!ActivityContext.TryParse(responseMessage.RequestInfo.ParentTraceContext?.TraceParent, responseMessage.RequestInfo.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) - { - return null; - } - - Activity? newActivity = StartActivityForCallingOrSignalingEntity( - instanceId, - EntityId.FromString(instanceId).Name, - responseMessage.RequestInfo.Operation!, - signalEntity: false, - responseMessage.RequestInfo.ScheduledTime, - parentTraceContext, - startTime: responseMessage.RequestInfo.RequestTime); - - newActivity?.SetSpanId(responseMessage.RequestInfo.ClientSpanId); - - return newActivity; - } - catch (Exception) - { - return null; - } - } } -} +} \ No newline at end of file From d8c10bb3b8ffd64162e0ad84fe18f2923edcd24e Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:03:21 -0700 Subject: [PATCH 37/60] missed some --- .../Command/SendEventOrchestratorAction.cs | 5 ----- src/DurableTask.Core/EventTags.cs | 22 ------------------- .../History/EventRaisedEvent.cs | 5 ----- src/DurableTask.Core/OrchestrationContext.cs | 15 +------------ src/DurableTask.Core/TaskHubClient.cs | 5 ++--- .../TaskOrchestrationContext.cs | 3 +-- src/DurableTask.Core/Tracing/Schema.cs | 2 -- .../Tracing/TraceActivityConstants.cs | 4 ---- 8 files changed, 4 insertions(+), 57 deletions(-) delete mode 100644 src/DurableTask.Core/EventTags.cs diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index 25f239f0d..1bf0a7a85 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -40,10 +40,5 @@ public class SendEventOrchestratorAction : OrchestratorAction /// The payload data of the external event. /// public string? EventData { get; set; } - - /// - /// The tags for the external event. - /// - public Dictionary? EventTags { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/EventTags.cs b/src/DurableTask.Core/EventTags.cs deleted file mode 100644 index f0e348733..000000000 --- a/src/DurableTask.Core/EventTags.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace DurableTask.Core -{ - /// - /// Tags to be used when sending external events. - /// - public static class EventTags - { - /// - /// Whether or not to create a trace for this entity request event - /// - public const string CreateEntityRequestEventTrace = "CreateSignalEntityEventTrace"; - - /// - /// Whether or not to create a trace for this entity response event - /// - public const string CreateEntityResponseEventTrace = "CreateEntityResponseEventTrace"; - } -} diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index e87227fba..74f90a22f 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -57,10 +57,5 @@ public EventRaisedEvent(int eventId, string input) /// [DataMember] public DistributedTraceContext ParentTraceContext { get; set; } - - /// - /// Gets or sets any tags associated with the event - /// - public IDictionary Tags { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 6855bebad..4b2d8c7fe 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -385,20 +385,7 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// Instance in which to raise the event /// Name of the event /// Data for the event - public void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) - { - SendEvent(orchestrationInstance, eventName, eventData, null); - } - - /// - /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the - /// orchestration to fire. - /// - /// Instance in which to raise the event - /// Name of the event - /// Data for the event - /// Tags for the event - public abstract void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary eventTags); + public abstract void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData); /// /// Checkpoint the orchestration instance by completing the current execution in the ContinueAsNew diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index f510796c6..eb55d1936 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -745,8 +745,7 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) /// Instance in which to raise the event /// Name of the event /// Data for the event - /// Tags for the event - public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary? eventTags = null) + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { if (string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) @@ -757,7 +756,7 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s string serializedInput = this.defaultConverter.SerializeInternal(eventData); // Distributed Tracing - EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName, Tags = eventTags}; + EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName }; using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance); var taskMessage = new TaskMessage diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index bedb5c68f..8a48cbe93 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -194,7 +194,7 @@ async Task CreateSubOrchestrationInstanceCore( } } - public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary eventTags = null) + public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { @@ -211,7 +211,6 @@ public override void SendEvent(OrchestrationInstance orchestrationInstance, stri Instance = orchestrationInstance, EventName = eventName, EventData = serializedEventData, - EventTags = eventTags, }; this.orchestratorActionsMap.Add(id, action); diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 80872f52d..8a91e3fb1 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -26,8 +26,6 @@ internal static class Task internal const string TaskId = "durabletask.task.task_id"; internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; internal const string FireAt = "durabletask.fire_at"; - internal const string Operation = "durabletask.task.operation"; - internal const string ScheduledTime = "durabletask.task.scheduled_time"; } internal static class Status diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index ef56f52d8..927d1bc93 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -20,12 +20,8 @@ internal class TraceActivityConstants public const string Activity = "activity"; public const string Event = "event"; public const string Timer = "timer"; - public const string Entity = "entity"; public const string CreateOrchestration = "create_orchestration"; public const string OrchestrationEvent = "orchestration_event"; - - public const string CallEntity = "call_entity"; - public const string SignalEntity = "signal_entity"; } } From 6b2bce10a564ef69e819bbea6e349ebf953106ea Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:04:37 -0700 Subject: [PATCH 38/60] and missed some more --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 1 - src/DurableTask.Core/Tracing/Schema.cs | 1 + src/DurableTask.Core/Tracing/TraceActivityConstants.cs | 1 + src/DurableTask.Core/Tracing/TraceHelper.cs | 5 ----- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 98341ca13..84611366a 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1153,7 +1153,6 @@ TaskMessage ProcessSendEventDecision( EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, sendEventAction.EventData) { Name = sendEventAction.EventName, - Tags = sendEventAction.EventTags, }; // Distributed Tracing: start a new trace activity derived from the orchestration diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 8a91e3fb1..c177aabab 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -26,6 +26,7 @@ internal static class Task internal const string TaskId = "durabletask.task.task_id"; internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; internal const string FireAt = "durabletask.fire_at"; + internal const string ScheduledTime = "durabletask.task.scheduled_time"; } internal static class Status diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index 927d1bc93..2c8a2e208 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -20,6 +20,7 @@ internal class TraceActivityConstants public const string Activity = "activity"; public const string Event = "event"; public const string Timer = "timer"; + public const string Entity = "entity"; public const string CreateOrchestration = "create_orchestration"; public const string OrchestrationEvent = "orchestration_event"; diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 4818746b4..b5c6c4e0c 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -500,11 +500,6 @@ static string CreateTimerSpanName(string orchestrationName) return $"{TraceActivityConstants.Orchestration}:{orchestrationName}:{TraceActivityConstants.Timer}"; } - static string CreateEntitySpanName(string entityName, string operationName) - { - return $"{TraceActivityConstants.Entity}:{entityName}:{operationName}"; - } - /// /// Simple trace with no iid or eid /// From 0b5687355d51400880058cf2ddcfa1d9a0a0ccb1 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:08:09 -0700 Subject: [PATCH 39/60] missed even more --- .../Command/SendEventOrchestratorAction.cs | 2 -- src/DurableTask.Core/History/EventRaisedEvent.cs | 3 +-- src/DurableTask.Core/TaskEntityDispatcher.cs | 8 ++++---- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index 1bf0a7a85..fb922dc09 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -11,8 +11,6 @@ // limitations under the License. // ---------------------------------------------------------------------------------- #nullable enable -using System.Collections.Generic; - namespace DurableTask.Core.Command { /// diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index 74f90a22f..6da23f27c 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -14,7 +14,6 @@ namespace DurableTask.Core.History { using DurableTask.Core.Tracing; - using System.Collections.Generic; using System.Diagnostics; using System.Runtime.Serialization; @@ -53,7 +52,7 @@ public EventRaisedEvent(int eventId, string input) public string Input { get; set; } /// - /// Gets or sets the W3C trace context associated with this event. + /// The W3C trace context associated with this event. /// [DataMember] public DistributedTraceContext ParentTraceContext { get; set; } diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 8150b5a66..8706d87af 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -57,7 +57,7 @@ internal TaskEntityDispatcher( this.errorPropagationMode = errorPropagationMode; this.entityOrchestrationService = (orchestrationService as IEntityOrchestrationService)!; this.entityBackendProperties = entityOrchestrationService.EntityBackendProperties; - + this.dispatcher = new WorkItemDispatcher( "TaskEntityDispatcher", item => item == null ? string.Empty : item.InstanceId, @@ -379,7 +379,7 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work } OrchestrationState instanceState = (runtimeState.ExecutionStartedEvent != null) ? - instanceState = Utils.BuildOrchestrationState(runtimeState) : null; + instanceState = Utils.BuildOrchestrationState(runtimeState) : null; if (workItem.RestoreOriginalRuntimeStateDuringCompletion) { @@ -742,7 +742,7 @@ void SendLockResponseMessage(WorkItemEffects effects, OrchestrationInstance targ var message = new ResponseMessage() { // content is ignored by receiver but helps with tracing - Result = ResponseMessage.LockAcquisitionCompletion, + Result = ResponseMessage.LockAcquisitionCompletion, }; this.ProcessSendEventMessage(effects, target, EntityMessageEventNames.ResponseMessageEventName(requestId), message); } @@ -871,7 +871,7 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => } var result = await taskEntity.ExecuteOperationBatchAsync(request); - + dispatchContext.SetProperty(result); }); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index b5c6c4e0c..3fbf9b574 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -736,4 +736,4 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } -} \ No newline at end of file +} From ac98d23d0ceab760b6b5383d89e270388c741864 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:10:19 -0700 Subject: [PATCH 40/60] will it ever end --- src/DurableTask.Core/TaskEntityDispatcher.cs | 2 +- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 2 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 8706d87af..aad78675d 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -57,7 +57,7 @@ internal TaskEntityDispatcher( this.errorPropagationMode = errorPropagationMode; this.entityOrchestrationService = (orchestrationService as IEntityOrchestrationService)!; this.entityBackendProperties = entityOrchestrationService.EntityBackendProperties; - + this.dispatcher = new WorkItemDispatcher( "TaskEntityDispatcher", item => item == null ? string.Empty : item.InstanceId, diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 84611366a..8bdf23172 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1152,7 +1152,7 @@ TaskMessage ProcessSendEventDecision( EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, sendEventAction.EventData) { - Name = sendEventAction.EventName, + Name = sendEventAction.EventName }; // Distributed Tracing: start a new trace activity derived from the orchestration diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 3fbf9b574..afe20d112 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -55,7 +55,6 @@ public class TraceHelper newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - startEvent.SetParentTraceContext(newActivity); } From 9ef781c8b46ed162f49761b021977b5acb14894b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:11:00 -0700 Subject: [PATCH 41/60] last one i think --- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index afe20d112..9372bbe03 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -54,7 +54,7 @@ public class TraceHelper { newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - + startEvent.SetParentTraceContext(newActivity); } From ee86dd2459ca5c21bc18f6f9438ef2bf21c22956 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 22 Apr 2025 16:23:30 -0700 Subject: [PATCH 42/60] moved activity for entity starting an orchestration back into webjobs --- src/DurableTask.Core/OrchestrationTags.cs | 5 --- .../TaskOrchestrationDispatcher.cs | 45 +++++-------------- 2 files changed, 12 insertions(+), 38 deletions(-) diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index 3f680ca56..c3065de46 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -46,11 +46,6 @@ public static class OrchestrationTags /// public const string TraceState = "MS_Entities_TraceState"; - /// - /// The time the request for a new orchestration was created. - /// - public const string RequestTime = "MS_Entities_RequestTime"; - /// /// Check whether the given tags contain the fire and forget tag /// diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 8bdf23172..183e01fe7 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1120,7 +1120,18 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( }; // If this is an orchestration triggered by an entity, we will attempt to use the trace context provided in the CreateSubOrchestrationAction.Tags as the parent trace context rather than the current Activity. - if (!EntityTriggeredOrchestration(createSubOrchestrationAction, runtimeState, startedEvent) && parentTraceActivity != null) + if (createSubOrchestrationAction.Tags != null + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) + { + // If a parent trace context was provided but we fail to parse it, we don't want to attach any parent trace context to the start event since that will incorrectly link the trace corresponding to the orchestration execution + // as a child of Activity.Current, which is not truly the parent of the request + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) + && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) + { + startedEvent.SetParentTraceContext(parentTraceContext); + } + } + else if (parentTraceActivity != null) { ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); historyEvent.ClientSpanId = clientSpanId.ToString(); @@ -1169,38 +1180,6 @@ TaskMessage ProcessSendEventDecision( Event = eventRaisedEvent }; } - - private static bool EntityTriggeredOrchestration( - CreateSubOrchestrationAction createSubOrchestrationAction, - OrchestrationRuntimeState runtimeState, - ExecutionStartedEvent startedEvent) - { - if (createSubOrchestrationAction.Tags != null - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) - { - if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) - && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) - { - var requestTime = DateTimeOffset.UtcNow; - if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.RequestTime, out string requestTimeString)) - { - DateTimeOffset.TryParse(requestTimeString, out requestTime); - } - using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( - runtimeState.OrchestrationInstance!.InstanceId, - EntityId.FromString(runtimeState.OrchestrationInstance!.InstanceId).Name, - createSubOrchestrationAction.InstanceId!, - parentTraceContext, - requestTime); - if (createOrchestrationActivity != null) - { - startedEvent.SetParentTraceContext(createOrchestrationActivity); - } - } - return true; - } - return false; - } internal class NonBlockingCountdownLock { From 2206569e102e49774d069c27341edc421be7ec46 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 22 Apr 2025 16:25:08 -0700 Subject: [PATCH 43/60] as always missed some stuff --- src/DurableTask.Core/Tracing/Schema.cs | 1 - .../Tracing/TraceActivityConstants.cs | 1 - src/DurableTask.Core/Tracing/TraceHelper.cs | 25 ------------------- 3 files changed, 27 deletions(-) diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index c177aabab..8a91e3fb1 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -26,7 +26,6 @@ internal static class Task internal const string TaskId = "durabletask.task.task_id"; internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; internal const string FireAt = "durabletask.fire_at"; - internal const string ScheduledTime = "durabletask.task.scheduled_time"; } internal static class Status diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index 2c8a2e208..927d1bc93 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -20,7 +20,6 @@ internal class TraceActivityConstants public const string Activity = "activity"; public const string Event = "event"; public const string Timer = "timer"; - public const string Entity = "entity"; public const string CreateOrchestration = "create_orchestration"; public const string OrchestrationEvent = "orchestration_event"; diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 9372bbe03..285d99437 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -447,31 +447,6 @@ internal static void EmitTraceActivityForTimer( } } - internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset? startTime, DateTime? scheduledTime = null) - { - Activity? newActivity = ActivityTraceSource.StartActivity( - CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null), - kind: ActivityKind.Producer, - parentContext: parentTraceContext, - startTime: startTime ?? default); - - if (newActivity == null) - { - return null; - } - - newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetInstanceId); - newActivity.SetTag(Schema.Task.InstanceId, entityId); - - if (scheduledTime != null) - { - newActivity.SetTag(Schema.Task.ScheduledTime, scheduledTime.Value.ToString()); - } - - return newActivity; - } - internal static void SetRuntimeStatusTag(string runtimeStatus) { DistributedTraceActivity.Current?.SetTag(Schema.Task.Status, runtimeStatus); From 5f9842cb91b5f4726a39a900e8d9abe4824a6482 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 22 Apr 2025 20:15:52 -0700 Subject: [PATCH 44/60] missed a spacing thing --- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 88aaeaae9..296c2c39f 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -73,7 +73,7 @@ public class TraceHelper { newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - + startEvent.SetParentTraceContext(newActivity); } From a22ba16adc9782ce1158d8a86b2c32f4d2e58cf1 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 23 Apr 2025 13:30:17 -0700 Subject: [PATCH 45/60] fixing a bug where we needed a parent trace context for orchestrations interacting with entities --- src/DurableTask.Core/TaskEntityDispatcher.cs | 51 ++++++++++++++----- .../TaskOrchestrationDispatcher.cs | 4 ++ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 6dd414288..7918019aa 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -495,23 +495,48 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc throw new EntitySchedulerException("Failed to deserialize incoming request message - may be corrupted or wrong version.", exception); } - // We only want to create a trace activity for signaling an entity in the case that we can successfully get the parent trace context of the request. - // Otherwise, we will create an unlinked trace activity with no parent. // Note that if we create the trace activity for signaling an entity here, then its duration will be longer since its end time will be set to once we // start processing the signal request rather than when the signal request is committed to storage. - if (requestMessage.CreateTrace && ActivityContext.TryParse(requestMessage.ParentTraceContext?.TraceParent, requestMessage.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) + if (requestMessage.CreateTrace) { - using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( - instanceId, - EntityId.FromString(instanceId).Name, - requestMessage.Operation, - requestMessage.IsSignal, - requestMessage.ScheduledTime, - parentTraceContext, - requestMessage.RequestTime); - if (traceActivity?.Id != null) + // In the case that we are calling an entity, we want to create the Activity once the result for the call is returned and so we do not create now + if (requestMessage.IsSignal) { - requestMessage.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); + var successfullyParsed = false; + ActivityContext parentTraceContext; + if (requestMessage.ParentTraceContext != null) + { + // If a parent trace context was provided but we fail to successfully parse it, we should not create the Activity even if the EventRaisedEvent has a parent trace context attached. + // Otherwise we will incorrectly link the created Activity to a context that is not truly its parent. + if (ActivityContext.TryParse(requestMessage.ParentTraceContext?.TraceParent, requestMessage.ParentTraceContext?.TraceState, out parentTraceContext)) + { + successfullyParsed = true; + } + } + else if (eventRaisedEvent.TryGetParentTraceContext(out parentTraceContext)) + { + successfullyParsed = true; + } + if (successfullyParsed) + { + using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( + instanceId, + EntityId.FromString(instanceId).Name, + requestMessage.Operation, + requestMessage.IsSignal, + requestMessage.ScheduledTime, + parentTraceContext, + requestMessage.RequestTime); + if (traceActivity?.Id != null) + { + requestMessage.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); + } + } + } + // We still want to attach a parent trace context to the request in the case of a call to an entity so that when we create the Activity for the call this information is available. + else if (requestMessage.ParentTraceContext == null && eventRaisedEvent.ParentTraceContext != null) + { + requestMessage.ParentTraceContext = eventRaisedEvent.ParentTraceContext; } } diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 183e01fe7..d1dc2fdee 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1165,6 +1165,10 @@ TaskMessage ProcessSendEventDecision( { Name = sendEventAction.EventName }; + if (Activity.Current != null) + { + eventRaisedEvent.SetParentTraceContext(Activity.Current.Context); + } // Distributed Tracing: start a new trace activity derived from the orchestration // for an EventRaisedEvent (external event) From ba758512745dd6b077e7855dd3777902c6144332 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 25 Apr 2025 14:28:52 -0700 Subject: [PATCH 46/60] added support for an entities enabled flag --- src/DurableTask.Core/TaskHubClient.cs | 5 +++-- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 7 ++++++- src/DurableTask.Core/Tracing/TraceHelper.cs | 11 ++++++----- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index eb55d1936..a75bf12a5 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -745,7 +745,8 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) /// Instance in which to raise the event /// Name of the event /// Data for the event - public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + /// Whether or not this event corresponds to an entity (false by default) + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, bool entityEvent = false) { if (string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) @@ -757,7 +758,7 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s // Distributed Tracing EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName }; - using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance); + using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance, entityEvent); var taskMessage = new TaskMessage { diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 183e01fe7..08e1f83df 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -112,6 +112,11 @@ public async Task StopAsync(bool forced) /// public bool IncludeParameters { get; set; } + /// + /// Gets or sets the flag for whether or not entities are enabled + /// + public bool EntitiesEnabled { get; set; } + /// /// Method to get the next work item to process within supplied timeout /// @@ -1168,7 +1173,7 @@ TaskMessage ProcessSendEventDecision( // Distributed Tracing: start a new trace activity derived from the orchestration // for an EventRaisedEvent (external event) - using Activity? traceActivity = TraceHelper.StartTraceActivityForEventRaisedFromWorker(eventRaisedEvent, runtimeState.OrchestrationInstance, sendEventAction.Instance?.InstanceId); + using Activity? traceActivity = TraceHelper.StartTraceActivityForEventRaisedFromWorker(eventRaisedEvent, runtimeState.OrchestrationInstance, this.EntitiesEnabled, sendEventAction.Instance?.InstanceId); this.logHelper.RaisingEvent(runtimeState.OrchestrationInstance!, historyEvent); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 285d99437..1e3dcd798 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -348,6 +348,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . + /// Whether or not entities are enabled, meaning this event could possibly correspond to entity /// The instance id of the orchestration that will receive the event. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. @@ -355,10 +356,10 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( internal static Activity? StartTraceActivityForEventRaisedFromWorker( EventRaisedEvent eventRaisedEvent, OrchestrationInstance? instance, + bool entitiesEnabled, string? targetInstanceId) { - // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + if (entitiesEnabled && (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty))) { return null; } @@ -391,13 +392,13 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . + /// Whether or not this event corresponds to an entity /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// - internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) + internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance, bool entityEvent) { - // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(instance.InstanceId)) + if (entityEvent) { return null; } From 14faf9036cf116b0e7ca2f863727ff71173c9de9 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 25 Apr 2025 15:10:33 -0700 Subject: [PATCH 47/60] added a start time to OperationResult --- .../Entities/OperationFormat/OperationResult.cs | 5 +++++ src/DurableTask.Core/Tracing/TraceHelper.cs | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs index 9e0a6a398..6ea787b88 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs @@ -49,6 +49,11 @@ public bool IsError /// public FailureDetails? FailureDetails { get; set; } + /// + /// The start time of the operation. + /// + public DateTime? StartTime { get; set; } + /// /// The completion time of the operation. /// diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 3dbdca98c..aa35d190f 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -561,6 +561,10 @@ internal static void EndActivitiesForProcessingEntityInvocation(List t { activity.SetTag(Schema.Task.ErrorMessage, result.ErrorMessage ?? result.FailureDetails!.ErrorMessage); } + if (result.StartTime is DateTime startTime) + { + activity.SetStartTime(startTime); + } if (result.EndTime is DateTime endTime) { activity.SetEndTime(endTime); From 92e5c268ac421c41fc15f08e3b07f126e00a129a Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 25 Apr 2025 15:54:39 -0700 Subject: [PATCH 48/60] added a null check for client span ID when creating the activity for scheduling a suborchestration --- src/DurableTask.Core/Tracing/TraceHelper.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 1e3dcd798..e35142e9e 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -284,7 +284,11 @@ internal static void EmitTraceActivityForTaskFailed( return null; } - activity.SetSpanId(createdEvent.ClientSpanId); + if (!string.IsNullOrEmpty(createdEvent.ClientSpanId)) + { + activity.SetSpanId(createdEvent.ClientSpanId); + + } activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); activity.SetTag(Schema.Task.Name, createdEvent.Name); From 13d56ad7424fb63c3aa79f1651c14405803acf3f Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 25 Apr 2025 15:58:31 -0700 Subject: [PATCH 49/60] getting rid of an extra line --- src/DurableTask.Core/Tracing/TraceHelper.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index e35142e9e..2c8eb17cb 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -287,7 +287,6 @@ internal static void EmitTraceActivityForTaskFailed( if (!string.IsNullOrEmpty(createdEvent.ClientSpanId)) { activity.SetSpanId(createdEvent.ClientSpanId); - } activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); From 9ad0099b75123f5fc77cc454833b2d7214f4f82c Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 28 Apr 2025 14:54:22 -0700 Subject: [PATCH 50/60] addressing PR comment --- src/DurableTask.Core/TaskHubClient.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index a75bf12a5..7c67fb27a 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -738,6 +738,18 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) CorrelationTraceClient.TrackRequestTelemetry(requestTraceContext); } + /// + /// Raises an event in the specified orchestration instance, which eventually causes the OnEvent() method in the + /// orchestration to fire. + /// + /// Instance in which to raise the event + /// Name of the event + /// Data for the event + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + { + await this.RaiseEventAsync(orchestrationInstance, eventName, eventData, entityEvent: false); + } + /// /// Raises an event in the specified orchestration instance, which eventually causes the OnEvent() method in the /// orchestration to fire. From e1725460bf6d696b738f5c0961d52405926162c0 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 6 May 2025 16:32:39 -0700 Subject: [PATCH 51/60] addressing a few PR comments --- src/DurableTask.Core/Tracing/TraceHelper.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 2c8eb17cb..078dd9e06 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -351,7 +351,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . - /// Whether or not entities are enabled, meaning this event could possibly correspond to entity + /// Whether or not entities are enabled, meaning this event could possibly correspond to entity. /// The instance id of the orchestration that will receive the event. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. @@ -362,6 +362,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( bool entitiesEnabled, string? targetInstanceId) { + // We don't want to emit tracing for external events when they are related to entities if (entitiesEnabled && (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty))) { return null; @@ -395,12 +396,13 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . - /// Whether or not this event corresponds to an entity + /// Whether or not this event corresponds to an entity. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance, bool entityEvent) { + // We don't want to emit tracing for external events when they are related to entities if (entityEvent) { return null; From 506203027ccce194dde483ed58e5f35af4aa619a Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 8 May 2025 14:30:03 -0700 Subject: [PATCH 52/60] addressing some PR comments, slightly changing the logic of the modified TaskHubClient method --- src/DurableTask.Core/TaskHubClient.cs | 16 ++++++++++++---- .../TaskOrchestrationDispatcher.cs | 2 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 9 +-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index 7c67fb27a..7cf2bc064 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -747,7 +747,7 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) /// Data for the event public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { - await this.RaiseEventAsync(orchestrationInstance, eventName, eventData, entityEvent: false); + await this.RaiseEventAsync(orchestrationInstance, eventName, eventData, emitTraceActivity: true); } /// @@ -757,8 +757,8 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s /// Instance in which to raise the event /// Name of the event /// Data for the event - /// Whether or not this event corresponds to an entity (false by default) - public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, bool entityEvent = false) + /// Whether or not to emit a trace activity for this event. + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, bool emitTraceActivity = true) { if (string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) @@ -770,7 +770,11 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s // Distributed Tracing EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName }; - using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance, entityEvent); + Activity? traceActivity = null; + if (emitTraceActivity) + { + traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance); + } var taskMessage = new TaskMessage { @@ -789,6 +793,10 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s TraceHelper.AddErrorDetailsToSpan(traceActivity, e); throw; } + finally + { + traceActivity?.Dispose(); + } } /// diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 59d42465e..dd1663e74 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1168,7 +1168,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - // If this is an orchestration triggered by an entity, we will attempt to use the trace context provided in the CreateSubOrchestrationAction.Tags as the parent trace context rather than the current Activity. + // If a parent trace context was provided via the CreateSubOrchestrationAction.Tags, we will use this as the parent trace context of the suborchestration execution Activity rather than Activity.Current.Context. if (createSubOrchestrationAction.Tags != null && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) { diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 078dd9e06..8df8c0fa4 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -396,18 +396,11 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . - /// Whether or not this event corresponds to an entity. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// - internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance, bool entityEvent) + internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - // We don't want to emit tracing for external events when they are related to entities - if (entityEvent) - { - return null; - } - Activity? newActivity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaised.Name, null), kind: ActivityKind.Producer, From a97b5fe66edaecbb284f3ab93134b9696eecdd07 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Sat, 10 May 2025 00:04:43 -0700 Subject: [PATCH 53/60] tiny style update --- src/DurableTask.Core/TaskEntityDispatcher.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 7918019aa..0289761eb 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -504,11 +504,11 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc { var successfullyParsed = false; ActivityContext parentTraceContext; - if (requestMessage.ParentTraceContext != null) + if (requestMessage.ParentTraceContext is { } parentContext) { // If a parent trace context was provided but we fail to successfully parse it, we should not create the Activity even if the EventRaisedEvent has a parent trace context attached. // Otherwise we will incorrectly link the created Activity to a context that is not truly its parent. - if (ActivityContext.TryParse(requestMessage.ParentTraceContext?.TraceParent, requestMessage.ParentTraceContext?.TraceState, out parentTraceContext)) + if (ActivityContext.TryParse(traceContext.TraceParent, traceContext.TraceState, out parentTraceContext)) { successfullyParsed = true; } From 0994ec5e6e7e4163b755fd1092a800c6d231607c Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Sat, 10 May 2025 00:06:36 -0700 Subject: [PATCH 54/60] missed a line --- src/DurableTask.Core/TaskEntityDispatcher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 0289761eb..a7434a08b 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -508,7 +508,7 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc { // If a parent trace context was provided but we fail to successfully parse it, we should not create the Activity even if the EventRaisedEvent has a parent trace context attached. // Otherwise we will incorrectly link the created Activity to a context that is not truly its parent. - if (ActivityContext.TryParse(traceContext.TraceParent, traceContext.TraceState, out parentTraceContext)) + if (ActivityContext.TryParse(parentContext.TraceParent, parentContext.TraceState, out parentTraceContext)) { successfullyParsed = true; } From 3ee6b03a3752c427214b56e8aeb6d04fd098990b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 12 May 2025 11:37:23 -0700 Subject: [PATCH 55/60] slight style update... decided not to unnecessarily check for a invalid ID in the case of a non-null Activity --- src/DurableTask.Core/TaskEntityDispatcher.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index a7434a08b..c598f42f3 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -527,7 +527,7 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc requestMessage.ScheduledTime, parentTraceContext, requestMessage.RequestTime); - if (traceActivity?.Id != null) + if (traceActivity != null) { requestMessage.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); } @@ -834,7 +834,7 @@ void SendSignalMessage(WorkItemEffects effects, SchedulerState schedulerState, S parentTraceContext, action.RequestTime, entityId: effects.InstanceId); - if (traceActivity?.Id != null) + if (traceActivity != null) { message.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); } @@ -939,7 +939,7 @@ internal void ProcessSendStartMessage(WorkItemEffects effects, OrchestrationRunt parentTraceContext, action.RequestTime, scheduledTime: action.ScheduledStartTime); - if (traceActivity?.Id != null) + if (traceActivity != null) { executionStartedEvent.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); } From b4054f5106f8b256066a1ad2493798356f64991f Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 12 May 2025 20:45:27 -0700 Subject: [PATCH 56/60] addressing PR comments --- .../Entities/OrchestrationEntityContext.cs | 8 +- src/DurableTask.Core/TaskEntityDispatcher.cs | 91 ++++++++++--------- src/DurableTask.Core/Tracing/TraceHelper.cs | 44 +++++++++ 3 files changed, 96 insertions(+), 47 deletions(-) diff --git a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs index 4c764e06f..8a84df162 100644 --- a/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs +++ b/src/DurableTask.Core/Entities/OrchestrationEntityContext.cs @@ -228,8 +228,8 @@ public IEnumerable EmitLockReleaseMessages() /// A unique identifier for this request. /// A time for which to schedule the delivery, or null if this is not a scheduled message /// The operation input - /// Whether or not to create an entity-specific trace for this event /// The time at which the request was made. + /// Whether or not to create an entity-specific trace for this event /// The event to send. public EntityMessageEvent EmitRequestMessage( OrchestrationInstance target, @@ -238,8 +238,8 @@ public EntityMessageEvent EmitRequestMessage( Guid operationId, (DateTime Original, DateTime Capped)? scheduledTimeUtc, string? input, - bool? createTrace, - DateTimeOffset? requestTime) + DateTimeOffset? requestTime = null, + bool createTrace = false) { this.CheckEntitySupport(); @@ -252,8 +252,8 @@ public EntityMessageEvent EmitRequestMessage( Operation = operationName, ScheduledTime = scheduledTimeUtc?.Original, Input = input, - CreateTrace = createTrace ?? false, RequestTime = requestTime, + CreateTrace = createTrace, }; this.AdjustOutgoingMessage(target.InstanceId, request, scheduledTimeUtc?.Capped, out string eventName); diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index c598f42f3..525585795 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -496,49 +496,8 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc } // Note that if we create the trace activity for signaling an entity here, then its duration will be longer since its end time will be set to once we - // start processing the signal request rather than when the signal request is committed to storage. - if (requestMessage.CreateTrace) - { - // In the case that we are calling an entity, we want to create the Activity once the result for the call is returned and so we do not create now - if (requestMessage.IsSignal) - { - var successfullyParsed = false; - ActivityContext parentTraceContext; - if (requestMessage.ParentTraceContext is { } parentContext) - { - // If a parent trace context was provided but we fail to successfully parse it, we should not create the Activity even if the EventRaisedEvent has a parent trace context attached. - // Otherwise we will incorrectly link the created Activity to a context that is not truly its parent. - if (ActivityContext.TryParse(parentContext.TraceParent, parentContext.TraceState, out parentTraceContext)) - { - successfullyParsed = true; - } - } - else if (eventRaisedEvent.TryGetParentTraceContext(out parentTraceContext)) - { - successfullyParsed = true; - } - if (successfullyParsed) - { - using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( - instanceId, - EntityId.FromString(instanceId).Name, - requestMessage.Operation, - requestMessage.IsSignal, - requestMessage.ScheduledTime, - parentTraceContext, - requestMessage.RequestTime); - if (traceActivity != null) - { - requestMessage.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); - } - } - } - // We still want to attach a parent trace context to the request in the case of a call to an entity so that when we create the Activity for the call this information is available. - else if (requestMessage.ParentTraceContext == null && eventRaisedEvent.ParentTraceContext != null) - { - requestMessage.ParentTraceContext = eventRaisedEvent.ParentTraceContext; - } - } + // start actually processing the signal request. + CreateTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId); IEnumerable deliverNow; @@ -1011,5 +970,51 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => return result; } + + private static void CreateTraceActivityForSignalingEntity(RequestMessage requestMessage, EventRaisedEvent eventRaisedEvent, string instanceId) + { + if (requestMessage.CreateTrace) + { + // In the case that we are calling an entity, we want to create the Activity once the result for the call is returned and so we do not create now + if (requestMessage.IsSignal) + { + var successfullyParsed = false; + ActivityContext parentTraceContext; + if (requestMessage.ParentTraceContext is { } parentContext) + { + // If a parent trace context was provided but we fail to successfully parse it, we should not create the Activity even if the EventRaisedEvent has a parent trace context attached. + // Otherwise we will incorrectly link the created Activity to a context that is not truly its parent. + if (ActivityContext.TryParse(parentContext.TraceParent, parentContext.TraceState, out parentTraceContext)) + { + successfullyParsed = true; + } + } + else if (eventRaisedEvent.TryGetParentTraceContext(out parentTraceContext)) + { + successfullyParsed = true; + } + if (successfullyParsed) + { + using var traceActivity = TraceHelper.StartActivityForCallingOrSignalingEntity( + instanceId, + EntityId.FromString(instanceId).Name, + requestMessage.Operation, + requestMessage.IsSignal, + requestMessage.ScheduledTime, + parentTraceContext, + requestMessage.RequestTime); + if (traceActivity != null) + { + requestMessage.ParentTraceContext = new DistributedTraceContext(traceActivity.Id, traceActivity.TraceStateString); + } + } + } + // We still want to attach a parent trace context to the request in the case of a call to an entity so that when we create the Activity for the call this information is available. + else if (requestMessage.ParentTraceContext == null && eventRaisedEvent.ParentTraceContext != null) + { + requestMessage.ParentTraceContext = eventRaisedEvent.ParentTraceContext; + } + } + } } } diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 995877624..9fd30b997 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -473,6 +473,20 @@ internal static void EmitTraceActivityForTimer( } } + /// + /// Starts a new trace activity for calling or signaling an entity. + /// + /// The instance ID of the entity being called or signaled + /// The entity name + /// The operation name + /// Whether or not this is a signal request (as opposed to a call request) + /// The scheduled time of the request + /// The trace context of the parent that is calling or signaling the entity + /// The start time of the Activity, which is the time the request to the entity was generated + /// In the case that this is an entity signaling another entity, the instance ID of the signaling entity + /// + /// Returns a newly started with entity-specific metadata. + /// internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, DateTime? scheduledTime, ActivityContext parentTraceContext, DateTimeOffset? startTime, string? entityId = null) { Activity? newActivity = ActivityTraceSource.StartActivity( @@ -503,6 +517,18 @@ internal static void EmitTraceActivityForTimer( return newActivity; } + /// + /// Starts a new trace Activity for an entity starting an orchestration. + /// + /// The instance ID of the entity starting the orchestration + /// The entity name + /// The instance ID of the orchestration being started + /// The trace context of the parent entity invocation that led to this start orchestration request + /// The start time of the Activity, which is the time the start orchestration request was generated + /// The scheduled time of the request + /// + /// Returns a newly started with entity and orchestration-specific metadata. + /// internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset? startTime, DateTime? scheduledTime = null) { Activity? newActivity = ActivityTraceSource.StartActivity( @@ -528,6 +554,17 @@ internal static void EmitTraceActivityForTimer( return newActivity; } + /// + /// Starts a new trace Activity for an entity processing a signal/call request. + /// + /// The instance ID of the entity being callled or signaled + /// The entity name + /// The name of the operation the entity is processing + /// Whether or not this is a signal request (as opposed to a call request) + /// The trace context of the parent signal/call request which led to this invocation + /// + /// Returns a newly started with entity-specific metadata. + /// internal static Activity? StartActivityForProcessingEntityInvocation(string entityId, string entityName, string operationName, bool signalEntity, ActivityContext? parentTraceContext) { Activity? newActivity = ActivityTraceSource.StartActivity( @@ -547,6 +584,13 @@ internal static void EmitTraceActivityForTimer( return newActivity; } + /// + /// Ends the activities for an entity processing a batch of signal/call requests. + /// + /// The batch of trace activities to end. + /// The results returned by the entity for the batch of that it processed. + /// The , if any were provided. This will be used to set the error message of all the Activities in the case that + /// the entity did not return results for all of the requests internal static void EndActivitiesForProcessingEntityInvocation(List traceActivities, List results, FailureDetails? batchFailureDetails) { if (results.Count == traceActivities.Count) From 8f4171bfc3a8cb8a594d5d89ec402f6992cd2109 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 14 May 2025 00:13:12 -0700 Subject: [PATCH 57/60] fix for the bug with the scheduled time - if the scheduled time for an orchestration signaling an entity is too short, then the message gets redelivered and a trace is created for each redelivery. we fixed this and only make the trace once --- src/DurableTask.Core/TaskEntityDispatcher.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 525585795..cfbfc85a9 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -495,10 +495,6 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc throw new EntitySchedulerException("Failed to deserialize incoming request message - may be corrupted or wrong version.", exception); } - // Note that if we create the trace activity for signaling an entity here, then its duration will be longer since its end time will be set to once we - // start actually processing the signal request. - CreateTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId); - IEnumerable deliverNow; if (requestMessage.ScheduledTime.HasValue) @@ -509,18 +505,28 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc // We handle this by rescheduling the message instead of processing it. deliverNow = Array.Empty(); batch.AddMessageToBeRescheduled(requestMessage); + + // We do not want to create the Activity for the request yet since it will be redelivered again later. In the case that the parent trace context was attached + // to the EventRaisedEvent and not the RequestMessage, we want to attach it to the RequestMessage such that when it is redelivered the parent trace context can be used + // to create the Activity for the request then. + if (requestMessage.ParentTraceContext == null && eventRaisedEvent.ParentTraceContext != null) + { + requestMessage.ParentTraceContext = eventRaisedEvent.ParentTraceContext; + } } else { // the message is scheduled to be delivered immediately. // There are no FIFO guarantees for scheduled messages, so we skip the message sorter. deliverNow = new RequestMessage[] { requestMessage }; + CreateTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId); } } else { // run this through the message sorter to help with reordering and duplicate filtering deliverNow = schedulerState.MessageSorter.ReceiveInOrder(requestMessage, this.entityBackendProperties.EntityMessageReorderWindow); + CreateTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId); } foreach (var message in deliverNow) From c22275b76fe33354383a2a573a8d36de687f2441 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 14 May 2025 15:10:07 -0700 Subject: [PATCH 58/60] tiny method name change --- src/DurableTask.Core/TaskEntityDispatcher.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index cfbfc85a9..fa677d2b8 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -519,14 +519,14 @@ void DetermineWork(OrchestrationRuntimeState runtimeState, out SchedulerState sc // the message is scheduled to be delivered immediately. // There are no FIFO guarantees for scheduled messages, so we skip the message sorter. deliverNow = new RequestMessage[] { requestMessage }; - CreateTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId); + StartTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId); } } else { // run this through the message sorter to help with reordering and duplicate filtering deliverNow = schedulerState.MessageSorter.ReceiveInOrder(requestMessage, this.entityBackendProperties.EntityMessageReorderWindow); - CreateTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId); + StartTraceActivityForSignalingEntity(requestMessage, eventRaisedEvent, instanceId); } foreach (var message in deliverNow) @@ -977,7 +977,7 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => return result; } - private static void CreateTraceActivityForSignalingEntity(RequestMessage requestMessage, EventRaisedEvent eventRaisedEvent, string instanceId) + private static void StartTraceActivityForSignalingEntity(RequestMessage requestMessage, EventRaisedEvent eventRaisedEvent, string instanceId) { if (requestMessage.CreateTrace) { From 0d30abf2dfc164700c715bf3c5a004521f5b427b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 14 May 2025 17:30:41 -0700 Subject: [PATCH 59/60] addressing PR comments --- src/DurableTask.Core/TaskEntityDispatcher.cs | 1 - src/DurableTask.Core/Tracing/TraceHelper.cs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index fa677d2b8..a91ae97e2 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -25,7 +25,6 @@ namespace DurableTask.Core using System; using System.Collections.Generic; using System.Diagnostics; - using System.Linq; using System.Threading; using System.Threading.Tasks; diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 9fd30b997..94d60c551 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -50,6 +50,7 @@ public class TraceHelper } DateTimeOffset? startTime = null; + // In the case that a request time for the start orchestration request is provided via ExecutionStartedEvent.Tags, we will use this as the start time of the Activity rather than the current time if (startEvent.Tags != null && startEvent.Tags.ContainsKey(OrchestrationTags.RequestTime) && DateTimeOffset.TryParse(startEvent.Tags[OrchestrationTags.RequestTime], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset requestTime)) { From 71d0f06d598bd7ed2f6953419710cd9ede749fe9 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 15 May 2025 21:28:37 -0700 Subject: [PATCH 60/60] changed startTime/endTime to startTimeUtc/endTimeUtc and made the request times DateTimeOffset --- .../Entities/OperationFormat/OperationResult.cs | 4 ++-- .../Entities/OperationFormat/SendSignalOperationAction.cs | 2 +- .../OperationFormat/StartNewOrchestrationOperationAction.cs | 2 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs index 6ea787b88..a4c97fa8b 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/OperationResult.cs @@ -52,11 +52,11 @@ public bool IsError /// /// The start time of the operation. /// - public DateTime? StartTime { get; set; } + public DateTime? StartTimeUtc { get; set; } /// /// The completion time of the operation. /// - public DateTime? EndTime { get; set; } + public DateTime? EndTimeUtc { get; set; } } } diff --git a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs index eb5446d1e..fe22cbf49 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/SendSignalOperationAction.cs @@ -50,7 +50,7 @@ public class SendSignalOperationAction : OperationAction /// /// The time the signal request was generated. /// - public DateTime? RequestTime { get; set; } + public DateTimeOffset? RequestTime { get; set; } /// /// The parent trace context for the signal, if any. diff --git a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs index d44040deb..29aca841b 100644 --- a/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs +++ b/src/DurableTask.Core/Entities/OperationFormat/StartNewOrchestrationOperationAction.cs @@ -56,7 +56,7 @@ public class StartNewOrchestrationOperationAction : OperationAction /// /// The time of the new orchestration request creation. /// - public DateTime? RequestTime { get; set; } + public DateTimeOffset? RequestTime { get; set; } /// /// The parent trace context for the operation, if any. diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 94d60c551..5f80784cd 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -604,11 +604,11 @@ internal static void EndActivitiesForProcessingEntityInvocation(List t { activity.SetTag(Schema.Task.ErrorMessage, result.ErrorMessage ?? result.FailureDetails!.ErrorMessage); } - if (result.StartTime is DateTime startTime) + if (result.StartTimeUtc is DateTime startTime) { activity.SetStartTime(startTime); } - if (result.EndTime is DateTime endTime) + if (result.EndTimeUtc is DateTime endTime) { activity.SetEndTime(endTime); }