From af21cb2dcb3b5739473cbfb211f28478cca639a1 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 6 Mar 2025 13:01:11 -0800 Subject: [PATCH 01/35] added distributed tracing for entities --- .../Command/CreateSubOrchestrationAction.cs | 21 ++++++++++++ src/DurableTask.Core/OrchestrationContext.cs | 18 ++++++++++ .../TaskOrchestrationContext.cs | 34 +++++++++++++++---- .../TaskOrchestrationDispatcher.cs | 16 ++++++--- src/DurableTask.Core/Tracing/TraceHelper.cs | 15 ++++++++ 5 files changed, 93 insertions(+), 11 deletions(-) diff --git a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs index 67a9e1013..388f73c84 100644 --- a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs +++ b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs @@ -14,6 +14,7 @@ namespace DurableTask.Core.Command { using System.Collections.Generic; + using System.Diagnostics; /// /// Orchestrator action for creating sub-orchestrations. @@ -50,5 +51,25 @@ public class CreateSubOrchestrationAction : OrchestratorAction /// Tags to be applied to the sub-orchestration. /// public IDictionary? Tags { get; set; } + + /// + /// Parent trace ID of the sub-orchestration. + /// + public string? ParentTraceId { get; set; } + + /// + /// Parent span ID of the sub-orchestration. + /// + public string? ParentSpanId { get; set; } + + /// + /// Parent trace flags of the sub-orchestration. + /// + public ActivityTraceFlags ParentTraceFlags { get; set; } + + /// + /// Parent trace state of the sub-orchestration. + /// + public string? ParentTraceState { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 4b2d8c7fe..5cbd22b3a 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -15,6 +15,7 @@ namespace DurableTask.Core { using System; using System.Collections.Generic; + using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Castle.DynamicProxy; @@ -377,6 +378,23 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, object input, IDictionary tags); + /// + /// Create a sub-orchestration of the specified name and version with the specific instance id + /// + /// Return Type of the TaskOrchestration.RunTask method + /// Name of the orchestration as specified by the ObjectCreator + /// Name of the orchestration as specified by the ObjectCreator + /// InstanceId of the sub-orchestration to create + /// Input for the TaskOrchestration.RunTask method + /// Dictionary of key/value tags associated with this instance + /// Trace ID of the parent instance creating this suborchestration + /// Span ID of the parent instance creating this suborchestration + /// Trace flags of the parent instance creating this suborchestration + /// Trace state of the parent instance creating this suborchestration + /// Task that represents the execution of the specified sub-orchestration + public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, + object input, IDictionary tags, string parentTraceId, string parentSpanId, ActivityTraceFlags parentTraceFlags, string parentTraceState); + /// /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 8a48cbe93..0e4083799 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -129,7 +129,7 @@ public override Task CreateSubOrchestrationInstance( string instanceId, object input) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null, null, null, ActivityTraceFlags.None, null); } public override Task CreateSubOrchestrationInstance( @@ -139,7 +139,7 @@ public override Task CreateSubOrchestrationInstance( object input, IDictionary tags) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, null, null, ActivityTraceFlags.None, null); } public override Task CreateSubOrchestrationInstance( @@ -147,7 +147,21 @@ public override Task CreateSubOrchestrationInstance( string version, object input) { - return CreateSubOrchestrationInstanceCore(name, version, null, input, null); + return CreateSubOrchestrationInstanceCore(name, version, null, input, null, null, null, ActivityTraceFlags.None, null); + } + + public override Task CreateSubOrchestrationInstance( + string name, + string version, + string instanceId, + object input, + IDictionary tags, + string parentTraceId, + string parentSpanId, + ActivityTraceFlags parentTraceFlags, + string parentTraceState) + { + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, parentTraceId, parentSpanId, parentTraceFlags, parentTraceState); } async Task CreateSubOrchestrationInstanceCore( @@ -155,7 +169,11 @@ async Task CreateSubOrchestrationInstanceCore( string version, string instanceId, object input, - IDictionary tags) + IDictionary tags, + string parentTraceId, + string parentSpanId, + ActivityTraceFlags parentTraceFlags, + string parentTraceState) { int id = this.idCounter++; string serializedInput = this.MessageDataConverter.SerializeInternal(input); @@ -173,9 +191,13 @@ async Task CreateSubOrchestrationInstanceCore( Name = name, Version = version, Input = serializedInput, - Tags = tags + Tags = tags, + ParentTraceId = parentTraceId, + ParentSpanId = parentSpanId, + ParentTraceFlags = parentTraceFlags, + ParentTraceState = parentTraceState }; - + this.orchestratorActionsMap.Add(id, action); if (OrchestrationTags.IsTaggedAsFireAndForget(tags)) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 61864a1a5..6892f8a56 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -430,7 +430,10 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work createSubOrchestrationAction, runtimeState, this.IncludeParameters, - traceActivity)); + createSubOrchestrationAction.ParentTraceId ?? traceActivity?.TraceId.ToString(), + createSubOrchestrationAction.ParentSpanId ?? traceActivity?.SpanId.ToString(), + createSubOrchestrationAction.ParentTraceId != null ? createSubOrchestrationAction.ParentTraceFlags : traceActivity?.ActivityTraceFlags, + createSubOrchestrationAction.ParentTraceState ?? traceActivity?.TraceStateString)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1083,7 +1086,10 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( CreateSubOrchestrationAction createSubOrchestrationAction, OrchestrationRuntimeState runtimeState, bool includeParameters, - Activity? parentTraceActivity) + string? parentTraceId, + string? parentSpanId, + ActivityTraceFlags? parentTraceFlags, + string? parentTraceState) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1096,7 +1102,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); + ActivitySpanId clientSpanId = parentSpanId != null ? ActivitySpanId.CreateFromString(parentSpanId.AsSpan()) : ActivitySpanId.CreateRandom(); historyEvent.ClientSpanId = clientSpanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1122,9 +1128,9 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - if (parentTraceActivity != null) + if (parentTraceId != null) { - ActivityContext activityContext = new ActivityContext(parentTraceActivity.TraceId, clientSpanId, parentTraceActivity.ActivityTraceFlags, parentTraceActivity.TraceStateString); + ActivityContext activityContext = new ActivityContext(ActivityTraceId.CreateFromString(parentTraceId.AsSpan()), clientSpanId, (ActivityTraceFlags)parentTraceFlags!, parentTraceState); startedEvent.SetParentTraceContext(activityContext); } diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 37f09c9d5..0e64ac001 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -357,6 +357,14 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { + + // If either the target or source of this event is an entity, we don't want to start a trace activity for it because entity tracing + // is handled separately in the WebJobs extension + if ((targetInstanceId != null && targetInstanceId.StartsWith("@")) || (instance != null && instance.InstanceId.StartsWith("@"))) + { + return null; + } + Activity? newActivity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaisedEvent.Name, null), kind: ActivityKind.Producer, @@ -390,6 +398,13 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { + // If either the target of this event is an entity, we don't want to start a trace activity for it because entity tracing + // is handled separately in the WebJobs extension + if (instance.InstanceId.StartsWith("@")) + { + return null; + } + Activity? newActivity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaised.Name, null), kind: ActivityKind.Producer, From 8846d7046b4bd2c6a49a023610ecd757049fb402 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 10 Mar 2025 13:10:16 -0700 Subject: [PATCH 02/35] change in how the distributed trace context is propagated in the case that an entity in WebJobs starts an orchestration --- .../Command/CreateSubOrchestrationAction.cs | 20 ++----------- src/DurableTask.Core/OrchestrationContext.cs | 8 ++--- .../TaskOrchestrationContext.cs | 23 +++++--------- .../TaskOrchestrationDispatcher.cs | 30 +++++++++---------- .../Tracing/DistributedTraceContext.cs | 12 ++++++++ 5 files changed, 40 insertions(+), 53 deletions(-) diff --git a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs index 388f73c84..fd118ece0 100644 --- a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs +++ b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs @@ -13,6 +13,7 @@ #nullable enable namespace DurableTask.Core.Command { + using DurableTask.Core.Tracing; using System.Collections.Generic; using System.Diagnostics; @@ -53,23 +54,8 @@ public class CreateSubOrchestrationAction : OrchestratorAction public IDictionary? Tags { get; set; } /// - /// Parent trace ID of the sub-orchestration. + /// Parent trace of the sub-orchestration. /// - public string? ParentTraceId { get; set; } - - /// - /// Parent span ID of the sub-orchestration. - /// - public string? ParentSpanId { get; set; } - - /// - /// Parent trace flags of the sub-orchestration. - /// - public ActivityTraceFlags ParentTraceFlags { get; set; } - - /// - /// Parent trace state of the sub-orchestration. - /// - public string? ParentTraceState { get; set; } + public DistributedTraceContext? ParentTraceContext { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 5cbd22b3a..7de1dcb58 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -21,6 +21,7 @@ namespace DurableTask.Core using Castle.DynamicProxy; using DurableTask.Core.Entities; using DurableTask.Core.Serializing; + using DurableTask.Core.Tracing; /// /// Context for an orchestration containing the instance, replay status, orchestration methods and proxy methods @@ -387,13 +388,10 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// InstanceId of the sub-orchestration to create /// Input for the TaskOrchestration.RunTask method /// Dictionary of key/value tags associated with this instance - /// Trace ID of the parent instance creating this suborchestration - /// Span ID of the parent instance creating this suborchestration - /// Trace flags of the parent instance creating this suborchestration - /// Trace state of the parent instance creating this suborchestration + /// Trace context of the parent instance creating this suborchestration /// Task that represents the execution of the specified sub-orchestration public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, - object input, IDictionary tags, string parentTraceId, string parentSpanId, ActivityTraceFlags parentTraceFlags, string parentTraceState); + object input, IDictionary tags, DistributedTraceContext parentTraceContext); /// diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 0e4083799..73cbda09c 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -129,7 +129,7 @@ public override Task CreateSubOrchestrationInstance( string instanceId, object input) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null, null, null, ActivityTraceFlags.None, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null, null); } public override Task CreateSubOrchestrationInstance( @@ -139,7 +139,7 @@ public override Task CreateSubOrchestrationInstance( object input, IDictionary tags) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, null, null, ActivityTraceFlags.None, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, null); } public override Task CreateSubOrchestrationInstance( @@ -147,7 +147,7 @@ public override Task CreateSubOrchestrationInstance( string version, object input) { - return CreateSubOrchestrationInstanceCore(name, version, null, input, null, null, null, ActivityTraceFlags.None, null); + return CreateSubOrchestrationInstanceCore(name, version, null, input, null, null); } public override Task CreateSubOrchestrationInstance( @@ -156,12 +156,9 @@ public override Task CreateSubOrchestrationInstance( string instanceId, object input, IDictionary tags, - string parentTraceId, - string parentSpanId, - ActivityTraceFlags parentTraceFlags, - string parentTraceState) + DistributedTraceContext parentTraceContext) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, parentTraceId, parentSpanId, parentTraceFlags, parentTraceState); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, parentTraceContext); } async Task CreateSubOrchestrationInstanceCore( @@ -170,10 +167,7 @@ async Task CreateSubOrchestrationInstanceCore( string instanceId, object input, IDictionary tags, - string parentTraceId, - string parentSpanId, - ActivityTraceFlags parentTraceFlags, - string parentTraceState) + DistributedTraceContext parentTraceContext) { int id = this.idCounter++; string serializedInput = this.MessageDataConverter.SerializeInternal(input); @@ -192,10 +186,7 @@ async Task CreateSubOrchestrationInstanceCore( Version = version, Input = serializedInput, Tags = tags, - ParentTraceId = parentTraceId, - ParentSpanId = parentSpanId, - ParentTraceFlags = parentTraceFlags, - ParentTraceState = parentTraceState + ParentTraceContext = parentTraceContext }; this.orchestratorActionsMap.Add(id, action); diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 6892f8a56..fd28c6d66 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -425,15 +425,18 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; + DistributedTraceContext? traceContext = null; + if (traceActivity != null) + { + traceContext = new DistributedTraceContext(traceActivity.TraceId.ToString(), traceActivity.TraceStateString); + traceContext.ParentTraceFlags = traceActivity.ActivityTraceFlags; + } orchestratorMessages.Add( - this.ProcessCreateSubOrchestrationInstanceDecision( - createSubOrchestrationAction, - runtimeState, - this.IncludeParameters, - createSubOrchestrationAction.ParentTraceId ?? traceActivity?.TraceId.ToString(), - createSubOrchestrationAction.ParentSpanId ?? traceActivity?.SpanId.ToString(), - createSubOrchestrationAction.ParentTraceId != null ? createSubOrchestrationAction.ParentTraceFlags : traceActivity?.ActivityTraceFlags, - createSubOrchestrationAction.ParentTraceState ?? traceActivity?.TraceStateString)); + this.ProcessCreateSubOrchestrationInstanceDecision( + createSubOrchestrationAction, + runtimeState, + this.IncludeParameters, + createSubOrchestrationAction.ParentTraceContext ?? traceContext)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1086,10 +1089,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( CreateSubOrchestrationAction createSubOrchestrationAction, OrchestrationRuntimeState runtimeState, bool includeParameters, - string? parentTraceId, - string? parentSpanId, - ActivityTraceFlags? parentTraceFlags, - string? parentTraceState) + DistributedTraceContext? parentTraceContext) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1102,7 +1102,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = parentSpanId != null ? ActivitySpanId.CreateFromString(parentSpanId.AsSpan()) : ActivitySpanId.CreateRandom(); + ActivitySpanId clientSpanId = parentTraceContext?.ParentSpanId != null ? ActivitySpanId.CreateFromString(parentTraceContext.ParentSpanId.AsSpan()) : ActivitySpanId.CreateRandom(); historyEvent.ClientSpanId = clientSpanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1128,9 +1128,9 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - if (parentTraceId != null) + if (parentTraceContext != null) { - ActivityContext activityContext = new ActivityContext(ActivityTraceId.CreateFromString(parentTraceId.AsSpan()), clientSpanId, (ActivityTraceFlags)parentTraceFlags!, parentTraceState); + ActivityContext activityContext = new ActivityContext(ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), clientSpanId, (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, parentTraceContext.TraceState); startedEvent.SetParentTraceContext(activityContext); } diff --git a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs index b69b1f5f6..39ce0c88e 100644 --- a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs +++ b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs @@ -14,6 +14,7 @@ namespace DurableTask.Core.Tracing { using System; + using System.Diagnostics; using System.Runtime.Serialization; /// @@ -80,5 +81,16 @@ public string? TraceState /// [DataMember] public DateTimeOffset? ActivityStartTime { get; set; } + + /// + /// The span ID of the parent trace + /// + [DataMember] + public string? ParentSpanId { get; set; } + + /// + /// The activity trace flags of the parent trace + /// + public ActivityTraceFlags? ParentTraceFlags { get; set; } } } From d3f25f3a8d4d1e9cae7348494c4d259e6f3890eb Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 11 Mar 2025 00:20:16 -0700 Subject: [PATCH 03/35] slight style update --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index fd28c6d66..302fb4df3 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1130,7 +1130,10 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( if (parentTraceContext != null) { - ActivityContext activityContext = new ActivityContext(ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), clientSpanId, (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, parentTraceContext.TraceState); + ActivityContext activityContext = new ActivityContext( + ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), + clientSpanId, (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, + parentTraceContext.TraceState); startedEvent.SetParentTraceContext(activityContext); } From ed72bf70edf46f9935d351c0091c921ea9223b86 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 11 Mar 2025 19:37:25 -0700 Subject: [PATCH 04/35] some more tiny style things --- src/DurableTask.Core/TaskOrchestrationContext.cs | 2 +- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 73cbda09c..656d852bd 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -188,7 +188,7 @@ async Task CreateSubOrchestrationInstanceCore( Tags = tags, ParentTraceContext = parentTraceContext }; - + this.orchestratorActionsMap.Add(id, action); if (OrchestrationTags.IsTaggedAsFireAndForget(tags)) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 302fb4df3..119f7f115 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1132,7 +1132,8 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( { ActivityContext activityContext = new ActivityContext( ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), - clientSpanId, (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, + clientSpanId, + (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, parentTraceContext.TraceState); startedEvent.SetParentTraceContext(activityContext); } From 1c271de54aed7ba589f863d961e90b44bd1c997b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 12 Mar 2025 12:24:19 -0700 Subject: [PATCH 05/35] addressing two more PR comments --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 8 ++++---- src/DurableTask.Core/Tracing/TraceHelper.cs | 6 ++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 119f7f115..eb506dc87 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -432,10 +432,10 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work traceContext.ParentTraceFlags = traceActivity.ActivityTraceFlags; } orchestratorMessages.Add( - this.ProcessCreateSubOrchestrationInstanceDecision( - createSubOrchestrationAction, - runtimeState, - this.IncludeParameters, + this.ProcessCreateSubOrchestrationInstanceDecision( + createSubOrchestrationAction, + runtimeState, + this.IncludeParameters, createSubOrchestrationAction.ParentTraceContext ?? traceContext)); break; case OrchestratorActionType.SendEvent: diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 0e64ac001..10231b7df 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -358,8 +358,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( string? targetInstanceId) { - // If either the target or source of this event is an entity, we don't want to start a trace activity for it because entity tracing - // is handled separately in the WebJobs extension + // We don't want to emit tracing for external events when they are related to entities if ((targetInstanceId != null && targetInstanceId.StartsWith("@")) || (instance != null && instance.InstanceId.StartsWith("@"))) { return null; @@ -398,8 +397,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - // If either the target of this event is an entity, we don't want to start a trace activity for it because entity tracing - // is handled separately in the WebJobs extension + // We don't want to emit tracing for external events when they are related to entities if (instance.InstanceId.StartsWith("@")) { return null; From ffe061ee6b4c67b47183c3bac71ed3e450985789 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Wed, 12 Mar 2025 17:44:28 -0700 Subject: [PATCH 06/35] forgot to add one annotation --- src/DurableTask.Core/Tracing/DistributedTraceContext.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs index 39ce0c88e..15062c919 100644 --- a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs +++ b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs @@ -91,6 +91,7 @@ public string? TraceState /// /// The activity trace flags of the parent trace /// + [DataMember] public ActivityTraceFlags? ParentTraceFlags { get; set; } } } From 7462c40c4ee821b98b6f1cc5d24d975e6d26b79f Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 13 Mar 2025 12:45:34 -0700 Subject: [PATCH 07/35] yet another small style change --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index eb506dc87..2634275a8 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -428,8 +428,12 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work DistributedTraceContext? traceContext = null; if (traceActivity != null) { - traceContext = new DistributedTraceContext(traceActivity.TraceId.ToString(), traceActivity.TraceStateString); - traceContext.ParentTraceFlags = traceActivity.ActivityTraceFlags; + traceContext = new DistributedTraceContext( + traceActivity.TraceId.ToString(), + traceActivity.TraceStateString) + { + ParentTraceFlags = traceActivity.ActivityTraceFlags + }; } orchestratorMessages.Add( this.ProcessCreateSubOrchestrationInstanceDecision( From 88718730fa556e30cf80be59e00ae79ae880e732 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 18 Mar 2025 11:32:04 -0700 Subject: [PATCH 08/35] addressing PR comments --- .../Command/CreateSubOrchestrationAction.cs | 5 --- src/DurableTask.Core/OrchestrationContext.cs | 15 ------- src/DurableTask.Core/OrchestrationTags.cs | 10 +++++ .../TaskOrchestrationContext.cs | 23 +++-------- .../TaskOrchestrationDispatcher.cs | 40 ++++++++++++------- 5 files changed, 40 insertions(+), 53 deletions(-) diff --git a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs index fd118ece0..7b55011f8 100644 --- a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs +++ b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs @@ -52,10 +52,5 @@ public class CreateSubOrchestrationAction : OrchestratorAction /// Tags to be applied to the sub-orchestration. /// public IDictionary? Tags { get; set; } - - /// - /// Parent trace of the sub-orchestration. - /// - public DistributedTraceContext? ParentTraceContext { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 7de1dcb58..dae4d592a 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -379,21 +379,6 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, object input, IDictionary tags); - /// - /// Create a sub-orchestration of the specified name and version with the specific instance id - /// - /// Return Type of the TaskOrchestration.RunTask method - /// Name of the orchestration as specified by the ObjectCreator - /// Name of the orchestration as specified by the ObjectCreator - /// InstanceId of the sub-orchestration to create - /// Input for the TaskOrchestration.RunTask method - /// Dictionary of key/value tags associated with this instance - /// Trace context of the parent instance creating this suborchestration - /// Task that represents the execution of the specified sub-orchestration - public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, - object input, IDictionary tags, DistributedTraceContext parentTraceContext); - - /// /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the /// orchestration to fire. diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index 960297143..974406418 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -36,6 +36,16 @@ public static class OrchestrationTags /// public const string FireAndForget = "FireAndForget"; + /// + /// The ID of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#traceparent-header) + /// + public const string TraceParent = "TraceParent"; + + /// + /// The trace state of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#tracestate-header) + /// + public const string TraceState = "TraceState"; + /// /// Check whether the given tags contain the fire and forget tag /// diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 656d852bd..8a48cbe93 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -129,7 +129,7 @@ public override Task CreateSubOrchestrationInstance( string instanceId, object input) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, null); } public override Task CreateSubOrchestrationInstance( @@ -139,7 +139,7 @@ public override Task CreateSubOrchestrationInstance( object input, IDictionary tags) { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, null); + return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags); } public override Task CreateSubOrchestrationInstance( @@ -147,18 +147,7 @@ public override Task CreateSubOrchestrationInstance( string version, object input) { - return CreateSubOrchestrationInstanceCore(name, version, null, input, null, null); - } - - public override Task CreateSubOrchestrationInstance( - string name, - string version, - string instanceId, - object input, - IDictionary tags, - DistributedTraceContext parentTraceContext) - { - return CreateSubOrchestrationInstanceCore(name, version, instanceId, input, tags, parentTraceContext); + return CreateSubOrchestrationInstanceCore(name, version, null, input, null); } async Task CreateSubOrchestrationInstanceCore( @@ -166,8 +155,7 @@ async Task CreateSubOrchestrationInstanceCore( string version, string instanceId, object input, - IDictionary tags, - DistributedTraceContext parentTraceContext) + IDictionary tags) { int id = this.idCounter++; string serializedInput = this.MessageDataConverter.SerializeInternal(input); @@ -185,8 +173,7 @@ async Task CreateSubOrchestrationInstanceCore( Name = name, Version = version, Input = serializedInput, - Tags = tags, - ParentTraceContext = parentTraceContext + Tags = tags }; this.orchestratorActionsMap.Add(id, action); diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 2634275a8..2d198d44c 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -425,22 +425,31 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; - DistributedTraceContext? traceContext = null; - if (traceActivity != null) + ActivityContext? parentTraceContext = traceActivity?.Context; + var createNewSpanId = true; + if (createSubOrchestrationAction.Tags != null + && createSubOrchestrationAction.Tags.ContainsKey(OrchestrationTags.TraceParent) + && createSubOrchestrationAction.Tags.ContainsKey(OrchestrationTags.TraceState)) { - traceContext = new DistributedTraceContext( - traceActivity.TraceId.ToString(), - traceActivity.TraceStateString) + try { - ParentTraceFlags = traceActivity.ActivityTraceFlags - }; + parentTraceContext = ActivityContext.Parse( + createSubOrchestrationAction.Tags[OrchestrationTags.TraceParent], + createSubOrchestrationAction.Tags[OrchestrationTags.TraceState]); + createNewSpanId = false; + } + catch (Exception e) + { + TraceHelper.TraceException(TraceEventType.Error, "TaskOrchestrationDispatcher-CreateSubOrchestration-MalformedParentTrace", e); + } } orchestratorMessages.Add( this.ProcessCreateSubOrchestrationInstanceDecision( createSubOrchestrationAction, runtimeState, this.IncludeParameters, - createSubOrchestrationAction.ParentTraceContext ?? traceContext)); + parentTraceContext, + createNewSpanId)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1093,7 +1102,8 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( CreateSubOrchestrationAction createSubOrchestrationAction, OrchestrationRuntimeState runtimeState, bool includeParameters, - DistributedTraceContext? parentTraceContext) + ActivityContext? parentTraceContext, + bool createNewSpanId) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1106,7 +1116,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = parentTraceContext?.ParentSpanId != null ? ActivitySpanId.CreateFromString(parentTraceContext.ParentSpanId.AsSpan()) : ActivitySpanId.CreateRandom(); + ActivitySpanId clientSpanId = createNewSpanId ? ActivitySpanId.CreateRandom() : ((ActivityContext)parentTraceContext!).SpanId; historyEvent.ClientSpanId = clientSpanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1132,13 +1142,13 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - if (parentTraceContext != null) + if (parentTraceContext is ActivityContext parentContext) { ActivityContext activityContext = new ActivityContext( - ActivityTraceId.CreateFromString(parentTraceContext.TraceParent.AsSpan()), - clientSpanId, - (ActivityTraceFlags)parentTraceContext.ParentTraceFlags!, - parentTraceContext.TraceState); + parentContext.TraceId, + clientSpanId, + parentContext.TraceFlags, + parentContext.TraceState); startedEvent.SetParentTraceContext(activityContext); } From d49c005ae5f745fd339dd1ab892d7cfc84fbb1f5 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 18 Mar 2025 11:33:42 -0700 Subject: [PATCH 09/35] forgot one file --- .../Tracing/DistributedTraceContext.cs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs index 15062c919..a6e3611f4 100644 --- a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs +++ b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs @@ -14,7 +14,6 @@ namespace DurableTask.Core.Tracing { using System; - using System.Diagnostics; using System.Runtime.Serialization; /// @@ -81,17 +80,5 @@ public string? TraceState /// [DataMember] public DateTimeOffset? ActivityStartTime { get; set; } - - /// - /// The span ID of the parent trace - /// - [DataMember] - public string? ParentSpanId { get; set; } - - /// - /// The activity trace flags of the parent trace - /// - [DataMember] - public ActivityTraceFlags? ParentTraceFlags { get; set; } } -} +} \ No newline at end of file From 717573743dc88eac394222c428600f6bee1cf8eb Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 25 Mar 2025 15:29:43 -0700 Subject: [PATCH 10/35] addressing more PR comments --- .../Command/CreateSubOrchestrationAction.cs | 2 -- .../Entities/EventFormat/RequestMessage.cs | 6 ++++++ src/DurableTask.Core/OrchestrationContext.cs | 3 +-- .../TaskOrchestrationDispatcher.cs | 21 ++++++++----------- .../Tracing/DistributedTraceContext.cs | 2 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 11 ++++------ 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs index 7b55011f8..67a9e1013 100644 --- a/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs +++ b/src/DurableTask.Core/Command/CreateSubOrchestrationAction.cs @@ -13,9 +13,7 @@ #nullable enable namespace DurableTask.Core.Command { - using DurableTask.Core.Tracing; using System.Collections.Generic; - using System.Diagnostics; /// /// Orchestrator action for creating sub-orchestrations. diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index e46d1759f..3043a0680 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -98,6 +98,12 @@ internal class RequestMessage : EntityMessage [DataMember] public bool IsLockRequest => LockSet != null; + /// + /// Whether this message already has a trace created for it + /// + [DataMember(Name = "traceCreated")] + public bool TraceCreated { get; set; } + /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index dae4d592a..4b2d8c7fe 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -15,13 +15,11 @@ namespace DurableTask.Core { using System; using System.Collections.Generic; - using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Castle.DynamicProxy; using DurableTask.Core.Entities; using DurableTask.Core.Serializing; - using DurableTask.Core.Tracing; /// /// Context for an orchestration containing the instance, replay status, orchestration methods and proxy methods @@ -379,6 +377,7 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve public abstract Task CreateSubOrchestrationInstance(string name, string version, string instanceId, object input, IDictionary tags); + /// /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the /// orchestration to fire. diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 2d198d44c..9f511a0a4 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -426,17 +426,15 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; ActivityContext? parentTraceContext = traceActivity?.Context; - var createNewSpanId = true; + var spanId = ActivitySpanId.CreateRandom(); if (createSubOrchestrationAction.Tags != null - && createSubOrchestrationAction.Tags.ContainsKey(OrchestrationTags.TraceParent) - && createSubOrchestrationAction.Tags.ContainsKey(OrchestrationTags.TraceState)) + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent) + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState)) { try { - parentTraceContext = ActivityContext.Parse( - createSubOrchestrationAction.Tags[OrchestrationTags.TraceParent], - createSubOrchestrationAction.Tags[OrchestrationTags.TraceState]); - createNewSpanId = false; + parentTraceContext = ActivityContext.Parse(traceParent, traceState); + spanId = parentTraceContext.Value.SpanId; } catch (Exception e) { @@ -449,7 +447,7 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work runtimeState, this.IncludeParameters, parentTraceContext, - createNewSpanId)); + spanId)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1103,7 +1101,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( OrchestrationRuntimeState runtimeState, bool includeParameters, ActivityContext? parentTraceContext, - bool createNewSpanId) + ActivitySpanId spanId) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1116,8 +1114,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = createNewSpanId ? ActivitySpanId.CreateRandom() : ((ActivityContext)parentTraceContext!).SpanId; - historyEvent.ClientSpanId = clientSpanId.ToString(); + historyEvent.ClientSpanId = spanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1146,7 +1143,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( { ActivityContext activityContext = new ActivityContext( parentContext.TraceId, - clientSpanId, + spanId, parentContext.TraceFlags, parentContext.TraceState); startedEvent.SetParentTraceContext(activityContext); diff --git a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs index a6e3611f4..b69b1f5f6 100644 --- a/src/DurableTask.Core/Tracing/DistributedTraceContext.cs +++ b/src/DurableTask.Core/Tracing/DistributedTraceContext.cs @@ -81,4 +81,4 @@ public string? TraceState [DataMember] public DateTimeOffset? ActivityStartTime { get; set; } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 10231b7df..d51be3009 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -54,7 +54,7 @@ public class TraceHelper { newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - + startEvent.SetParentTraceContext(newActivity); } @@ -357,10 +357,8 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { - - // We don't want to emit tracing for external events when they are related to entities - if ((targetInstanceId != null && targetInstanceId.StartsWith("@")) || (instance != null && instance.InstanceId.StartsWith("@"))) - { + if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + { return null; } @@ -397,8 +395,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - // We don't want to emit tracing for external events when they are related to entities - if (instance.InstanceId.StartsWith("@")) + if (Entities.IsEntityInstance(instance.InstanceId)) { return null; } From b2f859f168641e737a740e83e628a25ab248fa78 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 25 Mar 2025 15:34:57 -0700 Subject: [PATCH 11/35] forgot two comments --- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index d51be3009..01148ed50 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -357,6 +357,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { + // There is a possibility that we mislabel the event as an entity event if entities are not enabled if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) { return null; @@ -395,6 +396,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { + // There is a possibility that we mislabel the event as an entity event if entities are not enabled if (Entities.IsEntityInstance(instance.InstanceId)) { return null; From 869d1a24e7665559d9fbae20b1c0ff8c59215549 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 31 Mar 2025 15:51:38 -0700 Subject: [PATCH 12/35] addressing a few small comments --- src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs | 6 ------ src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index 3043a0680..e46d1759f 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -98,12 +98,6 @@ internal class RequestMessage : EntityMessage [DataMember] public bool IsLockRequest => LockSet != null; - /// - /// Whether this message already has a trace created for it - /// - [DataMember(Name = "traceCreated")] - public bool TraceCreated { get; set; } - /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 01148ed50..a8b0c73c0 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -54,7 +54,7 @@ public class TraceHelper { newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - + startEvent.SetParentTraceContext(newActivity); } From 31164634ce9808db8bf4e0b8a7eae62e4db35631 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 10 Apr 2025 22:27:03 -0700 Subject: [PATCH 13/35] refactored most of the tracing into this repo to more accurately reflect the end time of activites (like, when the call/signal request to an entity is actually sent, or when an orchestration is actually created by an entity) --- .../Entities/EventFormat/RequestMessage.cs | 19 ++ .../Entities/EventFormat/ResponseMessage.cs | 22 +++ src/DurableTask.Core/OrchestrationTags.cs | 5 + .../TaskOrchestrationDispatcher.cs | 57 +++--- src/DurableTask.Core/Tracing/Schema.cs | 11 ++ .../Tracing/TraceActivityConstants.cs | 4 + src/DurableTask.Core/Tracing/TraceHelper.cs | 164 +++++++++++++++++- 7 files changed, 249 insertions(+), 33 deletions(-) diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index e46d1759f..d9ed3460d 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -13,6 +13,7 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { + using DurableTask.Core.Tracing; using System; using System.Runtime.Serialization; @@ -98,6 +99,24 @@ internal class RequestMessage : EntityMessage [DataMember] public bool IsLockRequest => LockSet != null; + /// + /// Parent trace context of this request message. + /// + [DataMember(Name = "parentTraceContext", EmitDefaultValue = false)] + public DistributedTraceContext? ParentTraceContext { get; set; } + + /// + /// Whether or not to create an entity-specific trace for this request message + /// + [DataMember(Name = "createTrace")] + public bool CreateTrace { get; set; } + + /// + /// The time the request was generated. + /// + [DataMember(Name = "requestTime", EmitDefaultValue = false)] + public DateTimeOffset? RequestTime { get; set; } + /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs index 8f78516b5..eb06a76ea 100644 --- a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs @@ -13,6 +13,8 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { + using DurableTask.Core.Tracing; + using System; using System.Runtime.Serialization; [DataContract] @@ -32,6 +34,9 @@ internal class ResponseMessage : EntityMessage [IgnoreDataMember] public bool IsErrorResult => this.ErrorMessage != null || this.FailureDetails != null; + [DataMember(Name = "requestInfo", EmitDefaultValue = false)] + public RequestInformation? RequestInfo { get; set; } + public override string GetShortDescription() { if (this.IsErrorResult) @@ -47,5 +52,22 @@ public override string GetShortDescription() return $"[OperationSuccessful ({Result?.Length ?? 0} chars)]"; } } + internal class RequestInformation + { + [DataMember] + public string? Operation { get; set; } + + [DataMember] + public DateTime? ScheduledTime { get; set; } + + [DataMember] + public DateTimeOffset? RequestTime { get; set; } + + [DataMember] + public string? ClientSpanId { get; set; } + + [DataMember] + public DistributedTraceContext? ParentTraceContext { get; set; } + } } } diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index 974406418..f8c7492a2 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -46,6 +46,11 @@ public static class OrchestrationTags /// public const string TraceState = "TraceState"; + /// + /// The time the request for a new orchestration was created. + /// + public const string RequestTime = "RequestTime"; + /// /// Check whether the given tags contain the fire and forget tag /// diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 9f511a0a4..a58cabd71 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -425,29 +425,12 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work break; case OrchestratorActionType.CreateSubOrchestration: var createSubOrchestrationAction = (CreateSubOrchestrationAction)decision; - ActivityContext? parentTraceContext = traceActivity?.Context; - var spanId = ActivitySpanId.CreateRandom(); - if (createSubOrchestrationAction.Tags != null - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent) - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState)) - { - try - { - parentTraceContext = ActivityContext.Parse(traceParent, traceState); - spanId = parentTraceContext.Value.SpanId; - } - catch (Exception e) - { - TraceHelper.TraceException(TraceEventType.Error, "TaskOrchestrationDispatcher-CreateSubOrchestration-MalformedParentTrace", e); - } - } orchestratorMessages.Add( this.ProcessCreateSubOrchestrationInstanceDecision( createSubOrchestrationAction, runtimeState, this.IncludeParameters, - parentTraceContext, - spanId)); + traceActivity)); break; case OrchestratorActionType.SendEvent: var sendEventAction = (SendEventOrchestratorAction)decision; @@ -1100,8 +1083,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( CreateSubOrchestrationAction createSubOrchestrationAction, OrchestrationRuntimeState runtimeState, bool includeParameters, - ActivityContext? parentTraceContext, - ActivitySpanId spanId) + Activity? parentTraceActivity) { var historyEvent = new SubOrchestrationInstanceCreatedEvent(createSubOrchestrationAction.Id) { @@ -1114,7 +1096,8 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - historyEvent.ClientSpanId = spanId.ToString(); + ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); + historyEvent.ClientSpanId = clientSpanId.ToString(); runtimeState.AddEvent(historyEvent); @@ -1139,13 +1122,33 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - if (parentTraceContext is ActivityContext parentContext) + // If a trace parent was provided, then we will attempt to parse the parent trace context provided in the orchestration tags and use that as the + // parent trace context of the call to create a suborchestration rather than the current Activity's context. + if (createSubOrchestrationAction.Tags != null + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) + { + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) + && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) + { + var requestTime = DateTimeOffset.UtcNow; + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.RequestTime, out string requestTimeString)) + { + DateTimeOffset.TryParse(requestTimeString, out requestTime); + } + using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( + runtimeState.OrchestrationInstance!.InstanceId, + createSubOrchestrationAction.InstanceId!, + parentTraceContext, + requestTime); + if (createOrchestrationActivity != null) + { + startedEvent.SetParentTraceContext(createOrchestrationActivity); + } + } + } + else if (parentTraceActivity != null) { - ActivityContext activityContext = new ActivityContext( - parentContext.TraceId, - spanId, - parentContext.TraceFlags, - parentContext.TraceState); + ActivityContext activityContext = new ActivityContext(parentTraceActivity.TraceId, clientSpanId, parentTraceActivity.ActivityTraceFlags, parentTraceActivity.TraceStateString); startedEvent.SetParentTraceContext(activityContext); } diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 8a91e3fb1..2317a8c79 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -33,5 +33,16 @@ internal static class Status internal const string Code = "otel.status_code"; internal const string Description = "otel.status_description"; } + + internal static class Entity + { + internal const string Type = Task.Type; + internal const string EntityId = "durabletask.entity.entity_id"; + internal const string TargetInstanceId = "durabletask.entity.target_instance_id"; + internal const string TargetEntityId = "durabletask.entity.target_entity_id"; + internal const string EntityOperation = "durabletask.entity.entity_operation"; + internal const string ErrorMessage = "durabletask.entity.error_message"; + internal const string ScheduledTime = "durabletask.entity.scheduled_time"; + } } } diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index 927d1bc93..ef56f52d8 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -20,8 +20,12 @@ internal class TraceActivityConstants public const string Activity = "activity"; public const string Event = "event"; public const string Timer = "timer"; + public const string Entity = "entity"; public const string CreateOrchestration = "create_orchestration"; public const string OrchestrationEvent = "orchestration_event"; + + public const string CallEntity = "call_entity"; + public const string SignalEntity = "signal_entity"; } } diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index a8b0c73c0..4fd15dec3 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -20,6 +20,9 @@ namespace DurableTask.Core.Tracing using System.Runtime.ExceptionServices; using DurableTask.Core.Common; using DurableTask.Core.History; + using Newtonsoft.Json; + using DurableTask.Core.Entities.EventFormat; + using DurableTask.Core.Entities; /// /// Helper class for logging/tracing @@ -54,7 +57,7 @@ public class TraceHelper { newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - + startEvent.SetParentTraceContext(newActivity); } @@ -358,9 +361,13 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( string? targetInstanceId) { // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) - { - return null; + if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty)) + { + return TryParseEntityRequest(eventRaisedEvent, targetInstanceId!); + } + else if (Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + { + return TryParseEntityResponse(eventRaisedEvent, instance?.InstanceId!); } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -399,7 +406,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( // There is a possibility that we mislabel the event as an entity event if entities are not enabled if (Entities.IsEntityInstance(instance.InstanceId)) { - return null; + return TryParseEntityRequest(eventRaised, instance.InstanceId); } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -447,6 +454,56 @@ internal static void EmitTraceActivityForTimer( } } + internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, DateTime? scheduledTime, ActivityContext parentTraceContext, DateTimeOffset? startTime, string? entityId = null) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateEntitySpanName(entityName, operationName), + kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client, + parentContext: parentTraceContext, + startTime: startTime ?? DateTimeOffset.UtcNow); + + if (newActivity == null) + { + return null; + } + + newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Entity.EntityOperation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); + newActivity.SetTag(Schema.Entity.TargetEntityId, targetEntityId); + + if (!string.IsNullOrEmpty(entityId)) + { + newActivity.SetTag(Schema.Entity.EntityId, entityId); + } + + if (scheduledTime != null) + { + newActivity.SetTag(Schema.Entity.ScheduledTime, scheduledTime.Value.ToString()); + } + + return newActivity; + } + + internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset startTime) + { + Activity? newActivity = ActivityTraceSource.StartActivity( + CreateSpanName(TraceActivityConstants.Entity, TraceActivityConstants.CreateOrchestration, null), + kind: ActivityKind.Producer, + parentContext: parentTraceContext, + startTime: startTime); + + if (newActivity == null) + { + return null; + } + + newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Entity.TargetInstanceId, targetInstanceId); + newActivity.SetTag(Schema.Entity.EntityId, entityId); + + return newActivity; + } + internal static void SetRuntimeStatusTag(string runtimeStatus) { DistributedTraceActivity.Current?.SetTag(Schema.Task.Status, runtimeStatus); @@ -474,6 +531,11 @@ static string CreateTimerSpanName(string orchestrationName) return $"{TraceActivityConstants.Orchestration}:{orchestrationName}:{TraceActivityConstants.Timer}"; } + static string CreateEntitySpanName(string entityName, string operationName) + { + return $"{TraceActivityConstants.Entity}:{entityName}:{operationName}"; + } + /// /// Simple trace with no iid or eid /// @@ -709,5 +771,95 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } + + private static Activity? TryParseEntityRequest(EventRaisedEvent raisedEvent, string targetInstanceId) + { + try + { + var requestMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); + if (requestMessage == null || !requestMessage.CreateTrace) + { + return null; + } + + var parentTraceContext = Activity.Current?.Context; + if (requestMessage.ParentTraceContext != null) + { + if (ActivityContext.TryParse(requestMessage.ParentTraceContext.TraceParent, requestMessage.ParentTraceContext.TraceState, out ActivityContext activityContext)) + { + parentTraceContext = activityContext; + } + // If a parent trace context was passed with the request message, this should be the parent of the current Activity, so if we cannot parse it we should not create the Activity + // or else it will be incorrectly linked. + else + { + parentTraceContext = null; + } + } + + // We only want to create a trace activity for calling/signaling an entity in the case that we can successfully get the parent trace context of the request. + // Otherwise, we will create an unlinked trace activity with no parent. + if (parentTraceContext == null) + { + return null; + } + + Activity? newActivity = StartActivityForCallingOrSignalingEntity( + targetInstanceId!, + EntityId.FromString(targetInstanceId!).Name, + requestMessage.Operation!, + requestMessage.IsSignal, + requestMessage.ScheduledTime, + parentTraceContext.Value, + entityId: requestMessage.ParentInstanceId != null && Entities.IsEntityInstance(requestMessage.ParentInstanceId) ? requestMessage.ParentInstanceId : null, + startTime: requestMessage.RequestTime); + + if (!string.IsNullOrEmpty(newActivity?.Id)) + { + requestMessage.ParentTraceContext = new DistributedTraceContext(newActivity!.Id!, newActivity.TraceStateString); + raisedEvent.Input = JsonConvert.SerializeObject(requestMessage, Serializer.InternalSerializerSettings); + } + + return newActivity; + } + catch (Exception) + { + return null; + } + } + + private static Activity? TryParseEntityResponse(EventRaisedEvent raisedEvent, string instanceId) + { + try + { + var responseMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); + if (responseMessage == null || responseMessage.RequestInfo == null) + { + return null; + } + + if (!ActivityContext.TryParse(responseMessage.RequestInfo.ParentTraceContext?.TraceParent, responseMessage.RequestInfo.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) + { + return null; + } + + Activity? newActivity = StartActivityForCallingOrSignalingEntity( + instanceId, + EntityId.FromString(instanceId).Name, + responseMessage.RequestInfo.Operation!, + signalEntity: false, + responseMessage.RequestInfo.ScheduledTime, + parentTraceContext, + startTime: responseMessage.RequestInfo.RequestTime); + + newActivity.SetSpanId(responseMessage.RequestInfo.ClientSpanId); + + return newActivity; + } + catch (Exception) + { + return null; + } + } } -} +} \ No newline at end of file From 87776bc77e3250df74157bc9bfceb14356825392 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 11 Apr 2025 01:07:49 -0700 Subject: [PATCH 14/35] fixing spacing --- src/DurableTask.Core/Tracing/TraceHelper.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 4fd15dec3..e3f75ea7a 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -58,6 +58,7 @@ public class TraceHelper newActivity.SetTag(Schema.Task.Version, startEvent.Version); } + startEvent.SetParentTraceContext(newActivity); } @@ -862,4 +863,4 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } -} \ No newline at end of file +} From 97cd8e6703cc97a1dd1af665cd3ebb26eb2943ae Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 11 Apr 2025 12:04:47 -0700 Subject: [PATCH 15/35] slight change in formatting of the create orchestration trace --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 1 + src/DurableTask.Core/Tracing/TraceHelper.cs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index a58cabd71..a3903f481 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1137,6 +1137,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( } using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( runtimeState.OrchestrationInstance!.InstanceId, + EntityId.FromString(runtimeState.OrchestrationInstance!.InstanceId).Name, createSubOrchestrationAction.InstanceId!, parentTraceContext, requestTime); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index e3f75ea7a..3e6400ed5 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -485,10 +485,10 @@ internal static void EmitTraceActivityForTimer( return newActivity; } - internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset startTime) + internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset startTime) { Activity? newActivity = ActivityTraceSource.StartActivity( - CreateSpanName(TraceActivityConstants.Entity, TraceActivityConstants.CreateOrchestration, null), + CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null), kind: ActivityKind.Producer, parentContext: parentTraceContext, startTime: startTime); From d01179f9ae5c23c1c84e29ee32fb50131fa8161f Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 11:36:41 -0700 Subject: [PATCH 16/35] addressing PR comments --- .../Command/SendEventOrchestratorAction.cs | 7 ++ src/DurableTask.Core/EventTags.cs | 22 ++++++ .../History/EventRaisedEvent.cs | 8 ++- src/DurableTask.Core/OrchestrationContext.cs | 15 +++- src/DurableTask.Core/OrchestrationTags.cs | 6 +- src/DurableTask.Core/TaskHubClient.cs | 5 +- .../TaskOrchestrationContext.cs | 3 +- .../TaskOrchestrationDispatcher.cs | 68 +++++++++++-------- src/DurableTask.Core/Tracing/Schema.cs | 13 +--- src/DurableTask.Core/Tracing/TraceHelper.cs | 24 +++---- 10 files changed, 109 insertions(+), 62 deletions(-) create mode 100644 src/DurableTask.Core/EventTags.cs diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index fb922dc09..25f239f0d 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -11,6 +11,8 @@ // limitations under the License. // ---------------------------------------------------------------------------------- #nullable enable +using System.Collections.Generic; + namespace DurableTask.Core.Command { /// @@ -38,5 +40,10 @@ public class SendEventOrchestratorAction : OrchestratorAction /// The payload data of the external event. /// public string? EventData { get; set; } + + /// + /// The tags for the external event. + /// + public Dictionary? EventTags { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/EventTags.cs b/src/DurableTask.Core/EventTags.cs new file mode 100644 index 000000000..f0e348733 --- /dev/null +++ b/src/DurableTask.Core/EventTags.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DurableTask.Core +{ + /// + /// Tags to be used when sending external events. + /// + public static class EventTags + { + /// + /// Whether or not to create a trace for this entity request event + /// + public const string CreateEntityRequestEventTrace = "CreateSignalEntityEventTrace"; + + /// + /// Whether or not to create a trace for this entity response event + /// + public const string CreateEntityResponseEventTrace = "CreateEntityResponseEventTrace"; + } +} diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index 6da23f27c..e87227fba 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -14,6 +14,7 @@ namespace DurableTask.Core.History { using DurableTask.Core.Tracing; + using System.Collections.Generic; using System.Diagnostics; using System.Runtime.Serialization; @@ -52,9 +53,14 @@ public EventRaisedEvent(int eventId, string input) public string Input { get; set; } /// - /// The W3C trace context associated with this event. + /// Gets or sets the W3C trace context associated with this event. /// [DataMember] public DistributedTraceContext ParentTraceContext { get; set; } + + /// + /// Gets or sets any tags associated with the event + /// + public IDictionary Tags { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 4b2d8c7fe..6855bebad 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -385,7 +385,20 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// Instance in which to raise the event /// Name of the event /// Data for the event - public abstract void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData); + public void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + { + SendEvent(orchestrationInstance, eventName, eventData, null); + } + + /// + /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the + /// orchestration to fire. + /// + /// Instance in which to raise the event + /// Name of the event + /// Data for the event + /// Tags for the event + public abstract void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary eventTags); /// /// Checkpoint the orchestration instance by completing the current execution in the ContinueAsNew diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index f8c7492a2..3f680ca56 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -39,17 +39,17 @@ public static class OrchestrationTags /// /// The ID of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#traceparent-header) /// - public const string TraceParent = "TraceParent"; + public const string TraceParent = "MS_Entities_TraceParent"; /// /// The trace state of the parent trace that created this orchestration instance (see https://www.w3.org/TR/trace-context/#tracestate-header) /// - public const string TraceState = "TraceState"; + public const string TraceState = "MS_Entities_TraceState"; /// /// The time the request for a new orchestration was created. /// - public const string RequestTime = "RequestTime"; + public const string RequestTime = "MS_Entities_RequestTime"; /// /// Check whether the given tags contain the fire and forget tag diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index eb55d1936..f510796c6 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -745,7 +745,8 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) /// Instance in which to raise the event /// Name of the event /// Data for the event - public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + /// Tags for the event + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary? eventTags = null) { if (string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) @@ -756,7 +757,7 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s string serializedInput = this.defaultConverter.SerializeInternal(eventData); // Distributed Tracing - EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName }; + EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName, Tags = eventTags}; using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance); var taskMessage = new TaskMessage diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index 8a48cbe93..bedb5c68f 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -194,7 +194,7 @@ async Task CreateSubOrchestrationInstanceCore( } } - public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary eventTags = null) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { @@ -211,6 +211,7 @@ public override void SendEvent(OrchestrationInstance orchestrationInstance, stri Instance = orchestrationInstance, EventName = eventName, EventData = serializedEventData, + EventTags = eventTags, }; this.orchestratorActionsMap.Add(id, action); diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index a3903f481..98341ca13 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1096,9 +1096,6 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( historyEvent.Input = createSubOrchestrationAction.Input; } - ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); - historyEvent.ClientSpanId = clientSpanId.ToString(); - runtimeState.AddEvent(historyEvent); var taskMessage = new TaskMessage(); @@ -1122,33 +1119,11 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - // If a trace parent was provided, then we will attempt to parse the parent trace context provided in the orchestration tags and use that as the - // parent trace context of the call to create a suborchestration rather than the current Activity's context. - if (createSubOrchestrationAction.Tags != null - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) - { - if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) - && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) - { - var requestTime = DateTimeOffset.UtcNow; - if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.RequestTime, out string requestTimeString)) - { - DateTimeOffset.TryParse(requestTimeString, out requestTime); - } - using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( - runtimeState.OrchestrationInstance!.InstanceId, - EntityId.FromString(runtimeState.OrchestrationInstance!.InstanceId).Name, - createSubOrchestrationAction.InstanceId!, - parentTraceContext, - requestTime); - if (createOrchestrationActivity != null) - { - startedEvent.SetParentTraceContext(createOrchestrationActivity); - } - } - } - else if (parentTraceActivity != null) + // If this is an orchestration triggered by an entity, we will attempt to use the trace context provided in the CreateSubOrchestrationAction.Tags as the parent trace context rather than the current Activity. + if (!EntityTriggeredOrchestration(createSubOrchestrationAction, runtimeState, startedEvent) && parentTraceActivity != null) { + ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); + historyEvent.ClientSpanId = clientSpanId.ToString(); ActivityContext activityContext = new ActivityContext(parentTraceActivity.TraceId, clientSpanId, parentTraceActivity.ActivityTraceFlags, parentTraceActivity.TraceStateString); startedEvent.SetParentTraceContext(activityContext); } @@ -1177,7 +1152,8 @@ TaskMessage ProcessSendEventDecision( EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, sendEventAction.EventData) { - Name = sendEventAction.EventName + Name = sendEventAction.EventName, + Tags = sendEventAction.EventTags, }; // Distributed Tracing: start a new trace activity derived from the orchestration @@ -1194,6 +1170,38 @@ TaskMessage ProcessSendEventDecision( Event = eventRaisedEvent }; } + + private static bool EntityTriggeredOrchestration( + CreateSubOrchestrationAction createSubOrchestrationAction, + OrchestrationRuntimeState runtimeState, + ExecutionStartedEvent startedEvent) + { + if (createSubOrchestrationAction.Tags != null + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) + { + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) + && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) + { + var requestTime = DateTimeOffset.UtcNow; + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.RequestTime, out string requestTimeString)) + { + DateTimeOffset.TryParse(requestTimeString, out requestTime); + } + using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( + runtimeState.OrchestrationInstance!.InstanceId, + EntityId.FromString(runtimeState.OrchestrationInstance!.InstanceId).Name, + createSubOrchestrationAction.InstanceId!, + parentTraceContext, + requestTime); + if (createOrchestrationActivity != null) + { + startedEvent.SetParentTraceContext(createOrchestrationActivity); + } + } + return true; + } + return false; + } internal class NonBlockingCountdownLock { diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 2317a8c79..80872f52d 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -26,6 +26,8 @@ internal static class Task internal const string TaskId = "durabletask.task.task_id"; internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; internal const string FireAt = "durabletask.fire_at"; + internal const string Operation = "durabletask.task.operation"; + internal const string ScheduledTime = "durabletask.task.scheduled_time"; } internal static class Status @@ -33,16 +35,5 @@ internal static class Status internal const string Code = "otel.status_code"; internal const string Description = "otel.status_description"; } - - internal static class Entity - { - internal const string Type = Task.Type; - internal const string EntityId = "durabletask.entity.entity_id"; - internal const string TargetInstanceId = "durabletask.entity.target_instance_id"; - internal const string TargetEntityId = "durabletask.entity.target_entity_id"; - internal const string EntityOperation = "durabletask.entity.entity_operation"; - internal const string ErrorMessage = "durabletask.entity.error_message"; - internal const string ScheduledTime = "durabletask.entity.scheduled_time"; - } } } diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 3e6400ed5..345f6e287 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -361,12 +361,11 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { - // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty)) + if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) { return TryParseEntityRequest(eventRaisedEvent, targetInstanceId!); } - else if (Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + else if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityResponseEventTrace)) { return TryParseEntityResponse(eventRaisedEvent, instance?.InstanceId!); } @@ -404,8 +403,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(instance.InstanceId)) + if (eventRaised.Tags != null && eventRaised.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) { return TryParseEntityRequest(eventRaised, instance.InstanceId); } @@ -468,18 +466,18 @@ internal static void EmitTraceActivityForTimer( return null; } - newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Entity.EntityOperation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); - newActivity.SetTag(Schema.Entity.TargetEntityId, targetEntityId); + newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Task.Operation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); + newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetEntityId); if (!string.IsNullOrEmpty(entityId)) { - newActivity.SetTag(Schema.Entity.EntityId, entityId); + newActivity.SetTag(Schema.Task.InstanceId, entityId); } if (scheduledTime != null) { - newActivity.SetTag(Schema.Entity.ScheduledTime, scheduledTime.Value.ToString()); + newActivity.SetTag(Schema.Task.ScheduledTime, scheduledTime.Value.ToString()); } return newActivity; @@ -498,9 +496,9 @@ internal static void EmitTraceActivityForTimer( return null; } - newActivity.SetTag(Schema.Entity.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Entity.TargetInstanceId, targetInstanceId); - newActivity.SetTag(Schema.Entity.EntityId, entityId); + newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); + newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetInstanceId); + newActivity.SetTag(Schema.Task.InstanceId, entityId); return newActivity; } From 30db2fb36466101bce22c4c923c87c1df9422938 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 11:38:13 -0700 Subject: [PATCH 17/35] adding back new lines at the ends of files --- src/DurableTask.Core/Command/SendEventOrchestratorAction.cs | 2 +- src/DurableTask.Core/History/EventRaisedEvent.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index 25f239f0d..f5e7d0af7 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -46,4 +46,4 @@ public class SendEventOrchestratorAction : OrchestratorAction /// public Dictionary? EventTags { get; set; } } -} \ No newline at end of file +} diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index e87227fba..72ed160f3 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -63,4 +63,4 @@ public EventRaisedEvent(int eventId, string input) /// public IDictionary Tags { get; set; } } -} \ No newline at end of file +} From c4dc784786b6199daed7d46826b4463dc66cf1fa Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 11:49:18 -0700 Subject: [PATCH 18/35] trying to fix these line endings --- src/DurableTask.Core/Command/SendEventOrchestratorAction.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index f5e7d0af7..25f239f0d 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -46,4 +46,4 @@ public class SendEventOrchestratorAction : OrchestratorAction /// public Dictionary? EventTags { get; set; } } -} +} \ No newline at end of file From 41b0280cfc58d9b6248e3f66e0596cfd2f26a740 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 12:02:20 -0700 Subject: [PATCH 19/35] dealing with new lines again --- src/DurableTask.Core/History/EventRaisedEvent.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index 72ed160f3..e87227fba 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -63,4 +63,4 @@ public EventRaisedEvent(int eventId, string input) /// public IDictionary Tags { get; set; } } -} +} \ No newline at end of file From dcac8a8578c8faf2f212c780763addd1718508b0 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 12:27:45 -0700 Subject: [PATCH 20/35] tiny change --- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 345f6e287..8fe0bc182 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -459,7 +459,7 @@ internal static void EmitTraceActivityForTimer( CreateEntitySpanName(entityName, operationName), kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client, parentContext: parentTraceContext, - startTime: startTime ?? DateTimeOffset.UtcNow); + startTime: startTime ?? default); if (newActivity == null) { From 9a54fefd285ab18e2219eb193bcd00120003ca5b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 15 Apr 2025 14:07:09 -0700 Subject: [PATCH 21/35] missed a null check --- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 8fe0bc182..b840395d8 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -851,7 +851,7 @@ static void ExceptionHandlingWrapper(Action innerFunc) parentTraceContext, startTime: responseMessage.RequestInfo.RequestTime); - newActivity.SetSpanId(responseMessage.RequestInfo.ClientSpanId); + newActivity?.SetSpanId(responseMessage.RequestInfo.ClientSpanId); return newActivity; } From 9aec0fc557a54895d7008394a5ea9fe0265dc637 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 11:57:11 -0700 Subject: [PATCH 22/35] reverting to old design --- .../Entities/EventFormat/RequestMessage.cs | 19 --- .../Entities/EventFormat/ResponseMessage.cs | 22 --- src/DurableTask.Core/TaskEntityDispatcher.cs | 8 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 144 ++---------------- 4 files changed, 16 insertions(+), 177 deletions(-) diff --git a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs index d9ed3460d..e46d1759f 100644 --- a/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/RequestMessage.cs @@ -13,7 +13,6 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { - using DurableTask.Core.Tracing; using System; using System.Runtime.Serialization; @@ -99,24 +98,6 @@ internal class RequestMessage : EntityMessage [DataMember] public bool IsLockRequest => LockSet != null; - /// - /// Parent trace context of this request message. - /// - [DataMember(Name = "parentTraceContext", EmitDefaultValue = false)] - public DistributedTraceContext? ParentTraceContext { get; set; } - - /// - /// Whether or not to create an entity-specific trace for this request message - /// - [DataMember(Name = "createTrace")] - public bool CreateTrace { get; set; } - - /// - /// The time the request was generated. - /// - [DataMember(Name = "requestTime", EmitDefaultValue = false)] - public DateTimeOffset? RequestTime { get; set; } - /// public override string GetShortDescription() { diff --git a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs index eb06a76ea..8f78516b5 100644 --- a/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs +++ b/src/DurableTask.Core/Entities/EventFormat/ResponseMessage.cs @@ -13,8 +13,6 @@ #nullable enable namespace DurableTask.Core.Entities.EventFormat { - using DurableTask.Core.Tracing; - using System; using System.Runtime.Serialization; [DataContract] @@ -34,9 +32,6 @@ internal class ResponseMessage : EntityMessage [IgnoreDataMember] public bool IsErrorResult => this.ErrorMessage != null || this.FailureDetails != null; - [DataMember(Name = "requestInfo", EmitDefaultValue = false)] - public RequestInformation? RequestInfo { get; set; } - public override string GetShortDescription() { if (this.IsErrorResult) @@ -52,22 +47,5 @@ public override string GetShortDescription() return $"[OperationSuccessful ({Result?.Length ?? 0} chars)]"; } } - internal class RequestInformation - { - [DataMember] - public string? Operation { get; set; } - - [DataMember] - public DateTime? ScheduledTime { get; set; } - - [DataMember] - public DateTimeOffset? RequestTime { get; set; } - - [DataMember] - public string? ClientSpanId { get; set; } - - [DataMember] - public DistributedTraceContext? ParentTraceContext { get; set; } - } } } diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index aad78675d..8150b5a66 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -57,7 +57,7 @@ internal TaskEntityDispatcher( this.errorPropagationMode = errorPropagationMode; this.entityOrchestrationService = (orchestrationService as IEntityOrchestrationService)!; this.entityBackendProperties = entityOrchestrationService.EntityBackendProperties; - + this.dispatcher = new WorkItemDispatcher( "TaskEntityDispatcher", item => item == null ? string.Empty : item.InstanceId, @@ -379,7 +379,7 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work } OrchestrationState instanceState = (runtimeState.ExecutionStartedEvent != null) ? - instanceState = Utils.BuildOrchestrationState(runtimeState) : null; + instanceState = Utils.BuildOrchestrationState(runtimeState) : null; if (workItem.RestoreOriginalRuntimeStateDuringCompletion) { @@ -742,7 +742,7 @@ void SendLockResponseMessage(WorkItemEffects effects, OrchestrationInstance targ var message = new ResponseMessage() { // content is ignored by receiver but helps with tracing - Result = ResponseMessage.LockAcquisitionCompletion, + Result = ResponseMessage.LockAcquisitionCompletion, }; this.ProcessSendEventMessage(effects, target, EntityMessageEventNames.ResponseMessageEventName(requestId), message); } @@ -871,7 +871,7 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => } var result = await taskEntity.ExecuteOperationBatchAsync(request); - + dispatchContext.SetProperty(result); }); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index b840395d8..4818746b4 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -20,9 +20,6 @@ namespace DurableTask.Core.Tracing using System.Runtime.ExceptionServices; using DurableTask.Core.Common; using DurableTask.Core.History; - using Newtonsoft.Json; - using DurableTask.Core.Entities.EventFormat; - using DurableTask.Core.Entities; /// /// Helper class for logging/tracing @@ -361,13 +358,10 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( OrchestrationInstance? instance, string? targetInstanceId) { - if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) + // There is a possibility that we mislabel the event as an entity event if entities are not enabled + if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) { - return TryParseEntityRequest(eventRaisedEvent, targetInstanceId!); - } - else if (eventRaisedEvent.Tags != null && eventRaisedEvent.Tags.ContainsKey(EventTags.CreateEntityResponseEventTrace)) - { - return TryParseEntityResponse(eventRaisedEvent, instance?.InstanceId!); + return null; } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -403,9 +397,10 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - if (eventRaised.Tags != null && eventRaised.Tags.ContainsKey(EventTags.CreateEntityRequestEventTrace)) + // There is a possibility that we mislabel the event as an entity event if entities are not enabled + if (Entities.IsEntityInstance(instance.InstanceId)) { - return TryParseEntityRequest(eventRaised, instance.InstanceId); + return null; } Activity? newActivity = ActivityTraceSource.StartActivity( @@ -453,11 +448,11 @@ internal static void EmitTraceActivityForTimer( } } - internal static Activity? StartActivityForCallingOrSignalingEntity(string targetEntityId, string entityName, string operationName, bool signalEntity, DateTime? scheduledTime, ActivityContext parentTraceContext, DateTimeOffset? startTime, string? entityId = null) + internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset? startTime, DateTime? scheduledTime = null) { Activity? newActivity = ActivityTraceSource.StartActivity( - CreateEntitySpanName(entityName, operationName), - kind: signalEntity ? ActivityKind.Producer : ActivityKind.Client, + CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null), + kind: ActivityKind.Producer, parentContext: parentTraceContext, startTime: startTime ?? default); @@ -467,13 +462,8 @@ internal static void EmitTraceActivityForTimer( } newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Task.Operation, signalEntity ? TraceActivityConstants.SignalEntity : TraceActivityConstants.CallEntity); - newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetEntityId); - - if (!string.IsNullOrEmpty(entityId)) - { - newActivity.SetTag(Schema.Task.InstanceId, entityId); - } + newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetInstanceId); + newActivity.SetTag(Schema.Task.InstanceId, entityId); if (scheduledTime != null) { @@ -483,26 +473,6 @@ internal static void EmitTraceActivityForTimer( return newActivity; } - internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset startTime) - { - Activity? newActivity = ActivityTraceSource.StartActivity( - CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null), - kind: ActivityKind.Producer, - parentContext: parentTraceContext, - startTime: startTime); - - if (newActivity == null) - { - return null; - } - - newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetInstanceId); - newActivity.SetTag(Schema.Task.InstanceId, entityId); - - return newActivity; - } - internal static void SetRuntimeStatusTag(string runtimeStatus) { DistributedTraceActivity.Current?.SetTag(Schema.Task.Status, runtimeStatus); @@ -770,95 +740,5 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } - - private static Activity? TryParseEntityRequest(EventRaisedEvent raisedEvent, string targetInstanceId) - { - try - { - var requestMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); - if (requestMessage == null || !requestMessage.CreateTrace) - { - return null; - } - - var parentTraceContext = Activity.Current?.Context; - if (requestMessage.ParentTraceContext != null) - { - if (ActivityContext.TryParse(requestMessage.ParentTraceContext.TraceParent, requestMessage.ParentTraceContext.TraceState, out ActivityContext activityContext)) - { - parentTraceContext = activityContext; - } - // If a parent trace context was passed with the request message, this should be the parent of the current Activity, so if we cannot parse it we should not create the Activity - // or else it will be incorrectly linked. - else - { - parentTraceContext = null; - } - } - - // We only want to create a trace activity for calling/signaling an entity in the case that we can successfully get the parent trace context of the request. - // Otherwise, we will create an unlinked trace activity with no parent. - if (parentTraceContext == null) - { - return null; - } - - Activity? newActivity = StartActivityForCallingOrSignalingEntity( - targetInstanceId!, - EntityId.FromString(targetInstanceId!).Name, - requestMessage.Operation!, - requestMessage.IsSignal, - requestMessage.ScheduledTime, - parentTraceContext.Value, - entityId: requestMessage.ParentInstanceId != null && Entities.IsEntityInstance(requestMessage.ParentInstanceId) ? requestMessage.ParentInstanceId : null, - startTime: requestMessage.RequestTime); - - if (!string.IsNullOrEmpty(newActivity?.Id)) - { - requestMessage.ParentTraceContext = new DistributedTraceContext(newActivity!.Id!, newActivity.TraceStateString); - raisedEvent.Input = JsonConvert.SerializeObject(requestMessage, Serializer.InternalSerializerSettings); - } - - return newActivity; - } - catch (Exception) - { - return null; - } - } - - private static Activity? TryParseEntityResponse(EventRaisedEvent raisedEvent, string instanceId) - { - try - { - var responseMessage = JsonConvert.DeserializeObject(raisedEvent.Input, Serializer.InternalSerializerSettings); - if (responseMessage == null || responseMessage.RequestInfo == null) - { - return null; - } - - if (!ActivityContext.TryParse(responseMessage.RequestInfo.ParentTraceContext?.TraceParent, responseMessage.RequestInfo.ParentTraceContext?.TraceState, out ActivityContext parentTraceContext)) - { - return null; - } - - Activity? newActivity = StartActivityForCallingOrSignalingEntity( - instanceId, - EntityId.FromString(instanceId).Name, - responseMessage.RequestInfo.Operation!, - signalEntity: false, - responseMessage.RequestInfo.ScheduledTime, - parentTraceContext, - startTime: responseMessage.RequestInfo.RequestTime); - - newActivity?.SetSpanId(responseMessage.RequestInfo.ClientSpanId); - - return newActivity; - } - catch (Exception) - { - return null; - } - } } -} +} \ No newline at end of file From d8c10bb3b8ffd64162e0ad84fe18f2923edcd24e Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:03:21 -0700 Subject: [PATCH 23/35] missed some --- .../Command/SendEventOrchestratorAction.cs | 5 ----- src/DurableTask.Core/EventTags.cs | 22 ------------------- .../History/EventRaisedEvent.cs | 5 ----- src/DurableTask.Core/OrchestrationContext.cs | 15 +------------ src/DurableTask.Core/TaskHubClient.cs | 5 ++--- .../TaskOrchestrationContext.cs | 3 +-- src/DurableTask.Core/Tracing/Schema.cs | 2 -- .../Tracing/TraceActivityConstants.cs | 4 ---- 8 files changed, 4 insertions(+), 57 deletions(-) delete mode 100644 src/DurableTask.Core/EventTags.cs diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index 25f239f0d..1bf0a7a85 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -40,10 +40,5 @@ public class SendEventOrchestratorAction : OrchestratorAction /// The payload data of the external event. /// public string? EventData { get; set; } - - /// - /// The tags for the external event. - /// - public Dictionary? EventTags { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/EventTags.cs b/src/DurableTask.Core/EventTags.cs deleted file mode 100644 index f0e348733..000000000 --- a/src/DurableTask.Core/EventTags.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace DurableTask.Core -{ - /// - /// Tags to be used when sending external events. - /// - public static class EventTags - { - /// - /// Whether or not to create a trace for this entity request event - /// - public const string CreateEntityRequestEventTrace = "CreateSignalEntityEventTrace"; - - /// - /// Whether or not to create a trace for this entity response event - /// - public const string CreateEntityResponseEventTrace = "CreateEntityResponseEventTrace"; - } -} diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index e87227fba..74f90a22f 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -57,10 +57,5 @@ public EventRaisedEvent(int eventId, string input) /// [DataMember] public DistributedTraceContext ParentTraceContext { get; set; } - - /// - /// Gets or sets any tags associated with the event - /// - public IDictionary Tags { get; set; } } } \ No newline at end of file diff --git a/src/DurableTask.Core/OrchestrationContext.cs b/src/DurableTask.Core/OrchestrationContext.cs index 6855bebad..4b2d8c7fe 100644 --- a/src/DurableTask.Core/OrchestrationContext.cs +++ b/src/DurableTask.Core/OrchestrationContext.cs @@ -385,20 +385,7 @@ public abstract Task CreateSubOrchestrationInstance(string name, string ve /// Instance in which to raise the event /// Name of the event /// Data for the event - public void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) - { - SendEvent(orchestrationInstance, eventName, eventData, null); - } - - /// - /// Raises an event for the specified orchestration instance, which eventually causes the OnEvent() method in the - /// orchestration to fire. - /// - /// Instance in which to raise the event - /// Name of the event - /// Data for the event - /// Tags for the event - public abstract void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary eventTags); + public abstract void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData); /// /// Checkpoint the orchestration instance by completing the current execution in the ContinueAsNew diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index f510796c6..eb55d1936 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -745,8 +745,7 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) /// Instance in which to raise the event /// Name of the event /// Data for the event - /// Tags for the event - public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary? eventTags = null) + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { if (string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) @@ -757,7 +756,7 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s string serializedInput = this.defaultConverter.SerializeInternal(eventData); // Distributed Tracing - EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName, Tags = eventTags}; + EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName }; using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance); var taskMessage = new TaskMessage diff --git a/src/DurableTask.Core/TaskOrchestrationContext.cs b/src/DurableTask.Core/TaskOrchestrationContext.cs index bedb5c68f..8a48cbe93 100644 --- a/src/DurableTask.Core/TaskOrchestrationContext.cs +++ b/src/DurableTask.Core/TaskOrchestrationContext.cs @@ -194,7 +194,7 @@ async Task CreateSubOrchestrationInstanceCore( } } - public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData, Dictionary eventTags = null) + public override void SendEvent(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { if (string.IsNullOrWhiteSpace(orchestrationInstance?.InstanceId)) { @@ -211,7 +211,6 @@ public override void SendEvent(OrchestrationInstance orchestrationInstance, stri Instance = orchestrationInstance, EventName = eventName, EventData = serializedEventData, - EventTags = eventTags, }; this.orchestratorActionsMap.Add(id, action); diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 80872f52d..8a91e3fb1 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -26,8 +26,6 @@ internal static class Task internal const string TaskId = "durabletask.task.task_id"; internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; internal const string FireAt = "durabletask.fire_at"; - internal const string Operation = "durabletask.task.operation"; - internal const string ScheduledTime = "durabletask.task.scheduled_time"; } internal static class Status diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index ef56f52d8..927d1bc93 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -20,12 +20,8 @@ internal class TraceActivityConstants public const string Activity = "activity"; public const string Event = "event"; public const string Timer = "timer"; - public const string Entity = "entity"; public const string CreateOrchestration = "create_orchestration"; public const string OrchestrationEvent = "orchestration_event"; - - public const string CallEntity = "call_entity"; - public const string SignalEntity = "signal_entity"; } } From 6b2bce10a564ef69e819bbea6e349ebf953106ea Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:04:37 -0700 Subject: [PATCH 24/35] and missed some more --- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 1 - src/DurableTask.Core/Tracing/Schema.cs | 1 + src/DurableTask.Core/Tracing/TraceActivityConstants.cs | 1 + src/DurableTask.Core/Tracing/TraceHelper.cs | 5 ----- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 98341ca13..84611366a 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1153,7 +1153,6 @@ TaskMessage ProcessSendEventDecision( EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, sendEventAction.EventData) { Name = sendEventAction.EventName, - Tags = sendEventAction.EventTags, }; // Distributed Tracing: start a new trace activity derived from the orchestration diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index 8a91e3fb1..c177aabab 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -26,6 +26,7 @@ internal static class Task internal const string TaskId = "durabletask.task.task_id"; internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; internal const string FireAt = "durabletask.fire_at"; + internal const string ScheduledTime = "durabletask.task.scheduled_time"; } internal static class Status diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index 927d1bc93..2c8a2e208 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -20,6 +20,7 @@ internal class TraceActivityConstants public const string Activity = "activity"; public const string Event = "event"; public const string Timer = "timer"; + public const string Entity = "entity"; public const string CreateOrchestration = "create_orchestration"; public const string OrchestrationEvent = "orchestration_event"; diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 4818746b4..b5c6c4e0c 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -500,11 +500,6 @@ static string CreateTimerSpanName(string orchestrationName) return $"{TraceActivityConstants.Orchestration}:{orchestrationName}:{TraceActivityConstants.Timer}"; } - static string CreateEntitySpanName(string entityName, string operationName) - { - return $"{TraceActivityConstants.Entity}:{entityName}:{operationName}"; - } - /// /// Simple trace with no iid or eid /// From 0b5687355d51400880058cf2ddcfa1d9a0a0ccb1 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:08:09 -0700 Subject: [PATCH 25/35] missed even more --- .../Command/SendEventOrchestratorAction.cs | 2 -- src/DurableTask.Core/History/EventRaisedEvent.cs | 3 +-- src/DurableTask.Core/TaskEntityDispatcher.cs | 8 ++++---- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs index 1bf0a7a85..fb922dc09 100644 --- a/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs +++ b/src/DurableTask.Core/Command/SendEventOrchestratorAction.cs @@ -11,8 +11,6 @@ // limitations under the License. // ---------------------------------------------------------------------------------- #nullable enable -using System.Collections.Generic; - namespace DurableTask.Core.Command { /// diff --git a/src/DurableTask.Core/History/EventRaisedEvent.cs b/src/DurableTask.Core/History/EventRaisedEvent.cs index 74f90a22f..6da23f27c 100644 --- a/src/DurableTask.Core/History/EventRaisedEvent.cs +++ b/src/DurableTask.Core/History/EventRaisedEvent.cs @@ -14,7 +14,6 @@ namespace DurableTask.Core.History { using DurableTask.Core.Tracing; - using System.Collections.Generic; using System.Diagnostics; using System.Runtime.Serialization; @@ -53,7 +52,7 @@ public EventRaisedEvent(int eventId, string input) public string Input { get; set; } /// - /// Gets or sets the W3C trace context associated with this event. + /// The W3C trace context associated with this event. /// [DataMember] public DistributedTraceContext ParentTraceContext { get; set; } diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 8150b5a66..8706d87af 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -57,7 +57,7 @@ internal TaskEntityDispatcher( this.errorPropagationMode = errorPropagationMode; this.entityOrchestrationService = (orchestrationService as IEntityOrchestrationService)!; this.entityBackendProperties = entityOrchestrationService.EntityBackendProperties; - + this.dispatcher = new WorkItemDispatcher( "TaskEntityDispatcher", item => item == null ? string.Empty : item.InstanceId, @@ -379,7 +379,7 @@ protected async Task OnProcessWorkItemAsync(TaskOrchestrationWorkItem work } OrchestrationState instanceState = (runtimeState.ExecutionStartedEvent != null) ? - instanceState = Utils.BuildOrchestrationState(runtimeState) : null; + instanceState = Utils.BuildOrchestrationState(runtimeState) : null; if (workItem.RestoreOriginalRuntimeStateDuringCompletion) { @@ -742,7 +742,7 @@ void SendLockResponseMessage(WorkItemEffects effects, OrchestrationInstance targ var message = new ResponseMessage() { // content is ignored by receiver but helps with tracing - Result = ResponseMessage.LockAcquisitionCompletion, + Result = ResponseMessage.LockAcquisitionCompletion, }; this.ProcessSendEventMessage(effects, target, EntityMessageEventNames.ResponseMessageEventName(requestId), message); } @@ -871,7 +871,7 @@ await this.dispatchPipeline.RunAsync(dispatchContext, async _ => } var result = await taskEntity.ExecuteOperationBatchAsync(request); - + dispatchContext.SetProperty(result); }); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index b5c6c4e0c..3fbf9b574 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -736,4 +736,4 @@ static void ExceptionHandlingWrapper(Action innerFunc) } } } -} \ No newline at end of file +} From ac98d23d0ceab760b6b5383d89e270388c741864 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:10:19 -0700 Subject: [PATCH 26/35] will it ever end --- src/DurableTask.Core/TaskEntityDispatcher.cs | 2 +- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 2 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/DurableTask.Core/TaskEntityDispatcher.cs b/src/DurableTask.Core/TaskEntityDispatcher.cs index 8706d87af..aad78675d 100644 --- a/src/DurableTask.Core/TaskEntityDispatcher.cs +++ b/src/DurableTask.Core/TaskEntityDispatcher.cs @@ -57,7 +57,7 @@ internal TaskEntityDispatcher( this.errorPropagationMode = errorPropagationMode; this.entityOrchestrationService = (orchestrationService as IEntityOrchestrationService)!; this.entityBackendProperties = entityOrchestrationService.EntityBackendProperties; - + this.dispatcher = new WorkItemDispatcher( "TaskEntityDispatcher", item => item == null ? string.Empty : item.InstanceId, diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 84611366a..8bdf23172 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1152,7 +1152,7 @@ TaskMessage ProcessSendEventDecision( EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, sendEventAction.EventData) { - Name = sendEventAction.EventName, + Name = sendEventAction.EventName }; // Distributed Tracing: start a new trace activity derived from the orchestration diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 3fbf9b574..afe20d112 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -55,7 +55,6 @@ public class TraceHelper newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - startEvent.SetParentTraceContext(newActivity); } From 9ef781c8b46ed162f49761b021977b5acb14894b Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 18 Apr 2025 12:11:00 -0700 Subject: [PATCH 27/35] last one i think --- src/DurableTask.Core/Tracing/TraceHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index afe20d112..9372bbe03 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -54,7 +54,7 @@ public class TraceHelper { newActivity.SetTag(Schema.Task.Version, startEvent.Version); } - + startEvent.SetParentTraceContext(newActivity); } From ee86dd2459ca5c21bc18f6f9438ef2bf21c22956 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 22 Apr 2025 16:23:30 -0700 Subject: [PATCH 28/35] moved activity for entity starting an orchestration back into webjobs --- src/DurableTask.Core/OrchestrationTags.cs | 5 --- .../TaskOrchestrationDispatcher.cs | 45 +++++-------------- 2 files changed, 12 insertions(+), 38 deletions(-) diff --git a/src/DurableTask.Core/OrchestrationTags.cs b/src/DurableTask.Core/OrchestrationTags.cs index 3f680ca56..c3065de46 100644 --- a/src/DurableTask.Core/OrchestrationTags.cs +++ b/src/DurableTask.Core/OrchestrationTags.cs @@ -46,11 +46,6 @@ public static class OrchestrationTags /// public const string TraceState = "MS_Entities_TraceState"; - /// - /// The time the request for a new orchestration was created. - /// - public const string RequestTime = "MS_Entities_RequestTime"; - /// /// Check whether the given tags contain the fire and forget tag /// diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 8bdf23172..183e01fe7 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1120,7 +1120,18 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( }; // If this is an orchestration triggered by an entity, we will attempt to use the trace context provided in the CreateSubOrchestrationAction.Tags as the parent trace context rather than the current Activity. - if (!EntityTriggeredOrchestration(createSubOrchestrationAction, runtimeState, startedEvent) && parentTraceActivity != null) + if (createSubOrchestrationAction.Tags != null + && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) + { + // If a parent trace context was provided but we fail to parse it, we don't want to attach any parent trace context to the start event since that will incorrectly link the trace corresponding to the orchestration execution + // as a child of Activity.Current, which is not truly the parent of the request + if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) + && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) + { + startedEvent.SetParentTraceContext(parentTraceContext); + } + } + else if (parentTraceActivity != null) { ActivitySpanId clientSpanId = ActivitySpanId.CreateRandom(); historyEvent.ClientSpanId = clientSpanId.ToString(); @@ -1169,38 +1180,6 @@ TaskMessage ProcessSendEventDecision( Event = eventRaisedEvent }; } - - private static bool EntityTriggeredOrchestration( - CreateSubOrchestrationAction createSubOrchestrationAction, - OrchestrationRuntimeState runtimeState, - ExecutionStartedEvent startedEvent) - { - if (createSubOrchestrationAction.Tags != null - && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) - { - if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceState, out string traceState) - && ActivityContext.TryParse(traceParent, traceState, out ActivityContext parentTraceContext)) - { - var requestTime = DateTimeOffset.UtcNow; - if (createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.RequestTime, out string requestTimeString)) - { - DateTimeOffset.TryParse(requestTimeString, out requestTime); - } - using var createOrchestrationActivity = TraceHelper.StartActivityForEntityStartingAnOrchestration( - runtimeState.OrchestrationInstance!.InstanceId, - EntityId.FromString(runtimeState.OrchestrationInstance!.InstanceId).Name, - createSubOrchestrationAction.InstanceId!, - parentTraceContext, - requestTime); - if (createOrchestrationActivity != null) - { - startedEvent.SetParentTraceContext(createOrchestrationActivity); - } - } - return true; - } - return false; - } internal class NonBlockingCountdownLock { From 2206569e102e49774d069c27341edc421be7ec46 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 22 Apr 2025 16:25:08 -0700 Subject: [PATCH 29/35] as always missed some stuff --- src/DurableTask.Core/Tracing/Schema.cs | 1 - .../Tracing/TraceActivityConstants.cs | 1 - src/DurableTask.Core/Tracing/TraceHelper.cs | 25 ------------------- 3 files changed, 27 deletions(-) diff --git a/src/DurableTask.Core/Tracing/Schema.cs b/src/DurableTask.Core/Tracing/Schema.cs index c177aabab..8a91e3fb1 100644 --- a/src/DurableTask.Core/Tracing/Schema.cs +++ b/src/DurableTask.Core/Tracing/Schema.cs @@ -26,7 +26,6 @@ internal static class Task internal const string TaskId = "durabletask.task.task_id"; internal const string EventTargetInstanceId = "durabletask.event.target_instance_id"; internal const string FireAt = "durabletask.fire_at"; - internal const string ScheduledTime = "durabletask.task.scheduled_time"; } internal static class Status diff --git a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs index 2c8a2e208..927d1bc93 100644 --- a/src/DurableTask.Core/Tracing/TraceActivityConstants.cs +++ b/src/DurableTask.Core/Tracing/TraceActivityConstants.cs @@ -20,7 +20,6 @@ internal class TraceActivityConstants public const string Activity = "activity"; public const string Event = "event"; public const string Timer = "timer"; - public const string Entity = "entity"; public const string CreateOrchestration = "create_orchestration"; public const string OrchestrationEvent = "orchestration_event"; diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 9372bbe03..285d99437 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -447,31 +447,6 @@ internal static void EmitTraceActivityForTimer( } } - internal static Activity? StartActivityForEntityStartingAnOrchestration(string entityId, string entityName, string targetInstanceId, ActivityContext parentTraceContext, DateTimeOffset? startTime, DateTime? scheduledTime = null) - { - Activity? newActivity = ActivityTraceSource.StartActivity( - CreateSpanName(entityName, TraceActivityConstants.CreateOrchestration, null), - kind: ActivityKind.Producer, - parentContext: parentTraceContext, - startTime: startTime ?? default); - - if (newActivity == null) - { - return null; - } - - newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.Entity); - newActivity.SetTag(Schema.Task.EventTargetInstanceId, targetInstanceId); - newActivity.SetTag(Schema.Task.InstanceId, entityId); - - if (scheduledTime != null) - { - newActivity.SetTag(Schema.Task.ScheduledTime, scheduledTime.Value.ToString()); - } - - return newActivity; - } - internal static void SetRuntimeStatusTag(string runtimeStatus) { DistributedTraceActivity.Current?.SetTag(Schema.Task.Status, runtimeStatus); From ba758512745dd6b077e7855dd3777902c6144332 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 25 Apr 2025 14:28:52 -0700 Subject: [PATCH 30/35] added support for an entities enabled flag --- src/DurableTask.Core/TaskHubClient.cs | 5 +++-- src/DurableTask.Core/TaskOrchestrationDispatcher.cs | 7 ++++++- src/DurableTask.Core/Tracing/TraceHelper.cs | 11 ++++++----- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index eb55d1936..a75bf12a5 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -745,7 +745,8 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) /// Instance in which to raise the event /// Name of the event /// Data for the event - public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + /// Whether or not this event corresponds to an entity (false by default) + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, bool entityEvent = false) { if (string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) @@ -757,7 +758,7 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s // Distributed Tracing EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName }; - using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance); + using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance, entityEvent); var taskMessage = new TaskMessage { diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 183e01fe7..08e1f83df 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -112,6 +112,11 @@ public async Task StopAsync(bool forced) /// public bool IncludeParameters { get; set; } + /// + /// Gets or sets the flag for whether or not entities are enabled + /// + public bool EntitiesEnabled { get; set; } + /// /// Method to get the next work item to process within supplied timeout /// @@ -1168,7 +1173,7 @@ TaskMessage ProcessSendEventDecision( // Distributed Tracing: start a new trace activity derived from the orchestration // for an EventRaisedEvent (external event) - using Activity? traceActivity = TraceHelper.StartTraceActivityForEventRaisedFromWorker(eventRaisedEvent, runtimeState.OrchestrationInstance, sendEventAction.Instance?.InstanceId); + using Activity? traceActivity = TraceHelper.StartTraceActivityForEventRaisedFromWorker(eventRaisedEvent, runtimeState.OrchestrationInstance, this.EntitiesEnabled, sendEventAction.Instance?.InstanceId); this.logHelper.RaisingEvent(runtimeState.OrchestrationInstance!, historyEvent); diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 285d99437..1e3dcd798 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -348,6 +348,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . + /// Whether or not entities are enabled, meaning this event could possibly correspond to entity /// The instance id of the orchestration that will receive the event. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. @@ -355,10 +356,10 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( internal static Activity? StartTraceActivityForEventRaisedFromWorker( EventRaisedEvent eventRaisedEvent, OrchestrationInstance? instance, + bool entitiesEnabled, string? targetInstanceId) { - // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty)) + if (entitiesEnabled && (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty))) { return null; } @@ -391,13 +392,13 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . + /// Whether or not this event corresponds to an entity /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// - internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) + internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance, bool entityEvent) { - // There is a possibility that we mislabel the event as an entity event if entities are not enabled - if (Entities.IsEntityInstance(instance.InstanceId)) + if (entityEvent) { return null; } From 92e5c268ac421c41fc15f08e3b07f126e00a129a Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 25 Apr 2025 15:54:39 -0700 Subject: [PATCH 31/35] added a null check for client span ID when creating the activity for scheduling a suborchestration --- src/DurableTask.Core/Tracing/TraceHelper.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 1e3dcd798..e35142e9e 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -284,7 +284,11 @@ internal static void EmitTraceActivityForTaskFailed( return null; } - activity.SetSpanId(createdEvent.ClientSpanId); + if (!string.IsNullOrEmpty(createdEvent.ClientSpanId)) + { + activity.SetSpanId(createdEvent.ClientSpanId); + + } activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); activity.SetTag(Schema.Task.Name, createdEvent.Name); From 13d56ad7424fb63c3aa79f1651c14405803acf3f Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Fri, 25 Apr 2025 15:58:31 -0700 Subject: [PATCH 32/35] getting rid of an extra line --- src/DurableTask.Core/Tracing/TraceHelper.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index e35142e9e..2c8eb17cb 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -287,7 +287,6 @@ internal static void EmitTraceActivityForTaskFailed( if (!string.IsNullOrEmpty(createdEvent.ClientSpanId)) { activity.SetSpanId(createdEvent.ClientSpanId); - } activity.SetTag(Schema.Task.Type, TraceActivityConstants.Orchestration); From 9ad0099b75123f5fc77cc454833b2d7214f4f82c Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Mon, 28 Apr 2025 14:54:22 -0700 Subject: [PATCH 33/35] addressing PR comment --- src/DurableTask.Core/TaskHubClient.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index a75bf12a5..7c67fb27a 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -738,6 +738,18 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) CorrelationTraceClient.TrackRequestTelemetry(requestTraceContext); } + /// + /// Raises an event in the specified orchestration instance, which eventually causes the OnEvent() method in the + /// orchestration to fire. + /// + /// Instance in which to raise the event + /// Name of the event + /// Data for the event + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) + { + await this.RaiseEventAsync(orchestrationInstance, eventName, eventData, entityEvent: false); + } + /// /// Raises an event in the specified orchestration instance, which eventually causes the OnEvent() method in the /// orchestration to fire. From e1725460bf6d696b738f5c0961d52405926162c0 Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Tue, 6 May 2025 16:32:39 -0700 Subject: [PATCH 34/35] addressing a few PR comments --- src/DurableTask.Core/Tracing/TraceHelper.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 2c8eb17cb..078dd9e06 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -351,7 +351,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . - /// Whether or not entities are enabled, meaning this event could possibly correspond to entity + /// Whether or not entities are enabled, meaning this event could possibly correspond to entity. /// The instance id of the orchestration that will receive the event. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. @@ -362,6 +362,7 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( bool entitiesEnabled, string? targetInstanceId) { + // We don't want to emit tracing for external events when they are related to entities if (entitiesEnabled && (Entities.IsEntityInstance(targetInstanceId ?? string.Empty) || Entities.IsEntityInstance(instance?.InstanceId ?? string.Empty))) { return null; @@ -395,12 +396,13 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . - /// Whether or not this event corresponds to an entity + /// Whether or not this event corresponds to an entity. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance, bool entityEvent) { + // We don't want to emit tracing for external events when they are related to entities if (entityEvent) { return null; From 506203027ccce194dde483ed58e5f35af4aa619a Mon Sep 17 00:00:00 2001 From: Sophia Tevosyan Date: Thu, 8 May 2025 14:30:03 -0700 Subject: [PATCH 35/35] addressing some PR comments, slightly changing the logic of the modified TaskHubClient method --- src/DurableTask.Core/TaskHubClient.cs | 16 ++++++++++++---- .../TaskOrchestrationDispatcher.cs | 2 +- src/DurableTask.Core/Tracing/TraceHelper.cs | 9 +-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/DurableTask.Core/TaskHubClient.cs b/src/DurableTask.Core/TaskHubClient.cs index 7c67fb27a..7cf2bc064 100644 --- a/src/DurableTask.Core/TaskHubClient.cs +++ b/src/DurableTask.Core/TaskHubClient.cs @@ -747,7 +747,7 @@ void CreateAndTrackDependencyTelemetry(TraceContextBase? requestTraceContext) /// Data for the event public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData) { - await this.RaiseEventAsync(orchestrationInstance, eventName, eventData, entityEvent: false); + await this.RaiseEventAsync(orchestrationInstance, eventName, eventData, emitTraceActivity: true); } /// @@ -757,8 +757,8 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s /// Instance in which to raise the event /// Name of the event /// Data for the event - /// Whether or not this event corresponds to an entity (false by default) - public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, bool entityEvent = false) + /// Whether or not to emit a trace activity for this event. + public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, string eventName, object eventData, bool emitTraceActivity = true) { if (string.IsNullOrWhiteSpace(orchestrationInstance.InstanceId)) @@ -770,7 +770,11 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s // Distributed Tracing EventRaisedEvent eventRaisedEvent = new EventRaisedEvent(-1, serializedInput) { Name = eventName }; - using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance, entityEvent); + Activity? traceActivity = null; + if (emitTraceActivity) + { + traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(eventRaisedEvent, orchestrationInstance); + } var taskMessage = new TaskMessage { @@ -789,6 +793,10 @@ public async Task RaiseEventAsync(OrchestrationInstance orchestrationInstance, s TraceHelper.AddErrorDetailsToSpan(traceActivity, e); throw; } + finally + { + traceActivity?.Dispose(); + } } /// diff --git a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs index 59d42465e..dd1663e74 100644 --- a/src/DurableTask.Core/TaskOrchestrationDispatcher.cs +++ b/src/DurableTask.Core/TaskOrchestrationDispatcher.cs @@ -1168,7 +1168,7 @@ TaskMessage ProcessCreateSubOrchestrationInstanceDecision( Version = createSubOrchestrationAction.Version }; - // If this is an orchestration triggered by an entity, we will attempt to use the trace context provided in the CreateSubOrchestrationAction.Tags as the parent trace context rather than the current Activity. + // If a parent trace context was provided via the CreateSubOrchestrationAction.Tags, we will use this as the parent trace context of the suborchestration execution Activity rather than Activity.Current.Context. if (createSubOrchestrationAction.Tags != null && createSubOrchestrationAction.Tags.TryGetValue(OrchestrationTags.TraceParent, out string traceParent)) { diff --git a/src/DurableTask.Core/Tracing/TraceHelper.cs b/src/DurableTask.Core/Tracing/TraceHelper.cs index 078dd9e06..8df8c0fa4 100644 --- a/src/DurableTask.Core/Tracing/TraceHelper.cs +++ b/src/DurableTask.Core/Tracing/TraceHelper.cs @@ -396,18 +396,11 @@ internal static void EmitTraceActivityForSubOrchestrationFailed( /// /// The associated . /// The associated . - /// Whether or not this event corresponds to an entity. /// /// Returns a newly started with (task) activity and orchestration-specific metadata. /// - internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance, bool entityEvent) + internal static Activity? StartActivityForNewEventRaisedFromClient(EventRaisedEvent eventRaised, OrchestrationInstance instance) { - // We don't want to emit tracing for external events when they are related to entities - if (entityEvent) - { - return null; - } - Activity? newActivity = ActivityTraceSource.StartActivity( CreateSpanName(TraceActivityConstants.OrchestrationEvent, eventRaised.Name, null), kind: ActivityKind.Producer,