diff --git a/src/Temporalio/Worker/WorkflowInstance.cs b/src/Temporalio/Worker/WorkflowInstance.cs index af8e289b..afb9d9cb 100644 --- a/src/Temporalio/Worker/WorkflowInstance.cs +++ b/src/Temporalio/Worker/WorkflowInstance.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Reflection; using System.Runtime.ExceptionServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -68,6 +69,7 @@ internal class WorkflowInstance : TaskScheduler, IWorkflowInstance, IWorkflowCon private readonly Action onTaskCompleted; private readonly IReadOnlyCollection? workerLevelFailureExceptionTypes; private readonly bool disableCompletionCommandReordering; + private readonly Handlers inProgressHandlers = new(); private WorkflowActivationCompletion? completion; // Will be set to null after last use (i.e. when workflow actually started) private Lazy? startArgs; @@ -204,6 +206,9 @@ public WorkflowInstance(WorkflowInstanceDetails details) /// public bool TracingEventsEnabled { get; private init; } + /// + public bool AllHandlersFinished => inProgressHandlers.Count == 0; + /// public CancellationToken CancellationToken => cancellationTokenSource.Token; @@ -576,7 +581,13 @@ public WorkflowActivationCompletion Activate(WorkflowActivation act) } // Maybe apply workflow completion command reordering logic - ApplyCompletionCommandReordering(act, completion); + ApplyCompletionCommandReordering(act, completion, out var workflowComplete); + + // Log warnings if we have completed + if (workflowComplete && !IsReplaying) + { + inProgressHandlers.WarnIfAnyLeftOver(Info.WorkflowId, logger); + } // Unset the completion var toReturn = completion; @@ -886,6 +897,10 @@ private void ApplyDoUpdate(DoUpdate update) // Queue it up so it can run in workflow environment _ = QueueNewTaskAsync(() => { + // Make sure we have loaded the instance which may invoke the constructor thereby + // letting the constructor register update handlers at runtime + var ignored = Instance; + // Set the current update for the life of this task CurrentUpdateInfoLocal.Value = new(Id: update.Id, Name: update.Name); @@ -998,9 +1013,12 @@ private void ApplyDoUpdate(DoUpdate update) Definition: updateDefn, Args: argsForUpdate, Headers: update.Headers)); + var inProgress = inProgressHandlers.AddLast(new Handlers.Handler( + update.Name, update.Id, updateDefn.UnfinishedPolicy)); return task.ContinueWith( _ => { + inProgressHandlers.Remove(inProgress); // If workflow failure exception, it's an update failure. If it's some // other exception, it's a task failure. Otherwise it's a success. var exc = task.Exception?.InnerExceptions?.SingleOrDefault(); @@ -1080,6 +1098,10 @@ private void ApplyQueryWorkflow(QueryWorkflow query) // Queue it up so it can run in workflow environment _ = QueueNewTaskAsync(() => { + // Make sure we have loaded the instance which may invoke the constructor thereby + // letting the constructor register query handlers at runtime + var ignored = Instance; + var origCmdCount = completion?.Successful?.Commands?.Count; try { @@ -1241,11 +1263,21 @@ private void ApplySignalWorkflow(SignalWorkflow signal) return; } - await inbound.Value.HandleSignalAsync(new( - Signal: signal.SignalName, - Definition: signalDefn, - Args: args, - Headers: signal.Headers)).ConfigureAwait(true); + // Handle signal + var inProgress = inProgressHandlers.AddLast(new Handlers.Handler( + signal.SignalName, null, signalDefn.UnfinishedPolicy)); + try + { + await inbound.Value.HandleSignalAsync(new( + Signal: signal.SignalName, + Definition: signalDefn, + Args: args, + Headers: signal.Headers)).ConfigureAwait(true); + } + finally + { + inProgressHandlers.Remove(inProgress); + } })); } @@ -1394,7 +1426,9 @@ private string GetStackTrace() } private void ApplyCompletionCommandReordering( - WorkflowActivation act, WorkflowActivationCompletion completion) + WorkflowActivation act, + WorkflowActivationCompletion completion, + out bool workflowComplete) { // In earlier versions of the SDK we allowed commands to be sent after workflow // completion. These ended up being removed effectively making the result of the @@ -1404,40 +1438,42 @@ private void ApplyCompletionCommandReordering( // // Note this only applies for successful activations that don't have completion // reordering disabled and that are either not replaying or have the flag set. - if (completion.Successful == null || disableCompletionCommandReordering) - { - return; - } - if (IsReplaying && !act.AvailableInternalFlags.Contains((uint)WorkflowLogicFlag.ReorderWorkflowCompletion)) - { - return; - } - // We know we're on a newer SDK and can move completion to the end if we need to. First, - // find the completion command. + // Find the index of the completion command var completionCommandIndex = -1; - for (var i = completion.Successful.Commands.Count - 1; i >= 0; i--) + if (completion.Successful != null) { - var cmd = completion.Successful.Commands[i]; - // Set completion index if the command is a completion - if (cmd.CancelWorkflowExecution != null || - cmd.CompleteWorkflowExecution != null || - cmd.ContinueAsNewWorkflowExecution != null || - cmd.FailWorkflowExecution != null) + for (var i = completion.Successful.Commands.Count - 1; i >= 0; i--) { - completionCommandIndex = i; - break; + var cmd = completion.Successful.Commands[i]; + // Set completion index if the command is a completion + if (cmd.CancelWorkflowExecution != null || + cmd.CompleteWorkflowExecution != null || + cmd.ContinueAsNewWorkflowExecution != null || + cmd.FailWorkflowExecution != null) + { + completionCommandIndex = i; + break; + } } } - - // If there is no completion command or it's already at the end, nothing to do - if (completionCommandIndex == -1 || - completionCommandIndex == completion.Successful.Commands.Count - 1) + workflowComplete = completionCommandIndex >= 0; + + // This only applies for successful activations that have a completion not at the end, + // don't have completion reordering disabled, and that are either not replaying or have + // the flag set. + if (completion.Successful == null || + completionCommandIndex == -1 || + completionCommandIndex == completion.Successful.Commands.Count - 1 || + disableCompletionCommandReordering || + (IsReplaying && !act.AvailableInternalFlags.Contains( + (uint)WorkflowLogicFlag.ReorderWorkflowCompletion))) { return; } - // Now we know the completion is in the wrong spot, so set the SDK flag and move it + // Now we know the completion is in the wrong spot and we're on a newer SDK, so set the + // SDK flag and move it completion.Successful.UsedInternalFlags.Add((uint)WorkflowLogicFlag.ReorderWorkflowCompletion); var compCmd = completion.Successful.Commands[completionCommandIndex]; completion.Successful.Commands.RemoveAt(completionCommandIndex); @@ -2230,5 +2266,86 @@ public override Task SignalAsync( public override Task CancelAsync() => instance.outbound.Value.CancelExternalWorkflowAsync(new(Id: Id, RunId: RunId)); } + + private class Handlers : LinkedList + { +#pragma warning disable SA1118 // We're ok w/ string literals spanning lines + private static readonly Action SignalWarning = + LoggerMessage.Define( + LogLevel.Warning, + 0, + "Workflow {Id} finished while signal handlers are still running. This may " + + "have interrupted work that the signal handler was doing. You can wait for " + + "all update and signal handlers to complete by using `await " + + "Workflow.WaitConditionAsync(() => Workflow.AllHandlersFinished)`. " + + "Alternatively, if both you and the clients sending the signal are okay with " + + "interrupting running handlers when the workflow finishes, and causing " + + "clients to receive errors, then you can disable this warning via the signal " + + "handler attribute: " + + "`[WorkflowSignal(UnfinishedPolicy=HandlerUnfinishedPolicy.Abandon)]`. The " + + "following signals were unfinished (and warnings were not disabled for their " + + "handler): {Handlers}"); + + private static readonly Action UpdateWarning = + LoggerMessage.Define( + LogLevel.Warning, + 0, + "Workflow {Id} finished while update handlers are still running. This may " + + "have interrupted work that the update handler was doing, and the client " + + "that sent the update will receive a 'workflow execution already completed' " + + "RpcException instead of the update result. You can wait for all update and " + + "signal handlers to complete by using `await " + + "Workflow.WaitConditionAsync(() => Workflow.AllHandlersFinished)`. " + + "Alternatively, if both you and the clients sending the update are okay with " + + "interrupting running handlers when the workflow finishes, and causing " + + "clients to receive errors, then you can disable this warning via the update " + + "handler attribute: " + + "`[WorkflowUpdate(UnfinishedPolicy=HandlerUnfinishedPolicy.Abandon)]`. The " + + "following updates were unfinished (and warnings were not disabled for their " + + "handler): {Handlers}"); +#pragma warning restore SA1118 + + public void WarnIfAnyLeftOver(string id, ILogger logger) + { + var signals = this. + Where(h => h.UpdateId == null && h.UnfinishedPolicy == HandlerUnfinishedPolicy.WarnAndAbandon). + GroupBy(h => h.Name). + Select(h => (h.Key, h.Count())). + ToArray(); + if (signals.Length > 0) + { + SignalWarning(logger, id, new WarnableSignals { NamesAndCounts = signals }, null); + } + var updates = this. + Where(h => h.UpdateId != null && h.UnfinishedPolicy == HandlerUnfinishedPolicy.WarnAndAbandon). + Select(h => (h.Name, h.UpdateId!)). + ToArray(); + if (updates.Length > 0) + { + UpdateWarning(logger, id, new WarnableUpdates { NamesAndIds = updates }, null); + } + } + + public readonly struct WarnableSignals + { + public (string, int)[] NamesAndCounts { get; init; } + + public override string ToString() => JsonSerializer.Serialize( + NamesAndCounts.Select(v => new { name = v.Item1, count = v.Item2 }).ToArray()); + } + + public readonly struct WarnableUpdates + { + public (string, string)[] NamesAndIds { get; init; } + + public override string ToString() => JsonSerializer.Serialize( + NamesAndIds.Select(v => new { name = v.Item1, id = v.Item2 }).ToArray()); + } + + public record Handler( + string Name, + string? UpdateId, + HandlerUnfinishedPolicy UnfinishedPolicy); + } } } \ No newline at end of file diff --git a/src/Temporalio/Workflows/HandlerUnfinishedPolicy.cs b/src/Temporalio/Workflows/HandlerUnfinishedPolicy.cs new file mode 100644 index 00000000..34dbf90e --- /dev/null +++ b/src/Temporalio/Workflows/HandlerUnfinishedPolicy.cs @@ -0,0 +1,27 @@ +namespace Temporalio.Workflows +{ + /// + /// Actions taken if a workflow terminates with running handlers. + /// + /// + /// Policy defining actions taken when a workflow exits while update or signal handlers are + /// running. The workflow exit may be due to successful return, failure, cancellation, or + /// continue-as-new. + /// + public enum HandlerUnfinishedPolicy + { + /// + /// Issue a warning in addition to abandoning. + /// + WarnAndAbandon, + + /// + /// Abandon the handler. + /// + /// + /// In the case of an update handler this means that the client will receive an error rather + /// than the update result. + /// + Abandon, + } +} \ No newline at end of file diff --git a/src/Temporalio/Workflows/IWorkflowContext.cs b/src/Temporalio/Workflows/IWorkflowContext.cs index ec91c01d..9e5d65c0 100644 --- a/src/Temporalio/Workflows/IWorkflowContext.cs +++ b/src/Temporalio/Workflows/IWorkflowContext.cs @@ -13,6 +13,11 @@ namespace Temporalio.Workflows /// internal interface IWorkflowContext { + /// + /// Gets a value indicating whether is true. + /// + bool AllHandlersFinished { get; } + /// /// Gets value for . /// diff --git a/src/Temporalio/Workflows/Workflow.cs b/src/Temporalio/Workflows/Workflow.cs index 7cfea076..1bea1acf 100644 --- a/src/Temporalio/Workflows/Workflow.cs +++ b/src/Temporalio/Workflows/Workflow.cs @@ -18,6 +18,16 @@ namespace Temporalio.Workflows /// public static class Workflow { + /// + /// Gets a value indicating whether all update and signal handlers have finished executing. + /// + /// + /// Consider waiting on this condition before workflow return or continue-as-new, to prevent + /// interruption of in-progress handlers by workflow return: + /// await Workflow.WaitConditionAsync(() => Workflow.AllHandlersFinished). + /// + public static bool AllHandlersFinished => Context.AllHandlersFinished; + /// /// Gets the cancellation token for the workflow. /// diff --git a/src/Temporalio/Workflows/WorkflowSignalAttribute.cs b/src/Temporalio/Workflows/WorkflowSignalAttribute.cs index 8ee58044..0125997f 100644 --- a/src/Temporalio/Workflows/WorkflowSignalAttribute.cs +++ b/src/Temporalio/Workflows/WorkflowSignalAttribute.cs @@ -43,5 +43,11 @@ public WorkflowSignalAttribute(string name) /// an array of . /// public bool Dynamic { get; set; } + + /// + /// Gets or sets the actions taken if a workflow exits with a running instance of this + /// handler. Default is . + /// + public HandlerUnfinishedPolicy UnfinishedPolicy { get; set; } = HandlerUnfinishedPolicy.WarnAndAbandon; } } diff --git a/src/Temporalio/Workflows/WorkflowSignalDefinition.cs b/src/Temporalio/Workflows/WorkflowSignalDefinition.cs index 219a0bf0..f77af280 100644 --- a/src/Temporalio/Workflows/WorkflowSignalDefinition.cs +++ b/src/Temporalio/Workflows/WorkflowSignalDefinition.cs @@ -12,11 +12,16 @@ public class WorkflowSignalDefinition { private static readonly ConcurrentDictionary Definitions = new(); - private WorkflowSignalDefinition(string? name, MethodInfo? method, Delegate? del) + private WorkflowSignalDefinition( + string? name, + MethodInfo? method, + Delegate? del, + HandlerUnfinishedPolicy unfinishedPolicy) { Name = name; Method = method; Delegate = del; + UnfinishedPolicy = unfinishedPolicy; } /// @@ -39,6 +44,11 @@ private WorkflowSignalDefinition(string? name, MethodInfo? method, Delegate? del /// internal Delegate? Delegate { get; private init; } + /// + /// Gets the unfinished policy. + /// + internal HandlerUnfinishedPolicy UnfinishedPolicy { get; private init; } + /// /// Get a signal definition from a method or fail. The result is cached. /// @@ -63,12 +73,16 @@ public static WorkflowSignalDefinition FromMethod(MethodInfo method) /// /// Signal name. Null for dynamic signal. /// Signal delegate. + /// Actions taken if a workflow exits with a running instance + /// of this handler. /// Signal definition. public static WorkflowSignalDefinition CreateWithoutAttribute( - string? name, Delegate del) + string? name, + Delegate del, + HandlerUnfinishedPolicy unfinishedPolicy = HandlerUnfinishedPolicy.WarnAndAbandon) { AssertValid(del.Method, dynamic: name == null); - return new(name, null, del); + return new(name, null, del, unfinishedPolicy); } /// @@ -103,7 +117,7 @@ private static WorkflowSignalDefinition CreateFromMethod(MethodInfo method) name = name.Substring(0, name.Length - 5); } } - return new(name, method, null); + return new(name, method, null, attr.UnfinishedPolicy); } private static void AssertValid(MethodInfo method, bool dynamic) diff --git a/src/Temporalio/Workflows/WorkflowUpdateAttribute.cs b/src/Temporalio/Workflows/WorkflowUpdateAttribute.cs index 003bb68f..b24ff648 100644 --- a/src/Temporalio/Workflows/WorkflowUpdateAttribute.cs +++ b/src/Temporalio/Workflows/WorkflowUpdateAttribute.cs @@ -47,5 +47,11 @@ public WorkflowUpdateAttribute() /// an array of . /// public bool Dynamic { get; set; } + + /// + /// Gets or sets the actions taken if a workflow exits with a running instance of this + /// handler. Default is . + /// + public HandlerUnfinishedPolicy UnfinishedPolicy { get; set; } = HandlerUnfinishedPolicy.WarnAndAbandon; } } \ No newline at end of file diff --git a/src/Temporalio/Workflows/WorkflowUpdateDefinition.cs b/src/Temporalio/Workflows/WorkflowUpdateDefinition.cs index 33ed8cbf..d18e8663 100644 --- a/src/Temporalio/Workflows/WorkflowUpdateDefinition.cs +++ b/src/Temporalio/Workflows/WorkflowUpdateDefinition.cs @@ -18,13 +18,15 @@ private WorkflowUpdateDefinition( MethodInfo? method, MethodInfo? validatorMethod, Delegate? del, - Delegate? validatorDel) + Delegate? validatorDel, + HandlerUnfinishedPolicy unfinishedPolicy) { Name = name; Method = method; ValidatorMethod = validatorMethod; Delegate = del; ValidatorDelegate = validatorDel; + UnfinishedPolicy = unfinishedPolicy; } /// @@ -57,6 +59,11 @@ private WorkflowUpdateDefinition( /// internal Delegate? ValidatorDelegate { get; private init; } + /// + /// Gets the unfinished policy. + /// + internal HandlerUnfinishedPolicy UnfinishedPolicy { get; private init; } + /// /// Get an update definition from a method or fail. The result is cached. /// @@ -92,12 +99,17 @@ public static WorkflowUpdateDefinition FromMethod( /// Update name. Null for dynamic update. /// Update delegate. /// Optional validator delegate. + /// Actions taken if a workflow exits with a running instance + /// of this handler. /// Update definition. public static WorkflowUpdateDefinition CreateWithoutAttribute( - string? name, Delegate del, Delegate? validatorDel = null) + string? name, + Delegate del, + Delegate? validatorDel = null, + HandlerUnfinishedPolicy unfinishedPolicy = HandlerUnfinishedPolicy.WarnAndAbandon) { AssertValid(del.Method, dynamic: name == null, validatorDel?.Method); - return new(name, null, null, del, validatorDel); + return new(name, null, null, del, validatorDel, unfinishedPolicy); } /// @@ -133,7 +145,7 @@ private static WorkflowUpdateDefinition CreateFromMethod( name = name.Substring(0, name.Length - 5); } } - return new(name, method, validatorMethod, null, null); + return new(name, method, validatorMethod, null, null, attr.UnfinishedPolicy); } private static void AssertValid( diff --git a/tests/Temporalio.Tests/Worker/WorkflowWorkerTests.cs b/tests/Temporalio.Tests/Worker/WorkflowWorkerTests.cs index cf22c8b0..d44fdc2b 100644 --- a/tests/Temporalio.Tests/Worker/WorkflowWorkerTests.cs +++ b/tests/Temporalio.Tests/Worker/WorkflowWorkerTests.cs @@ -5269,6 +5269,230 @@ await handle.StartUpdateAsync( new TemporalWorkerOptions().AddAllActivities(acts)); } + [Workflow] + public class UnfinishedHandlersWorkflow + { + public enum WorkflowFinish + { + Succeed, + Fail, + Cancel, + } + + private bool startedHandler; + private bool handlerMayReturn; + private bool handlerFinished; + + public UnfinishedHandlersWorkflow() + { + // Add manual update/signal handlers + Workflow.Updates["MyUpdateManual"] = WorkflowUpdateDefinition.CreateWithoutAttribute( + "MyUpdateManual", DoUpdateOrSignal, null); + Workflow.Updates["MyUpdateManualAbandon"] = WorkflowUpdateDefinition.CreateWithoutAttribute( + "MyUpdateManualAbandon", DoUpdateOrSignal, null, HandlerUnfinishedPolicy.Abandon); + Workflow.Signals["MySignalManual"] = WorkflowSignalDefinition.CreateWithoutAttribute( + "MySignalManual", DoUpdateOrSignal); + Workflow.Signals["MySignalManualAbandon"] = WorkflowSignalDefinition.CreateWithoutAttribute( + "MySignalManualAbandon", DoUpdateOrSignal, HandlerUnfinishedPolicy.Abandon); + } + + [WorkflowRun] + public async Task RunAsync(bool waitAllHandlersFinished, WorkflowFinish finish) + { + // Wait for started and finished if requested + await Workflow.WaitConditionAsync(() => startedHandler); + if (waitAllHandlersFinished) + { + handlerMayReturn = true; + await Workflow.WaitConditionAsync(() => Workflow.AllHandlersFinished); + } + + // Cancel or fail + if (finish == WorkflowFinish.Cancel) + { + await Workflow.ExecuteActivityAsync( + (Activities acts) => acts.CancelWorkflowAsync(), + new() { StartToCloseTimeout = TimeSpan.FromHours(1) }); + await Workflow.WaitConditionAsync(() => false); + } + else if (finish == WorkflowFinish.Fail) + { + throw new ApplicationFailureException("Intentional failure"); + } + + return handlerFinished; + } + + [WorkflowUpdate] + public Task MyUpdateAsync() => DoUpdateOrSignal(); + + [WorkflowUpdate(UnfinishedPolicy = HandlerUnfinishedPolicy.Abandon)] + public Task MyUpdateAbandonAsync() => DoUpdateOrSignal(); + + [WorkflowSignal] + public Task MySignalAsync() => DoUpdateOrSignal(); + + [WorkflowSignal(UnfinishedPolicy = HandlerUnfinishedPolicy.Abandon)] + public Task MySignalAbandonAsync() => DoUpdateOrSignal(); + + private async Task DoUpdateOrSignal() + { + startedHandler = true; + // Shield from cancellation + await Workflow.WaitConditionAsync(() => handlerMayReturn, default(CancellationToken)); + handlerFinished = true; + } + + public class Activities + { + private readonly ITemporalClient client; + + public Activities(ITemporalClient client) => this.client = client; + + [Activity] + public Task CancelWorkflowAsync() => + client.GetWorkflowHandle(ActivityExecutionContext.Current.Info.WorkflowId).CancelAsync(); + } + } + + [Theory] + [InlineData(UnfinishedHandlersWorkflow.WorkflowFinish.Succeed)] + [InlineData(UnfinishedHandlersWorkflow.WorkflowFinish.Fail)] + [InlineData(UnfinishedHandlersWorkflow.WorkflowFinish.Cancel)] + public async Task ExecuteWorkflowAsync_UnfinishedHandlers_WarnProperly( + UnfinishedHandlersWorkflow.WorkflowFinish finish) + { + async Task AssertWarnings( + Func, Task> interaction, + bool waitAllHandlersFinished, + bool shouldWarn, + bool interactionShouldFailWithNotFound = false) + { + // Setup log capture + var loggerFactory = new TestUtils.LogCaptureFactory(LoggerFactory); + // Run the workflow + var acts = new UnfinishedHandlersWorkflow.Activities(Client); + await ExecuteWorkerAsync( + async worker => + { + // Start workflow + var handle = await Client.StartWorkflowAsync( + (UnfinishedHandlersWorkflow wf) => wf.RunAsync(waitAllHandlersFinished, finish), + new(id: $"wf-{Guid.NewGuid()}", taskQueue: worker.Options.TaskQueue!)); + // Perform interaction + try + { + await interaction(handle); + Assert.False(interactionShouldFailWithNotFound); + } + catch (RpcException e) when (e.Code == RpcException.StatusCode.NotFound) + { + Assert.True(interactionShouldFailWithNotFound); + } + // Wait for workflow completion + try + { + await handle.GetResultAsync(); + Assert.Equal(UnfinishedHandlersWorkflow.WorkflowFinish.Succeed, finish); + } + catch (WorkflowFailedException e) when ( + e.InnerException is ApplicationFailureException appEx && + appEx.Message == "Intentional failure") + { + Assert.Equal(UnfinishedHandlersWorkflow.WorkflowFinish.Fail, finish); + } + catch (WorkflowFailedException e) when ( + e.InnerException is CanceledFailureException) + { + Assert.Equal(UnfinishedHandlersWorkflow.WorkflowFinish.Cancel, finish); + } + }, + new TemporalWorkerOptions() { LoggerFactory = loggerFactory }. + AddAllActivities(acts)); + // Check warnings + Assert.Equal( + shouldWarn ? 1 : 0, + loggerFactory.Logs. + Select(e => e.Formatted). + // We have to dedupe logs because cancel causes unhandled command which causes + // workflow complete to replay + Distinct(). + Count(s => s.Contains("handlers are still running"))); + } + + // All update scenarios + await AssertWarnings( + interaction: h => h.ExecuteUpdateAsync(wf => wf.MyUpdateAsync()), + waitAllHandlersFinished: false, + shouldWarn: true, + interactionShouldFailWithNotFound: true); + await AssertWarnings( + interaction: h => h.ExecuteUpdateAsync(wf => wf.MyUpdateAsync()), + waitAllHandlersFinished: true, + shouldWarn: false); + await AssertWarnings( + interaction: h => h.ExecuteUpdateAsync(wf => wf.MyUpdateAbandonAsync()), + waitAllHandlersFinished: false, + shouldWarn: false, + interactionShouldFailWithNotFound: true); + await AssertWarnings( + interaction: h => h.ExecuteUpdateAsync(wf => wf.MyUpdateAbandonAsync()), + waitAllHandlersFinished: true, + shouldWarn: false); + await AssertWarnings( + interaction: h => h.ExecuteUpdateAsync("MyUpdateManual", Array.Empty()), + waitAllHandlersFinished: false, + shouldWarn: true, + interactionShouldFailWithNotFound: true); + await AssertWarnings( + interaction: h => h.ExecuteUpdateAsync("MyUpdateManual", Array.Empty()), + waitAllHandlersFinished: true, + shouldWarn: false); + await AssertWarnings( + interaction: h => h.ExecuteUpdateAsync("MyUpdateManualAbandon", Array.Empty()), + waitAllHandlersFinished: false, + shouldWarn: false, + interactionShouldFailWithNotFound: true); + await AssertWarnings( + interaction: h => h.ExecuteUpdateAsync("MyUpdateManualAbandon", Array.Empty()), + waitAllHandlersFinished: true, + shouldWarn: false); + + // All signal scenarios + await AssertWarnings( + interaction: h => h.SignalAsync(wf => wf.MySignalAsync()), + waitAllHandlersFinished: false, + shouldWarn: true); + await AssertWarnings( + interaction: h => h.SignalAsync(wf => wf.MySignalAsync()), + waitAllHandlersFinished: true, + shouldWarn: false); + await AssertWarnings( + interaction: h => h.SignalAsync(wf => wf.MySignalAbandonAsync()), + waitAllHandlersFinished: false, + shouldWarn: false); + await AssertWarnings( + interaction: h => h.SignalAsync(wf => wf.MySignalAbandonAsync()), + waitAllHandlersFinished: true, + shouldWarn: false); + await AssertWarnings( + interaction: h => h.SignalAsync("MySignalManual", Array.Empty()), + waitAllHandlersFinished: false, + shouldWarn: true); + await AssertWarnings( + interaction: h => h.SignalAsync("MySignalManual", Array.Empty()), + waitAllHandlersFinished: true, + shouldWarn: false); + await AssertWarnings( + interaction: h => h.SignalAsync("MySignalManualAbandon", Array.Empty()), + waitAllHandlersFinished: false, + shouldWarn: false); + await AssertWarnings( + interaction: h => h.SignalAsync("MySignalManualAbandon", Array.Empty()), + waitAllHandlersFinished: true, + shouldWarn: false); + } + internal static Task AssertTaskFailureContainsEventuallyAsync( WorkflowHandle handle, string messageContains) {