diff --git a/core/src/main/java/com/google/adk/agents/Context.java b/core/src/main/java/com/google/adk/agents/Context.java index a046498dd..0dd89a80b 100644 --- a/core/src/main/java/com/google/adk/agents/Context.java +++ b/core/src/main/java/com/google/adk/agents/Context.java @@ -18,6 +18,7 @@ import com.google.adk.annotations.Experimental; import com.google.adk.artifacts.ListArtifactsResponse; +import com.google.adk.events.Event; import com.google.adk.events.EventActions; import com.google.adk.events.ToolConfirmation; import com.google.adk.memory.SearchMemoryResponse; @@ -29,7 +30,9 @@ import com.google.adk.workflow.Route; import com.google.adk.workflow.RunNodeConfig; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.google.genai.types.Content; import com.google.genai.types.Part; import io.reactivex.rxjava3.core.Completable; import io.reactivex.rxjava3.core.Maybe; @@ -37,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CancellationException; import org.jspecify.annotations.Nullable; /** @@ -345,6 +349,45 @@ public final int attemptCount() { return requireNodeState().attemptCount(); } + /** + * Sends {@code event} from this workflow node's {@code FunctionNode} body, after the events of + * earlier calls and before the output. It returns once the run has stored the event and handed it + * to the run's caller, and any thread of the body may call it. The {@link + * com.google.adk.workflow.FunctionNode} documentation gives the rules for state, threads, events, + * cancellation and resumes. + * + * @throws IllegalArgumentException if {@code event} has an output, or has the ID of an event + * another {@code emit} call still waits on + * @throws IllegalStateException if no {@code FunctionNode} body of this node is running + * @throws CancellationException if the node is canceled, or the calling thread is interrupted + * while this call waits; the thread stays interrupted + */ + @Experimental + public final void emit(Event event) { + Preconditions.checkNotNull(event); + if (Strings.isNullOrEmpty(event.id())) { + event = event.toBuilder().id(invocationContext.newUuid()).build(); + } + if (nodeState == null || !nodeState.emitFromBody(event)) { + throw new IllegalStateException( + "Context.emit works only while the FunctionNode body of this context's node runs; a" + + " custom node sends events as items of its Flowable, and a tool or callback" + + " returns its result."); + } + } + + /** Sends {@code message} as an event without an output, as {@link #emit(Event)} does. */ + @Experimental + public final void emit(Content message) { + Preconditions.checkNotNull(message); + emit( + Event.builder() + .author("") + .timestamp(invocationContext.now().toEpochMilli()) + .content(message) + .build()); + } + /** * Runs {@code node} as a child of this workflow node without an input, as {@link #runNode(Node, * Object, RunNodeConfig)} does with the default {@link RunNodeConfig}. @@ -372,9 +415,9 @@ public final Maybe runNode(Node node, @Nullable Object nodeInput) { /** * Returns a {@link Maybe} that, when subscribed, runs {@code node} as a child of this workflow * node and emits the child's output, or completes empty if it has none. Include it in this node's - * run and catch only failures you handle, such as {@link DynamicNodeFailedException}: when the - * child waits, it fails with an internal {@link RuntimeException} that must propagate to end this - * node's run, so that it waits too. + * run: when the child waits, the {@code Maybe} fails with an internal {@link Error} so that this + * node waits too, which {@code catch (Exception e)} lets through but {@code catch (Throwable t)} + * and Rx error handlers such as {@code onErrorResumeNext} stop. * *

It fails with {@link DynamicNodeFailedException} if the child fails; with {@link * IllegalStateException} if this is not a workflow node's context, if this node's {@link diff --git a/core/src/main/java/com/google/adk/agents/InvocationContext.java b/core/src/main/java/com/google/adk/agents/InvocationContext.java index b04dfee16..23211ba8e 100644 --- a/core/src/main/java/com/google/adk/agents/InvocationContext.java +++ b/core/src/main/java/com/google/adk/agents/InvocationContext.java @@ -370,8 +370,8 @@ public String userId() { * Returns the {@link Scheduler} on which this invocation runs the work ADK hands to worker * threads: the sub-agents of a {@code ParallelAgent}, tool calls in {@code * ToolExecutionMode.PARALLEL_SUBSCRIBE} mode, workflow node retries and attempts with a timeout, - * the bodies of workflow {@code FunctionNode}s, and the hop that starts the live send loop. An - * agent-level setting ({@code ParallelAgent.Builder#scheduler}, {@code + * the hand-back once a workflow {@code FunctionNode} body returns, and the hop that starts the + * live send loop. An agent-level setting ({@code ParallelAgent.Builder#scheduler}, {@code * LlmAgent.Builder#executor}) takes precedence over it. Defaults to {@link Schedulers#io()}. */ public Scheduler scheduler() { diff --git a/core/src/main/java/com/google/adk/runner/Runner.java b/core/src/main/java/com/google/adk/runner/Runner.java index 0387a9039..203559400 100644 --- a/core/src/main/java/com/google/adk/runner/Runner.java +++ b/core/src/main/java/com/google/adk/runner/Runner.java @@ -185,15 +185,17 @@ public Builder plugins(Plugin... plugins) { * Sets the {@link Scheduler} for the work ADK hands off to worker threads in this runner's * invocations: the sub-agents of a {@code ParallelAgent}, tool calls in {@link * RunConfig.ToolExecutionMode#PARALLEL_SUBSCRIBE} mode, workflow node retries and attempts with - * a timeout, the bodies of workflow {@code FunctionNode}s, and the hop that starts the live - * send loop. The runs that an {@code AgentTool} nests inherit it. An agent-level {@code - * ParallelAgent.Builder#scheduler} or {@code LlmAgent.Builder#executor} takes precedence. - * {@code null} (the default) selects {@code Schedulers.io()}. + * a timeout, the hand-back once a workflow {@code FunctionNode} body returns, and the hop that + * starts the live send loop. The runs that an {@code AgentTool} nests inherit it. An + * agent-level {@code ParallelAgent.Builder#scheduler} or {@code LlmAgent.Builder#executor} + * takes precedence. {@code null} (the default) selects {@code Schedulers.io()}. * - *

Only these hand-offs are affected; work that completes asynchronously elsewhere (model - * clients, session services, MCP, plugins) stays on its own threads. {@code - * Schedulers.trampoline()} runs the hand-offs inline, so sub-agents and tools are subscribed - * sequentially, but anything they start asynchronously can still overlap. + *

Only these hand-offs are affected: work that completes asynchronously elsewhere (model + * clients, session services, MCP, plugins) stays on its own threads, and a workflow {@code + * FunctionNode} runs its body on {@code Schedulers.io()} unless {@code + * FunctionNode.Builder#scheduler} sets another. {@code Schedulers.trampoline()} runs the + * hand-offs inline, so sub-agents and tools are subscribed sequentially, but anything they + * start asynchronously can still overlap. */ @CanIgnoreReturnValue public Builder scheduler(@Nullable Scheduler scheduler) { diff --git a/core/src/main/java/com/google/adk/workflow/FunctionNode.java b/core/src/main/java/com/google/adk/workflow/FunctionNode.java index eec246f56..e8e706343 100644 --- a/core/src/main/java/com/google/adk/workflow/FunctionNode.java +++ b/core/src/main/java/com/google/adk/workflow/FunctionNode.java @@ -24,19 +24,29 @@ import com.google.adk.annotations.Experimental; import com.google.adk.events.Event; import com.google.adk.events.ToolConfirmation; +import com.google.adk.telemetry.Tracing; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.errorprone.annotations.concurrent.GuardedBy; import com.google.genai.types.Schema; +import io.reactivex.rxjava3.core.BackpressureStrategy; +import io.reactivex.rxjava3.core.Completable; import io.reactivex.rxjava3.core.Flowable; +import io.reactivex.rxjava3.core.FlowableEmitter; import io.reactivex.rxjava3.core.Maybe; -import io.reactivex.rxjava3.core.MaybeEmitter; +import io.reactivex.rxjava3.core.Scheduler; import io.reactivex.rxjava3.schedulers.Schedulers; import io.reactivex.rxjava3.subjects.CompletableSubject; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; +import java.util.concurrent.Phaser; import org.jspecify.annotations.Nullable; /** @@ -48,11 +58,36 @@ * same value whether it comes live or from a stored session. A body that gives its output to a * child with {@code useAsOutput} outputs the child's output written as {@code O}, as if it had * returned it. + * + *

The body may send events before its output with {@link Context#emit(Event)}: + * + *

+ * + *

Only the workflow runner runs a {@code FunctionNode}; from another node's body, run it with + * {@code ctx.runNode}, since calling {@link #runNode} directly fails with an {@code + * IllegalStateException}. */ @Experimental public final class FunctionNode extends BaseNode { - private static final PortCodec ANY = + static final PortCodec ANY = PortCodec.output(PortCodec.javaType(Object.class), "an event output"); private static final ConfirmationPredicate ALWAYS_CONFIRM = (context, input) -> true; @@ -66,6 +101,7 @@ public final class FunctionNode extends BaseNode { private final NodeBody body; private final @Nullable ConfirmationPredicate confirmationPredicate; private final @Nullable Boolean explicitRerunOnResume; + private final @Nullable Scheduler scheduler; private FunctionNode(Builder builder) { super( @@ -84,6 +120,7 @@ private FunctionNode(Builder builder) { this.body = checkNotNull(builder.body); this.confirmationPredicate = builder.confirmationPredicate; this.explicitRerunOnResume = builder.rerunOnResume; + this.scheduler = builder.scheduler; } /** @@ -103,6 +140,7 @@ public Builder toBuilder() { .inputSchema(super.inputSchema().orElse(null)) .outputSchema(super.outputSchema().orElse(null)) .stateSchema(stateSchema().orElse(null)) + .scheduler(scheduler) .body(body); } @@ -151,51 +189,90 @@ boolean edgeInputSchemaInferred() { @Override public Flowable runNode(Context context, @Nullable Object nodeInput) { - // Maybe.create rather than defer, so that canceling the node reaches the body while it runs. - return Maybe.create( - emitter -> { - // A timeout fails the node only once the body returns, so a retry cannot overlap it. - CompletableSubject returned = CompletableSubject.create(); - NodeExecutionState state = context.requireNodeState(); - // A body run under another node's context must not replace that node's signal. - if (state.node() == this) { - // Not on the body's thread, which may be interrupted, nor the Runner's busy pool. - state.markBodyStarted(returned.observeOn(Schedulers.computation())); - } - try { - runBody(context, nodeInput, emitter); - } finally { - returned.onComplete(); - } - }) - .subscribeOn(context.invocationContext().scheduler()) - .toFlowable(); + // defer, so that each subscription has its own body run and keeps its caller's trace context. + return Flowable.defer(() -> runOnce(context, nodeInput)); + } + + private Flowable runOnce(Context context, @Nullable Object nodeInput) { + io.opentelemetry.context.Context traceContext = io.opentelemetry.context.Context.current(); + Scheduler bodyScheduler = scheduler != null ? scheduler : Schedulers.io(); + Scheduler runnerScheduler = context.invocationContext().scheduler(); + BodyRun run = new BodyRun(); + NodeExecutionState state = context.requireNodeState(); + // Only the runner gives a run a state of its own; claimed on subscribe, before any user code. + if (state.node() != this || !state.claimBodyRun(run)) { + return Flowable.error( + new IllegalStateException( + "Node '" + + name() + + "' runs only through the workflow runner; to run it from a node's body, call" + + " ctx.runNode.")); + } + // create rather than defer, so that canceling the node reaches the body while it runs. + Flowable outcome = + Flowable.create( + emitter -> { + // The timeout waits for the body to return, so a retry cannot overlap it. + CompletableSubject returned = CompletableSubject.create(); + // Not on the body's thread, which may be interrupted, or the busy Runner. + state.markBodyStarted(returned.observeOn(Schedulers.computation())); + try { + // Serialized, as the body's helper threads may emit at the same time. + runBody(context, nodeInput, run, emitter.serialize()); + } finally { + returned.onComplete(); + } + }, + BackpressureStrategy.BUFFER) + .compose(Tracing.withContext(traceContext)); + // Unlike subscribeOn, canceling this hop does not interrupt the handling of the body's output. + Flowable events = + // Via a Maybe, as andThen(Publisher) can interrupt the thread on a cancel. + Completable.complete() + .observeOn(bodyScheduler) + .andThen(Maybe.just(outcome)) + .flatMapPublisher(f -> f); + // Back on the Runner's scheduler, through an Executor so that a cancel cannot interrupt it. + return (bodyScheduler.equals(runnerScheduler) + ? events + : events + .observeOn(Schedulers.from(runnerScheduler::scheduleDirect), /* delayError= */ true) + .compose(Tracing.withContext(traceContext))) + // A cancel ends the waits; after completion, each wait ends when its event is stored. + .doOnCancel(run::cancel); } /** - * Runs the confirmation predicate and the body on the calling thread and sends the output or - * failure to {@code emitter}. Canceling the node interrupts that thread. + * Runs the confirmation predicate and the body on the calling thread and sends the body's events, + * then its outcome, to {@code events}. Canceling the node interrupts that thread. */ - private void runBody(Context context, @Nullable Object nodeInput, MaybeEmitter emitter) { - Worker> worker = new Worker<>(() -> runFunction(context, nodeInput, emitter)); - emitter.setCancellable(worker::cancel); + private void runBody( + Context context, @Nullable Object nodeInput, BodyRun run, FlowableEmitter events) { + Worker> worker = + new Worker<>(() -> { + try { + return runFunction(context, nodeInput, run, events); + } finally { + run.close(); + } + }); + events.setCancellable(worker::cancel); Object output; try { RunOutcome outcome = worker.run(); - if (outcome == null || emitter.isDisposed()) { + if (outcome == null || events.isCancelled()) { return; } output = outcome.replacement() != null ? outcome.replacement() : finishRun(context, outcome); - } catch (Exception e) { - // tryOnError, as a canceled run can no longer take a failure. - emitter.tryOnError(e); + } catch (Exception | NodeInterruptedException e) { + // tryOnError, as a canceled run can no longer take a failure or a pause. + events.tryOnError(e); return; } - if (output == null) { - emitter.onComplete(); - } else { - emitter.onSuccess(output); + if (output != null) { + events.onNext(output); } + events.onComplete(); } /** @@ -203,7 +280,8 @@ private void runBody(Context context, @Nullable Object nodeInput, MaybeEmitter runFunction( - Context context, @Nullable Object nodeInput, MaybeEmitter emitter) throws Exception { + Context context, @Nullable Object nodeInput, BodyRun run, FlowableEmitter events) + throws Exception { String what = "input of node '" + name() + "'"; I input = inputCodec.decode(nodeInput, what); ToolConfirmation confirmation = @@ -214,7 +292,7 @@ private void runBody(Context context, @Nullable Object nodeInput, MaybeEmitter(confirmation, REJECTED_OUTPUT, null); } } else if (confirmationPredicate != null - && !emitter.isDisposed() + && !events.isCancelled() && confirmationPredicate.test(context, input)) { RequestInput request = ToolConfirmations.request( @@ -225,7 +303,8 @@ private void runBody(Context context, @Nullable Object nodeInput, MaybeEmitter(null, request, null); } - return emitter.isDisposed() + run.open(events); + return events.isCancelled() ? null : new RunOutcome<>(confirmation, null, body.apply(context, input)); } @@ -301,8 +380,8 @@ private void encodeDelegatedOutput(NodeExecutionState state) { throw new IllegalArgumentException( "node '" + name() - + "' returned a RequestInput, which a NodeBody cannot send; return the output" - + " instead."); + + "' returned a RequestInput, which a NodeBody cannot send; call" + + " FunctionNodes.requestInput(...) instead."); } if (result instanceof Event event) { // A returned event's output is sent in JSON form too, as a stored session hands it back. @@ -352,6 +431,96 @@ void cancel() { } } + /** Delivers the events of one body run's {@code Context.emit} calls. */ + static final class BodyRun { + // Counts the calls from admission to the end of their send; close waits for them. + private final Phaser sending = new Phaser(1); + + @GuardedBy("this") + private final Map> pending = new HashMap<>(); + + @GuardedBy("this") + private @Nullable FlowableEmitter events; + + @GuardedBy("this") + private boolean canceled; + + synchronized void open(FlowableEmitter events) { + this.events = events; + } + + /** + * Sends {@code event} and waits for its store unless it is partial. Returns false once the body + * has returned, and throws a CancellationException once the node is canceled. + */ + boolean emit(Event event) { + boolean partial = event.partial().orElse(false); + CompletableFuture processed = new CompletableFuture<>(); + FlowableEmitter sink; + synchronized (this) { + if (canceled) { + throw new CancellationException("The node was canceled."); + } + if (events == null) { + return false; + } + checkArgument( + event.output().isEmpty() + && !event.nodeInfo().map(NodeInfo::messageAsOutput).orElse(false), + "Context.emit takes an event without an output; return the value from the body."); + checkArgument( + partial || pending.putIfAbsent(event.id(), processed) == null, + "Another Context.emit call still waits on an event with this ID."); + sending.register(); + sink = events; + } + try { + sink.onNext(event); // Outside the lock: the downstream may run on this thread. + } finally { + sending.arriveAndDeregister(); + } + if (!partial) { + awaitStore(processed); + } + return true; + } + + /** Waits until the run stores the event; a cancel or an interrupt ends the wait. */ + private static void awaitStore(CompletableFuture processed) { + try { + processed.get(); // Throws CancellationException once the node is canceled. + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new CancellationException("Interrupted while waiting for the event's store."); + } catch (ExecutionException e) { + throw new IllegalStateException(e); // Unreachable: the signal never fails. + } + } + + /** Stops admitting events, then waits until each admitted call has handed its event on. */ + void close() { + synchronized (this) { + events = null; + } + // Afterwards serialize() has sent or queued every admitted event, so all precede the output. + sending.arriveAndAwaitAdvance(); + } + + /** Ends each wait with a CancellationException, and makes every later call throw one. */ + synchronized void cancel() { + canceled = true; + pending.values().forEach(signal -> signal.cancel(/* mayInterruptIfRunning= */ false)); + pending.clear(); + } + + synchronized void markProcessed(@Nullable String eventId) { + CompletableFuture signal = pending.remove(eventId); + if (signal != null) { + signal.complete(null); + } + } + } + /** * Fluent builder for {@link FunctionNode}. Any property left unset falls back to the default of * the matching {@link Node} property, except that {@code rerunOnResume} defaults to true for a @@ -370,6 +539,7 @@ public static final class Builder { private @Nullable Schema outputSchema; private @Nullable Schema stateSchema; private @Nullable NodeBody body; + private @Nullable Scheduler scheduler; Builder(PortCodec inputCodec, PortCodec outputCodec) { this.inputCodec = inputCodec; @@ -392,7 +562,10 @@ public Builder description(String description) { /** * Sets whether a resume runs the node again rather than using the resuming answer. Unset, it is - * true for a node that requires confirmation and false otherwise. + * true for a node that requires confirmation and false otherwise. A body that runs child nodes + * through {@link Context#runNode(Node, Object, RunNodeConfig)} requires {@code true}: on resume + * the body runs again from the start, a child that asked resumes with its answer, and the other + * children resume as {@link RunNodeConfig#runId} describes. */ @CanIgnoreReturnValue public Builder rerunOnResume(boolean rerunOnResume) { @@ -472,6 +645,19 @@ public Builder body(NodeBody body) { return this; } + /** + * Sets the scheduler the body runs on, or {@code null} (the default) for {@code + * Schedulers.io()}. A body that waits for other work needs a thread of its own, so an inline or + * small fixed-size scheduler can hang the run. Once the body returns, its outcome is handed + * back on the {@code Runner}'s scheduler, or stays on the body's thread if the {@code Runner}'s + * scheduler runs work inline. + */ + @CanIgnoreReturnValue + public Builder scheduler(@Nullable Scheduler scheduler) { + this.scheduler = scheduler; + return this; + } + /** * Builds the node. * diff --git a/core/src/main/java/com/google/adk/workflow/FunctionNodes.java b/core/src/main/java/com/google/adk/workflow/FunctionNodes.java index 2bb78b25f..659c72d2b 100644 --- a/core/src/main/java/com/google/adk/workflow/FunctionNodes.java +++ b/core/src/main/java/com/google/adk/workflow/FunctionNodes.java @@ -16,14 +16,18 @@ package com.google.adk.workflow; +import static com.google.common.base.Preconditions.checkNotNull; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JavaType; import com.google.adk.annotations.Experimental; +import com.google.genai.types.Schema; +import org.jspecify.annotations.Nullable; /** * Creates {@link FunctionNode}s with a type per port, as a {@code Class} or, for a generic type * such as {@code List}, a {@code TypeReference}, read and written as {@link FunctionNode} - * describes. + * describes. A node's body asks the user for input with {@link #requestInput}. */ @Experimental public final class FunctionNodes { @@ -32,7 +36,8 @@ public final class FunctionNodes { /** * Returns a node named {@code name} that runs {@code body} on each input read as {@code - * inputType} and writes its {@code outputType} output in JSON form. + * inputType} and writes its {@code outputType} output in JSON form. The body may send events + * first with {@link com.google.adk.agents.Context#emit}. * * @throws WorkflowConfigurationException if Jackson cannot read and write {@code inputType} or * write {@code outputType}, or if a port type has no single concrete JSON shape or is an @@ -105,6 +110,82 @@ private static FunctionNode.Builder builder( PortCodec.output(outputType, "the output of " + node)); } + /** + * Asks the user for input; it never returns. Unless the node reruns on resume, the answer, + * checked against any {@code responseSchema}, becomes the node's output without being read as its + * output type; if it reruns, the body runs again and must ask only when {@code + * ctx.resumeInputs()} lacks the answer: + * + *
{@code
+   * if (!ctx.resumeInputs().containsKey("q")) {
+   *   return FunctionNodes.requestInput("q", "How many?");
+   * }
+   * Object answer = ctx.resumeInputs().get("q");
+   * }
+ * + *

Call it on the body's thread: it ends the body by throwing an internal {@link Error}, which + * {@code catch (Exception e)} lets through and {@code catch (Throwable t)} stops. + * + * @param interruptId the ID the answer carries; keep it stable across reruns + * @param message the text to show the user, or {@code null} + * @param payload data for rendering the request, written in JSON form as a node output is, or + * {@code null} + * @param responseSchema the schema the answer must match, or {@code null} + * @throws IllegalArgumentException if the payload has no JSON form + */ + @SuppressWarnings("TypeParameterUnusedInFormals") // T only lets a body return the call. + public static T requestInput( + String interruptId, + @Nullable String message, + @Nullable Object payload, + @Nullable Schema responseSchema) { + return requestInput( + RequestInput.builder() + .interruptId(interruptId) + .message(message) + .payload(payload) + .responseSchema(responseSchema) + .build()); + } + + /** Like {@link #requestInput(String, String, Object, Schema)}, without a response schema. */ + @SuppressWarnings("TypeParameterUnusedInFormals") + public static T requestInput( + String interruptId, @Nullable String message, @Nullable Object payload) { + return requestInput(interruptId, message, payload, /* responseSchema= */ null); + } + + /** Like {@link #requestInput(String, String, Object, Schema)}, with only a message. */ + @SuppressWarnings("TypeParameterUnusedInFormals") + public static T requestInput(String interruptId, @Nullable String message) { + return requestInput(interruptId, message, /* payload= */ null, /* responseSchema= */ null); + } + + /** Like {@link #requestInput(String, String, Object, Schema)}, with neither message nor data. */ + @SuppressWarnings("TypeParameterUnusedInFormals") + public static T requestInput(String interruptId) { + return requestInput( + interruptId, /* message= */ null, /* payload= */ null, /* responseSchema= */ null); + } + + /** Like {@link #requestInput(String, String, Object, Schema)}, with a random interrupt ID. */ + @SuppressWarnings("TypeParameterUnusedInFormals") + public static T requestInput() { + return requestInput(RequestInput.builder().build()); + } + + /** Like {@link #requestInput(String, String, Object, Schema)}, for a built {@code request}. */ + @SuppressWarnings("TypeParameterUnusedInFormals") + public static T requestInput(RequestInput request) { + Object payload = checkNotNull(request).payload().orElse(null); + throw new NodeInterruptedException( + payload == null + ? request + : request.toBuilder() + .payload(FunctionNode.ANY.encode(payload, "payload of requestInput")) + .build()); + } + private static FunctionNode node( String name, JavaType inputType, JavaType outputType, NodeBody body) { return FunctionNodes.builder(inputType, outputType, "node '" + name + "'") diff --git a/core/src/main/java/com/google/adk/workflow/Node.java b/core/src/main/java/com/google/adk/workflow/Node.java index 752df2f55..bb7870d94 100644 --- a/core/src/main/java/com/google/adk/workflow/Node.java +++ b/core/src/main/java/com/google/adk/workflow/Node.java @@ -102,7 +102,9 @@ default boolean requiresAllPredecessors() { } /** - * Runs the node on {@code nodeInput}, the output it receives from its predecessors. + * Runs the node on {@code nodeInput}, the output it receives from its predecessors. The workflow + * runner calls this method; to run a node from another node's body, call {@link + * Context#runNode(Node, Object)}. * *

An emitted {@link Event} passes through, an emitted {@link RequestInput} pauses the workflow * until the user answers it, and any other emitted value becomes the node's output (at most one diff --git a/core/src/main/java/com/google/adk/workflow/NodeBody.java b/core/src/main/java/com/google/adk/workflow/NodeBody.java index 90c65c71a..4c48e3b09 100644 --- a/core/src/main/java/com/google/adk/workflow/NodeBody.java +++ b/core/src/main/java/com/google/adk/workflow/NodeBody.java @@ -22,11 +22,10 @@ /** * The body of a {@link FunctionNode}, which takes the node run's context and its single input; the - * input may be shared with other nodes, so the body must not mutate it. It runs on the {@code - * Runner}'s scheduler, so it may block, and a small fixed-size scheduler can deadlock bodies that - * wait for one another. Canceling the node interrupts the body's thread and drops its result; only - * the node's own timeout waits for the body to return, so a body that ignores the interrupt holds - * up the retry and the run. + * input may be shared with other nodes, so the body must not mutate it. It runs on the node's + * scheduler ({@link FunctionNode.Builder#scheduler}), so it may block. Canceling the node + * interrupts the body's thread and drops its result; only the node's own timeout waits for the body + * to return, so a body that ignores the interrupt holds up the retry and the run. */ @Experimental @FunctionalInterface @@ -36,7 +35,8 @@ public interface NodeBody { * Returns {@code null} for no output, an {@code Event} to send with any output in JSON form, or * the output, which is sent in its JSON form; a returned {@code Content} is the output, not a * message. {@code input} is null when the predecessor sent no output or the port is {@code Void}; - * a primitive port fails with {@link NodeInputValidationException} instead. + * a primitive port fails with {@link NodeInputValidationException} instead. To ask the user for + * input, call {@link FunctionNodes#requestInput}. */ @Nullable O apply(Context context, @Nullable I input) throws Exception; } diff --git a/core/src/main/java/com/google/adk/workflow/NodeExecutionState.java b/core/src/main/java/com/google/adk/workflow/NodeExecutionState.java index 191aca877..6faff90c3 100644 --- a/core/src/main/java/com/google/adk/workflow/NodeExecutionState.java +++ b/core/src/main/java/com/google/adk/workflow/NodeExecutionState.java @@ -16,12 +16,15 @@ package com.google.adk.workflow; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Strings.isNullOrEmpty; import com.google.adk.agents.Context; import com.google.adk.agents.InvocationContext; import com.google.adk.annotations.Experimental; +import com.google.adk.events.Event; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -83,6 +86,7 @@ public final class NodeExecutionState { private @Nullable Exception failure; private @Nullable String failureNodePath; private @Nullable ImmutableMap recoveredChildren; + private final AtomicReference bodyRun = new AtomicReference<>(); private NodeExecutionState(Builder builder) { checkState(builder.node != null, "A node state needs a node."); @@ -252,6 +256,33 @@ NodeRunner.EventSink eventSink() { return eventSink; } + /** + * Sends {@code event}, which needs an ID, on behalf of {@link Context#emit(Event)}; false if no + * body runs. + */ + public boolean emitFromBody(Event event) { + checkArgument(!isNullOrEmpty(checkNotNull(event).id()), "Context.emit needs an event ID."); + FunctionNode.BodyRun run = bodyRun.get(); + return run != null && run.emit(event); + } + + /** + * Makes {@code run} the body run that {@link #emitFromBody} sends to; false if another run + * claimed this state first. + */ + boolean claimBodyRun(FunctionNode.BodyRun run) { + return bodyRun.compareAndSet(null, run); + } + + /** Releases the {@link #emitFromBody} call that waits for {@code event}, if any. */ + void markProcessed(Event event) { + FunctionNode.BodyRun run = bodyRun.get(); + // A partial event never waits, so it must not release a later event with the same ID. + if (run != null && !event.partial().orElse(false)) { + run.markProcessed(event.id()); + } + } + /** * Returns the paths of the nodes that use this run's output as their output, starting with this * run's path. diff --git a/core/src/main/java/com/google/adk/workflow/NodeInterruptedException.java b/core/src/main/java/com/google/adk/workflow/NodeInterruptedException.java index 026b66f7f..a83662f7f 100644 --- a/core/src/main/java/com/google/adk/workflow/NodeInterruptedException.java +++ b/core/src/main/java/com/google/adk/workflow/NodeInterruptedException.java @@ -17,16 +17,29 @@ package com.google.adk.workflow; import com.google.adk.annotations.Experimental; +import org.jspecify.annotations.Nullable; /** - * A signal that ends a node's run when a child it ran with {@link NodeRunner#runChild} waits, so - * the node waits too. Node code that catches this signal (for example, by catching every {@link - * RuntimeException}) keeps running past the wait, and anything it then emits is still recorded. + * A signal that ends a node's run while it waits for input: thrown by {@link + * FunctionNodes#requestInput} with its request, or when a child the node ran with {@link + * NodeRunner#runChild} waits, so the node waits too. A pause is not an {@link Exception}, so {@code + * catch (Exception e)} in a body does not stop it; {@code catch (Throwable t)} still does. */ @Experimental -final class NodeInterruptedException extends RuntimeException { +final class NodeInterruptedException extends Error { + private final @Nullable RequestInput request; NodeInterruptedException() { + this(null); + } + + NodeInterruptedException(@Nullable RequestInput request) { super("Node interrupted."); + this.request = request; + } + + /** Returns the request for input to send, or null when a child waits. */ + @Nullable RequestInput request() { + return request; } } diff --git a/core/src/main/java/com/google/adk/workflow/NodeRunner.java b/core/src/main/java/com/google/adk/workflow/NodeRunner.java index 14f004d27..9a73bc268 100644 --- a/core/src/main/java/com/google/adk/workflow/NodeRunner.java +++ b/core/src/main/java/com/google/adk/workflow/NodeRunner.java @@ -377,7 +377,7 @@ private static Maybe childResult( /** * Runs the node on {@code nodeInput}, retrying it as its retry policy allows, and returns the * context of its last attempt. An exception from the node is recorded on the context's node state - * rather than signaled, while an {@code Error} propagates. + * rather than signaled, while an {@code Error} other than a pause propagates. */ Single run(@Nullable Object nodeInput) { return Single.defer( @@ -546,9 +546,9 @@ private Completable withTimeout(Completable run) { } private Completable onError(Throwable error) { - if (error instanceof NodeInterruptedException) { - // A child waits, or raiseOnWait ended the run; any interrupt IDs are already recorded. - return flushPending(); + if (error instanceof NodeInterruptedException pause) { + // A waiting child's interrupt IDs are already recorded; ask sends and records a request's. + return pause.request() == null ? flushPending() : ask(pause.request()); } if (error instanceof DynamicNodeFailedException childFailure) { // The child already sent an error event, so the node takes on its failure without one. @@ -627,12 +627,24 @@ private Completable send(Event event) { Event outgoing = stamped.partial().orElse(false) ? stamped : withPendingDeltas(stamped); return eventSink .send(outgoing, /* nodeFailure= */ null) - .doOnComplete( - () -> { - if (outgoing.output().isPresent() || messageAsOutput) { - nodeState.markOutputEmitted(); - } - }); + .doOnComplete(() -> { + if (outgoing.output().isPresent() || messageAsOutput) { + nodeState.markOutputEmitted(); + } + // Releases a body that waits in Context.emit for this event. + nodeState.markProcessed(outgoing); + }); + } + + /** Asks for {@code request}; a request that has no JSON form fails the node. */ + private Completable ask(RequestInput request) { + Event event; + try { + event = request.toEvent(context.invocationContext()); + } catch (IllegalArgumentException e) { + return fail(e); + } + return dispatch(event).andThen(Completable.defer(this::flushPending)); } /** Sends a final event carrying any output, routes, or deltas that no earlier event carried. */ diff --git a/core/src/test/java/com/google/adk/agents/ContextTest.java b/core/src/test/java/com/google/adk/agents/ContextTest.java index 1075a0f00..908790736 100644 --- a/core/src/test/java/com/google/adk/agents/ContextTest.java +++ b/core/src/test/java/com/google/adk/agents/ContextTest.java @@ -105,6 +105,18 @@ public void runNode_onCallbackContext_failsWithIllegalStateException() { context.runNode(TestNode.of("child")).test().assertError(IllegalStateException.class); } + @Test + public void emit_fromAToolContext_throwsNamingTheAlternative() { + ToolContext tool = ToolContext.builder(newInvocationContext()).build(); + + Content message = Content.fromParts(Part.fromText("x")); + + IllegalStateException error = + assertThrows(IllegalStateException.class, () -> tool.emit(message)); + + assertThat(error).hasMessageThat().contains("a tool or callback returns its result"); + } + @Test public void runNode_configPassedAsTheInput_throwsIllegalArgumentException() { CallbackContext context = new CallbackContext(newInvocationContext(), null); diff --git a/core/src/test/java/com/google/adk/workflow/DynamicDispatchTest.java b/core/src/test/java/com/google/adk/workflow/DynamicDispatchTest.java index b6a5a0ae7..abf889b9a 100644 --- a/core/src/test/java/com/google/adk/workflow/DynamicDispatchTest.java +++ b/core/src/test/java/com/google/adk/workflow/DynamicDispatchTest.java @@ -432,9 +432,9 @@ public void runChild_childWaitingOnAnInterrupt_leavesTheCallerWaiting() { } @Test - public void runChild_callerCatchingTheWaitSignal_keepsRunningButStillWaits() { + public void runChild_callerCatchingTheWaitSignalAsThrowable_keepsRunningButStillWaits() { Node dispatcher = - handling("dispatcher", interrupting("child", "ask"), DEFAULTS, RuntimeException.class, "X"); + handling("dispatcher", interrupting("child", "ask"), DEFAULTS, Throwable.class, "X"); Node after = emitter("after", "AFTER"); Context context = run(workflow(new Edge(START, dispatcher), new Edge(dispatcher, after))); @@ -602,7 +602,7 @@ private static Node handling( String name, Node child, RunNodeConfig config, - Class handled, + Class handled, @Nullable Object fallback) { return dispatcher( name, diff --git a/core/src/test/java/com/google/adk/workflow/FunctionNodeTest.java b/core/src/test/java/com/google/adk/workflow/FunctionNodeTest.java index 2d22a08f3..7abf62b6e 100644 --- a/core/src/test/java/com/google/adk/workflow/FunctionNodeTest.java +++ b/core/src/test/java/com/google/adk/workflow/FunctionNodeTest.java @@ -16,13 +16,16 @@ package com.google.adk.workflow; +import static com.google.adk.testing.WorkflowTestNodes.dispatcher; import static com.google.adk.testing.WorkflowTestNodes.emitter; import static com.google.adk.workflow.WorkflowResumeTesting.answer; import static com.google.adk.workflow.WorkflowResumeTesting.captureInput; import static com.google.adk.workflow.WorkflowResumeTesting.go; import static com.google.adk.workflow.WorkflowResumeTesting.longRunningAsk; +import static com.google.adk.workflow.WorkflowResumeTesting.onlyRequest; import static com.google.adk.workflow.WorkflowResumeTesting.replyInput; import static com.google.adk.workflow.WorkflowResumeTesting.requestAnswer; +import static com.google.adk.workflow.WorkflowResumeTesting.requestResult; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.MoreCollectors.onlyElement; @@ -62,6 +65,11 @@ import com.google.adk.events.ToolConfirmation; import com.google.adk.plugins.BasePlugin; import com.google.adk.runner.Runner; +import com.google.adk.sessions.BaseSessionService; +import com.google.adk.sessions.GetSessionConfig; +import com.google.adk.sessions.InMemorySessionService; +import com.google.adk.sessions.ListEventsResponse; +import com.google.adk.sessions.ListSessionsResponse; import com.google.adk.sessions.Session; import com.google.adk.testing.TestNode; import com.google.adk.testing.TestUtils; @@ -78,10 +86,14 @@ import com.google.genai.types.Schema; import com.google.genai.types.Type; import com.google.genai.types.VideoMetadata; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.context.Scope; +import io.reactivex.rxjava3.core.BackpressureStrategy; import io.reactivex.rxjava3.core.Completable; import io.reactivex.rxjava3.core.Flowable; import io.reactivex.rxjava3.core.Maybe; import io.reactivex.rxjava3.core.Scheduler; +import io.reactivex.rxjava3.core.Single; import io.reactivex.rxjava3.disposables.Disposable; import io.reactivex.rxjava3.plugins.RxJavaPlugins; import io.reactivex.rxjava3.schedulers.Schedulers; @@ -94,8 +106,10 @@ import java.net.URI; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -104,13 +118,19 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -133,6 +153,7 @@ public final class FunctionNodeTest { private static final String PAY_CONFIRMATION = "wf_tool_confirmation:wf@1/pay@1"; private static final ImmutableMap REJECTED = ImmutableMap.of("error", "This tool call is rejected."); + private static final Duration SLOW_APPEND = Duration.ofMillis(100); private static final Schema ANY = Schema.builder().build(); private static final Schema STRING = typed(Type.Known.STRING); private static final Schema INTEGER = typed(Type.Known.INTEGER); @@ -642,6 +663,8 @@ record Authored(String author) {} record TreeNode(String label, List children) {} + record Leave(LocalDate day) {} + /** Holds a declaration annotation named like the {@code @Nullable} of other packages. */ static final class OtherNullness { @Retention(RetentionPolicy.RUNTIME) @@ -775,6 +798,49 @@ public void builder_bodyRerunOnResume_writesTheReplayedDelegatedOutputAsO() { assertThat(replyInput(events)).isEqualTo(asTicket); } + @Test + public void resume_bodyRunningChildren_sendsItsEventsAgainAndReplaysTheFinishedChild() { + // Arrange: the body sends a message, runs first, then asker, which pauses. + AtomicInteger firstRuns = new AtomicInteger(); + FunctionNode first = + FunctionNodes.of( + "first", + Object.class, + String.class, + (ctx, input) -> { + firstRuns.incrementAndGet(); + return "a_out"; + }); + Node asker = longRunningAsk("asker", "name_1"); + FunctionNode parent = + FunctionNodes.builder(Content.class, Object.class) + .name("parent") + .rerunOnResume(true) + .body((ctx, message) -> { + ctx.emit(text("working")); + Object firstOutput = ctx.runNode(first).blockingGet(); + Object answer = ctx.runNode(asker, firstOutput).blockingGet(); + // Arrays.asList takes a null answer, so a missed pause shows as an early output. + return Arrays.asList(firstOutput, answer); + }) + .build(); + WorkflowSession session = new WorkflowSession(chain(parent, captureInput("reply"))); + ImmutableList paused = session.turn(go()); + ImmutableMap details = ImmutableMap.of("name", "John"); + + ImmutableList events = session.turn(answer("name_1", details)); + + assertThat( + paused.stream().filter(e -> e.output().isPresent()).map(WorkflowResumeTesting::pathOf)) + .doesNotContain("wf@1/parent@1"); + assertThat(paused.stream().flatMap(e -> e.content().stream()).map(Content::text)) + .contains("working"); + assertThat(events.stream().flatMap(e -> e.content().stream()).map(Content::text)) + .contains("working"); + assertThat(firstRuns.get()).isEqualTo(1); + assertThat(replyInput(events)).isEqualTo(ImmutableList.of("a_out", details)); + } + @Test public void builder_eventBodyDelegatingARecord_outputsItsJsonForm() { RunResult result = run(chain(delegating(Event.class, new Order("a1", 3)))); @@ -1121,9 +1187,11 @@ public void of_valueThatIsNotAContent_failsAContentPortWithoutQuotingTheValue() } @Test - public void of_body_runsOnTheRunnersScheduler() { + public void of_body_runsOnIoRatherThanTheRunnersScheduler() { ExecutorService executor = Executors.newSingleThreadExecutor(task -> new Thread(task, "app-scheduler")); + ExecutorService io = Executors.newCachedThreadPool(task -> new Thread(task, "io-stand-in")); + var previousIoHandler = RxJavaPlugins.getIoSchedulerHandler(); FunctionNode where = FunctionNodes.of( "where", @@ -1133,13 +1201,172 @@ public void of_body_runsOnTheRunnersScheduler() { RunResult result; try { + RxJavaPlugins.setIoSchedulerHandler(unused -> Schedulers.from(io)); result = run(chain(where), Schedulers.from(executor)); + } finally { + RxJavaPlugins.setIoSchedulerHandler(previousIoHandler); + executor.shutdownNow(); + io.shutdownNow(); + } + + result.assertCompleted(); + assertThat(result.outputAt("wf@1/where@1")).isEqualTo("io-stand-in"); + } + + @Test + public void singleThreadRunner_parallelBodiesThatWaitForEachOther_complete() { + // Each body blocks until both run, so they need a thread each. + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch bothRunning = new CountDownLatch(2); + List met = new CopyOnWriteArrayList<>(); + NodeBody body = + (ctx, message) -> { + bothRunning.countDown(); + met.add(bothRunning.await(10, SECONDS)); + return null; // A parallel pair may not both output as terminal nodes. + }; + FunctionNode a = FunctionNodes.of("a", Content.class, Object.class, body); + FunctionNode b = FunctionNodes.of("b", Content.class, Object.class, body); + Workflow workflow = + Workflow.builder() + .name("wf") + .edges(new Edge(Start.INSTANCE, a), new Edge(Start.INSTANCE, b)) + .build(); + + RunResult result; + try { + result = run(workflow, Schedulers.from(executor)); } finally { executor.shutdownNow(); } result.assertCompleted(); - assertThat(result.outputAt("wf@1/where@1")).isEqualTo("app-scheduler"); + assertThat(met).containsExactly(true, true); + } + + @Test + public void builderScheduler_body_runsOnThatScheduler() { + ExecutorService nodeExecutor = + Executors.newSingleThreadExecutor(task -> new Thread(task, "node-scheduler")); + ExecutorService runnerExecutor = + Executors.newSingleThreadExecutor(task -> new Thread(task, "app-scheduler")); + FunctionNode where = + FunctionNodes.builder(Content.class, String.class) + .name("where") + .scheduler(Schedulers.from(nodeExecutor)) + .body((ctx, message) -> Thread.currentThread().getName()) + .build(); + + RunResult result; + try { + result = run(chain(where), Schedulers.from(runnerExecutor)); + } finally { + nodeExecutor.shutdownNow(); + runnerExecutor.shutdownNow(); + } + + result.assertCompleted(); + assertThat(result.outputAt("wf@1/where@1")).isEqualTo("node-scheduler"); + } + + @Test + public void builderScheduler_afterTheBody_successorAndCallerRunOnTheRunnersScheduler() { + ExecutorService nodeExecutor = + Executors.newSingleThreadExecutor(task -> new Thread(task, "node-scheduler")); + ExecutorService appExecutor = + Executors.newSingleThreadExecutor(task -> new Thread(task, "app-scheduler")); + Map threads = new ConcurrentHashMap<>(); + FunctionNode first = + FunctionNodes.builder(Content.class, String.class) + .name("first") + .scheduler(Schedulers.from(nodeExecutor)) + .body((ctx, message) -> { + threads.put("body", Thread.currentThread().getName()); + // Returns after the Runner's thread is idle, so a broken hand-back cannot hide. + appExecutor.submit(() -> {}).get(10, SECONDS); + return "out"; + }) + .build(); + Node second = + dispatcher( + "second", + (ctx, input) -> + Flowable.fromCallable(() -> { + threads.put("successor", Thread.currentThread().getName()); + return "done"; + })); + Runner runner = runner(chain(first, second), Schedulers.from(appExecutor)); + Session session = runner.sessionService().createSession(APP, USER).blockingGet(); + AtomicReference error = new AtomicReference<>(); + + try { + runner + .runAsync(USER, session.id(), Content.fromParts(Part.fromText("go"))) + // The Runner's thread subscribes, so a quick first node cannot start "second" on main. + .subscribeOn(Schedulers.from(appExecutor)) + .doOnNext(event -> { + if (RunResult.pathOf(event).equals("wf@1/first@1") && event.output().isPresent()) { + threads.put("onNext", Thread.currentThread().getName()); + } + }) + .timeout(30, SECONDS) + // Requests every event up front, so that no later request drains them on this thread. + .ignoreElements() + .blockingSubscribe(() -> {}, error::set); + } finally { + nodeExecutor.shutdownNow(); + appExecutor.shutdownNow(); + } + + assertThat(error.get()).isNull(); + assertThat(threads) + .containsExactly( + "body", "node-scheduler", "successor", "app-scheduler", "onNext", "app-scheduler"); + } + + @Test + public void body_callerTraceContext_reachesTheBodyItsChildAndTheOutputsCallback() { + ContextKey key = ContextKey.named("caller"); + List seen = new CopyOnWriteArrayList<>(); + ExecutorService nodeExecutor = Executors.newSingleThreadExecutor(); + Node child = + dispatcher( + "child", + (ctx, input) -> + Flowable.fromCallable(() -> { + seen.add("child: " + io.opentelemetry.context.Context.current().get(key)); + return "done"; + })); + FunctionNode parent = + FunctionNodes.builder(Content.class, Object.class) + .name("parent") + .rerunOnResume(true) + // Adds the hop back to the Runner's scheduler, which must keep the context too. + .scheduler(Schedulers.from(nodeExecutor)) + .body((ctx, message) -> { + seen.add("body: " + io.opentelemetry.context.Context.current().get(key)); + return ctx.runNode(child).blockingGet(); + }) + .build(); + BasePlugin recorder = + new BasePlugin("recorder") { + @Override + public Maybe onEventCallback(InvocationContext invocationContext, Event event) { + if (RunResult.pathOf(event).equals("wf@1/parent@1") && event.output().isPresent()) { + seen.add("output: " + io.opentelemetry.context.Context.current().get(key)); + } + return Maybe.empty(); + } + }; + + try (Scope unused = io.opentelemetry.context.Context.current().with(key, "set").makeCurrent()) { + ImmutableList unusedEvents = + new WorkflowSession(chain(parent), /* resumable= */ false, recorder).turn(go()); + } finally { + nodeExecutor.shutdownNow(); + } + + assertThat(seen).containsExactly("body: set", "child: set", "output: set").inOrder(); } @Test @@ -1147,12 +1374,12 @@ public void runDisposed_sleepingBody_isInterrupted() throws InterruptedException // Arrange: an executor scheduler, which does not interrupt on its own when disposed. CountDownLatch started = new CountDownLatch(1); CountDownLatch interrupted = new CountDownLatch(1); + ExecutorService executor = Executors.newCachedThreadPool(); FunctionNode sleeper = - FunctionNodes.of( - "sleeper", - Content.class, - String.class, - (ctx, message) -> { + FunctionNodes.builder(Content.class, String.class) + .name("sleeper") + .scheduler(Schedulers.from(executor)) + .body((ctx, message) -> { started.countDown(); try { Thread.sleep(60_000); @@ -1160,10 +1387,10 @@ public void runDisposed_sleepingBody_isInterrupted() throws InterruptedException interrupted.countDown(); } return "late"; - }); - ExecutorService executor = Executors.newCachedThreadPool(); + }) + .build(); try { - Runner runner = runner(chain(sleeper), Schedulers.from(executor)); + Runner runner = runner(chain(sleeper), /* scheduler= */ null); Session session = runner.sessionService().createSession(APP, USER).blockingGet(); Disposable run = runner.runAsync(USER, session.id(), Content.fromParts(Part.fromText("go"))).subscribe(); @@ -1275,11 +1502,10 @@ public void runDisposed_bodyThatKeepsTheInterrupt_leavesTheThreadUninterruptedFo CountDownLatch finished = new CountDownLatch(1); AtomicBoolean sawInterrupt = new AtomicBoolean(); FunctionNode spinner = - FunctionNodes.of( - "spinner", - Content.class, - String.class, - (ctx, message) -> { + FunctionNodes.builder(Content.class, String.class) + .name("spinner") + .scheduler(Schedulers.from(tasks::add)) + .body((ctx, message) -> { started.countDown(); long deadline = System.nanoTime() + SECONDS.toNanos(10); while (!Thread.currentThread().isInterrupted() && System.nanoTime() < deadline) { @@ -1288,7 +1514,8 @@ public void runDisposed_bodyThatKeepsTheInterrupt_leavesTheThreadUninterruptedFo sawInterrupt.set(Thread.currentThread().isInterrupted()); finished.countDown(); return "late"; - }); + }) + .build(); try { Runner runner = runner(chain(spinner), Schedulers.from(tasks::add)); Session session = runner.sessionService().createSession(APP, USER).blockingGet(); @@ -1316,11 +1543,10 @@ public void inlineScheduler_runDisposedFromAnotherThread_interruptsTheBodyAndCle CountDownLatch started = new CountDownLatch(1); AtomicBoolean sawInterrupt = new AtomicBoolean(); FunctionNode sleeper = - FunctionNodes.of( - "sleeper", - Content.class, - String.class, - (ctx, message) -> { + FunctionNodes.builder(Content.class, String.class) + .name("sleeper") + .scheduler(Schedulers.trampoline()) + .body((ctx, message) -> { started.countDown(); try { Thread.sleep(60_000); @@ -1329,7 +1555,8 @@ public void inlineScheduler_runDisposedFromAnotherThread_interruptsTheBodyAndCle Thread.currentThread().interrupt(); } return "late"; - }); + }) + .build(); TestSubscriber subscriber = new TestSubscriber<>(); ExecutorService caller = Executors.newSingleThreadExecutor(); try { @@ -1352,14 +1579,14 @@ public void inlineScheduler_callerAlreadyInterrupted_keepsTheInterruptAfterTheBo // Arrange: a trampoline scheduler runs the body on the caller's thread, interrupted beforehand. AtomicBoolean bodySawInterrupt = new AtomicBoolean(); FunctionNode quick = - FunctionNodes.of( - "quick", - Content.class, - String.class, - (ctx, message) -> { + FunctionNodes.builder(Content.class, String.class) + .name("quick") + .scheduler(Schedulers.trampoline()) + .body((ctx, message) -> { bodySawInterrupt.set(Thread.currentThread().isInterrupted()); return "done"; - }); + }) + .build(); TestSubscriber subscriber = new TestSubscriber<>(); ExecutorService caller = Executors.newSingleThreadExecutor(); try { @@ -1384,11 +1611,10 @@ public void inlineScheduler_callerInterruptedThenRunDisposed_keepsTheCallersInte AtomicBoolean sawCancel = new AtomicBoolean(); AtomicBoolean cancelInterruptedTheBody = new AtomicBoolean(); FunctionNode waiter = - FunctionNodes.of( - "waiter", - Content.class, - String.class, - (ctx, message) -> { + FunctionNodes.builder(Content.class, String.class) + .name("waiter") + .scheduler(Schedulers.trampoline()) + .body((ctx, message) -> { boolean callerInterrupt = Thread.interrupted(); started.countDown(); sawCancel.set(Uninterruptibles.awaitUninterruptibly(canceled, 10, SECONDS)); @@ -1397,7 +1623,8 @@ public void inlineScheduler_callerInterruptedThenRunDisposed_keepsTheCallersInte Thread.currentThread().interrupt(); } return "late"; - }); + }) + .build(); TestSubscriber subscriber = new TestSubscriber<>(); ExecutorService caller = Executors.newSingleThreadExecutor(); try { @@ -1420,14 +1647,14 @@ public void inlineScheduler_callerInterruptedThenRunDisposed_keepsTheCallersInte public void inlineScheduler_bodyInterruptsItself_callerSeesTheInterrupt() throws Exception { // Arrange: nothing cancels the node, so only the body interrupts the caller's thread. FunctionNode restorer = - FunctionNodes.of( - "restorer", - Content.class, - String.class, - (ctx, message) -> { + FunctionNodes.builder(Content.class, String.class) + .name("restorer") + .scheduler(Schedulers.trampoline()) + .body((ctx, message) -> { Thread.currentThread().interrupt(); return "done"; - }); + }) + .build(); TestSubscriber subscriber = new TestSubscriber<>(); ExecutorService caller = Executors.newSingleThreadExecutor(); try { @@ -1446,7 +1673,11 @@ public void inlineScheduler_bodyInterruptsItself_callerSeesTheInterrupt() throws public void inlineScheduler_bodyReturns_leavesTheCallerUninterrupted() throws Exception { // Arrange: RxJava calls the node's cancellable after the body's result is sent. FunctionNode quick = - FunctionNodes.of("quick", Content.class, String.class, (ctx, message) -> "done"); + FunctionNodes.builder(Content.class, String.class) + .name("quick") + .scheduler(Schedulers.trampoline()) + .body((ctx, message) -> "done") + .build(); TestSubscriber subscriber = new TestSubscriber<>(); ExecutorService caller = Executors.newSingleThreadExecutor(); try { @@ -1478,16 +1709,10 @@ public void nodeTimeout_sleepingBody_isInterruptedAndFailsWithNodeTimeoutExcepti return "late"; }) .build(); - ExecutorService executor = Executors.newCachedThreadPool(); - RunResult result; - try { - result = run(chain(sleeper), Schedulers.from(executor)); - assertThat(interrupted.await(10, SECONDS)).isTrue(); - } finally { - executor.shutdownNow(); - } + RunResult result = run(chain(sleeper)); + assertThat(interrupted.await(10, SECONDS)).isTrue(); assertThat(result.error()).isInstanceOf(NodeTimeoutException.class); assertThat(result.errorMessage("NodeTimeoutException")).contains("'sleeper'"); } @@ -1581,9 +1806,9 @@ public void nodeTimeoutWithRetry_bodyIgnoringTheInterrupt_retryStartsOnlyAfterIt } @Test - public void nodeTimeoutWithRetry_bodyRunsAnotherNodeUnderItsContext_retryWaitsForTheOuterBody() + public void nodeTimeoutWithRetry_bodyTriesToRunAnotherNodeUnderItsContext_retryWaitsForTheBody() throws Exception { - // "inner" runs under "outer"'s context and returns at once; "outer" then outlives its timeout. + // "outer" tries to run "inner" under its own context, which fails, then outlives its timeout. AtomicInteger attempts = new AtomicInteger(); List timeline = Collections.synchronizedList(new ArrayList<>()); CountDownLatch retryStarted = new CountDownLatch(1); @@ -1607,8 +1832,9 @@ public void nodeTimeoutWithRetry_bodyRunsAnotherNodeUnderItsContext_retryWaitsFo int attempt = attempts.incrementAndGet(); timeline.add("start " + attempt); if (attempt == 1) { - inner.runNode(ctx, null).blockingSubscribe(); - timeline.add("inner returned"); + Completable innerRun = inner.runNode(ctx, null).ignoreElements(); + assertThrows(IllegalStateException.class, innerRun::blockingAwait); + timeline.add("inner refused"); var unused = Uninterruptibles.awaitUninterruptibly(retryStarted, 2, SECONDS); timeline.add("end 1"); firstReturned.countDown(); @@ -1628,7 +1854,7 @@ public void nodeTimeoutWithRetry_bodyRunsAnotherNodeUnderItsContext_retryWaitsFo executor.shutdownNow(); } - assertThat(timeline).containsExactly("start 1", "inner returned", "end 1", "start 2").inOrder(); + assertThat(timeline).containsExactly("start 1", "inner refused", "end 1", "start 2").inOrder(); result.assertCompleted(); assertThat(result.outputAt("wf@1/outer@1")).isEqualTo("retried"); } @@ -1641,10 +1867,12 @@ public void nodeTimeout_oneThreadRunnerWithABusySibling_failsTheRunAndCancelsThe List timeline = Collections.synchronizedList(new ArrayList<>()); CountDownLatch release = new CountDownLatch(1); ExecutorService executor = Executors.newSingleThreadExecutor(); + Scheduler one = Schedulers.from(executor); FunctionNode sleeper = FunctionNodes.builder(Content.class, String.class) .name("sleeper") .config(NodeConfig.builder().timeout(Duration.ofMillis(200)).build()) + .scheduler(one) .body((ctx, message) -> { timeline.add("sleeper started"); try { @@ -1659,6 +1887,7 @@ public void nodeTimeout_oneThreadRunnerWithABusySibling_failsTheRunAndCancelsThe FunctionNodes.builder(Content.class, String.class) .name("busy") .config(NodeConfig.builder().timeout(Duration.ofHours(1)).build()) + .scheduler(one) .body((ctx, message) -> { timeline.add("busy started"); release.await(); @@ -1671,7 +1900,7 @@ public void nodeTimeout_oneThreadRunnerWithABusySibling_failsTheRunAndCancelsThe .name("wf") .edges(new Edge(Start.INSTANCE, sleeper), new Edge(Start.INSTANCE, busy)) .build(); - Runner runner = runner(workflow, Schedulers.from(executor)); + Runner runner = runner(workflow, one); Session session = runner.sessionService().createSession(APP, USER).blockingGet(); TestSubscriber subscriber = new TestSubscriber<>(); @@ -1704,6 +1933,7 @@ public void nodeTimeout_oneThreadRunnerWithABusySibling_failsTheRunAndCancelsThe FunctionNodes.builder(Content.class, String.class) .name("stubborn") .config(NodeConfig.builder().timeout(Duration.ofMillis(200)).build()) + .scheduler(Schedulers.trampoline()) .body((ctx, message) -> { Uninterruptibles.sleepUninterruptibly(400, MILLISECONDS); return "late"; @@ -1751,6 +1981,100 @@ public Maybe onEventCallback(InvocationContext context, Event event) { .isFalse(); } + @Test + public void nodeTimeout_sleepingBodyOnBuilderScheduler_isInterrupted() + throws InterruptedException { + // An executor scheduler does not interrupt its task on its own when disposed. + ExecutorService executor = + Executors.newSingleThreadExecutor(task -> new Thread(task, "node-scheduler")); + AtomicReference interruptedOn = new AtomicReference<>(); + CountDownLatch interrupted = new CountDownLatch(1); + FunctionNode sleeper = + FunctionNodes.builder(Content.class, String.class) + .name("sleeper") + .config(NodeConfig.builder().timeout(Duration.ofSeconds(1)).build()) + .scheduler(Schedulers.from(executor)) + .body((ctx, message) -> { + try { + Thread.sleep(60_000); + } catch (InterruptedException e) { + interruptedOn.set(Thread.currentThread().getName()); + interrupted.countDown(); + } + return "late"; + }) + .build(); + + RunResult result; + try { + result = run(chain(sleeper)); + assertThat(interrupted.await(10, SECONDS)).isTrue(); + } finally { + executor.shutdownNow(); + } + + assertThat(result.error()).isInstanceOf(NodeTimeoutException.class); + assertThat(interruptedOn.get()).isEqualTo("node-scheduler"); + } + + @Test + public void + nodeTimeout_afterABodyOnAnotherSchedulerReturned_doesNotInterruptTheHandlingOfItsOutput() { + // The run hops back to io(), the Runner's default, whose canceled observeOn would interrupt. + ExecutorService executor = Executors.newCachedThreadPool(); + try { + assertTimeoutLeavesTheOutputHandlerUninterrupted(Schedulers.from(executor)); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void nodeTimeout_afterTheBodyReturned_doesNotInterruptTheHandlingOfItsOutput() { + // The node and the Runner default to io(), which interrupts a canceled subscribeOn. + assertTimeoutLeavesTheOutputHandlerUninterrupted(/* nodeScheduler= */ null); + } + + /** Times out a node on {@code nodeScheduler} while the caller still handles its output. */ + private static void assertTimeoutLeavesTheOutputHandlerUninterrupted( + @Nullable Scheduler nodeScheduler) { + FunctionNode quick = + FunctionNodes.builder(Content.class, String.class) + .name("quick") + .config(NodeConfig.builder().timeout(Duration.ofSeconds(1)).build()) + .scheduler(nodeScheduler) + .body((ctx, message) -> "out") + .build(); + Runner runner = runner(chain(quick), /* scheduler= */ null); + Session session = runner.sessionService().createSession(APP, USER).blockingGet(); + AtomicBoolean handled = new AtomicBoolean(); + AtomicBoolean interrupted = new AtomicBoolean(); + AtomicReference error = new AtomicReference<>(); + + runner + .runAsync(USER, session.id(), Content.fromParts(Part.fromText("go"))) + .doOnNext(event -> { + if (RunResult.pathOf(event).equals("wf@1/quick@1") && event.output().isPresent()) { + try { + // Outlasts the timeout on the thread that delivers the node's output. + Thread.sleep(2000); + handled.set(true); + } catch (InterruptedException e) { + interrupted.set(true); + Thread.currentThread().interrupt(); + } + } + }) + .timeout(30, SECONDS) + .blockingSubscribe(event -> {}, error::set); + + assertWithMessage("whether the timeout interrupted the handler") + .that(interrupted.get()) + .isFalse(); + assertWithMessage("whether the output event reached the handler").that(handled.get()).isTrue(); + assertThat(error.get()).isInstanceOf(NodeTimeoutException.class); + } + @Test public void runDisposed_bodyThatThrowsAfterTheInterrupt_reportsNoUndeliverableError() throws InterruptedException { @@ -1758,12 +2082,12 @@ public void runDisposed_bodyThatThrowsAfterTheInterrupt_reportsNoUndeliverableEr CountDownLatch finished = new CountDownLatch(1); List undeliverable = new CopyOnWriteArrayList<>(); var previousErrorHandler = RxJavaPlugins.getErrorHandler(); + ExecutorService executor = Executors.newCachedThreadPool(); FunctionNode sleeper = - FunctionNodes.of( - "sleeper", - Content.class, - String.class, - (ctx, message) -> { + FunctionNodes.builder(Content.class, String.class) + .name("sleeper") + .scheduler(Schedulers.from(executor)) + .body((ctx, message) -> { started.countDown(); try { Thread.sleep(60_000); @@ -1771,8 +2095,8 @@ public void runDisposed_bodyThatThrowsAfterTheInterrupt_reportsNoUndeliverableEr } finally { finished.countDown(); } - }); - ExecutorService executor = Executors.newCachedThreadPool(); + }) + .build(); RxJavaPlugins.setErrorHandler(undeliverable::add); try { Runner runner = runner(chain(sleeper), Schedulers.from(executor)); @@ -1798,10 +2122,12 @@ public void runDisposed_bodyThatDelegatedItsOutput_doesNotConvertTheChildsOutput throws InterruptedException { // Arrange: the child outputs a map, so only the node would convert it to a Constructed. CountDownLatch started = new CountDownLatch(1); + ExecutorService executor = Executors.newCachedThreadPool(); FunctionNode router = FunctionNodes.builder(Content.class, Constructed.class) .name("router") .rerunOnResume(true) + .scheduler(Schedulers.from(executor)) .body((ctx, message) -> { Object unused = ctx.runNode( @@ -1818,10 +2144,9 @@ public void runDisposed_bodyThatDelegatedItsOutput_doesNotConvertTheChildsOutput return null; }) .build(); - ExecutorService executor = Executors.newCachedThreadPool(); Constructed.instances.set(0); try { - Runner runner = runner(chain(router), Schedulers.from(executor)); + Runner runner = runner(chain(router), /* scheduler= */ null); Session session = runner.sessionService().createSession(APP, USER).blockingGet(); Disposable run = runner.runAsync(USER, session.id(), Content.fromParts(Part.fromText("go"))).subscribe(); @@ -1842,12 +2167,12 @@ public void siblingFails_sleepingBody_isInterrupted() throws InterruptedExceptio // Arrange: the workflow cancels the nodes still running when one fails. CountDownLatch started = new CountDownLatch(1); CountDownLatch interrupted = new CountDownLatch(1); + ExecutorService executor = Executors.newCachedThreadPool(); FunctionNode sleeper = - FunctionNodes.of( - "sleeper", - Content.class, - Void.class, - (ctx, message) -> { + FunctionNodes.builder(Content.class, Void.class) + .name("sleeper") + .scheduler(Schedulers.from(executor)) + .body((ctx, message) -> { started.countDown(); try { Thread.sleep(60_000); @@ -1855,7 +2180,8 @@ public void siblingFails_sleepingBody_isInterrupted() throws InterruptedExceptio interrupted.countDown(); } return null; - }); + }) + .build(); FunctionNode broken = FunctionNodes.of( "broken", @@ -1872,11 +2198,10 @@ public void siblingFails_sleepingBody_isInterrupted() throws InterruptedExceptio .name("wf") .edges(new Edge(Start.INSTANCE, sleeper), new Edge(Start.INSTANCE, broken)) .build(); - ExecutorService executor = Executors.newCachedThreadPool(); RunResult result; try { - result = run(workflow, Schedulers.from(executor)); + result = run(workflow); assertThat(interrupted.await(10, SECONDS)).isTrue(); } finally { executor.shutdownNow(); @@ -1903,11 +2228,33 @@ public void toBuilder_copy_keepsThePortsAndTheBody() { } @Test - public void toBuilder_copy_keepsOnlyTheExplicitSchema() { - // Arrange - Schema explicit = Schema.builder().type(Type.Known.OBJECT).build(); - FunctionNode node = - FunctionNodes.builder(Order.class, Order.class) + public void toBuilder_copy_keepsTheScheduler() { + ExecutorService executor = + Executors.newSingleThreadExecutor(task -> new Thread(task, "node-scheduler")); + FunctionNode where = + FunctionNodes.builder(Content.class, String.class) + .name("where") + .scheduler(Schedulers.from(executor)) + .body((ctx, message) -> Thread.currentThread().getName()) + .build(); + + RunResult result; + try { + result = run(chain(where.toBuilder().name("copy").build())); + } finally { + executor.shutdownNow(); + } + + result.assertCompleted(); + assertThat(result.outputAt("wf@1/copy@1")).isEqualTo("node-scheduler"); + } + + @Test + public void toBuilder_copy_keepsOnlyTheExplicitSchema() { + // Arrange + Schema explicit = Schema.builder().type(Type.Known.OBJECT).build(); + FunctionNode node = + FunctionNodes.builder(Order.class, Order.class) .name("n") .inputSchema(explicit) .body((ctx, order) -> order) @@ -3086,71 +3433,836 @@ public void of_genericOutputPortGivenElementsInJsonForm_writesThemAsThePortType( (List) (List) ImmutableList.of(ImmutableMap.of("id", "a1", "quantity", 2.0))); - RunResult result = run(chain(orders)); + RunResult result = run(chain(orders)); + + result.assertCompleted(); + assertThat(result.outputAt("wf@1/orders@1")) + .isEqualTo(ImmutableList.of(ImmutableMap.of("id", "a1", "quantity", 2))); + } + + @Test + public void of_genericOutputOfATypeJacksonCannotRead_isWritten() { + // Arrange: an output port need not be readable. + FunctionNode> boxed = + FunctionNodes.of( + "boxed", + Content.class, + new TypeReference>() {}, + (ctx, m) -> new Box<>(new NoCreator("x", 1))); + + RunResult result = run(chain(boxed)); + + result.assertCompleted(); + assertThat(result.outputAt("wf@1/boxed@1")) + .isEqualTo(ImmutableMap.of("item", ImmutableMap.of("label", "x"))); + } + + @Test + public void of_typeReadThroughADelegatingMapCreator_roundTrips() { + FunctionNode makeBag = + FunctionNodes.of( + "make_bag", + Content.class, + Bag.class, + (ctx, m) -> new Bag(ImmutableMap.of("name", "x"))); + FunctionNode readBag = + FunctionNodes.of("read_bag", Bag.class, String.class, (ctx, bag) -> "bag " + bag.getName()); + + RunResult result = run(chain(makeBag, readBag)); + + result.assertCompleted(); + assertThat(result.outputAt("wf@1/make_bag@1")).isEqualTo(ImmutableMap.of("name", "x")); + assertThat(result.outputAt("wf@1/read_bag@1")).isEqualTo("bag x"); + } + + @Test + public void of_interfaceOutputWithoutACreator_failsAtBuild() { + WorkflowConfigurationException error = + assertThrows( + WorkflowConfigurationException.class, + () -> FunctionNodes.of("n", String.class, Decision.class, (ctx, in) -> null)); + + assertThat(error) + .hasMessageThat() + .contains("interface or abstract class without a Jackson creator"); + } + + @Test + public void of_untypedAndAdkPorts_needNoJacksonShape() { + FunctionNode echo = + FunctionNodes.of("echo", Content.class, Object.class, (ctx, message) -> message.text()); + FunctionNode sink = + FunctionNodes.of("sink", Object.class, Void.class, (ctx, in) -> null); + + RunResult result = run(chain(echo, sink)); + + result.assertCompleted(); + assertThat(result.outputAt("wf@1/echo@1")).isEqualTo("go"); + } + + @Test + public void emit_stateWrittenAroundEvents_isSentWithTheNextEventAndReadsBackAfterTheStore() + throws Exception { + onEveryScheduler(scheduler -> { + SlowStore store = new SlowStore(scheduler, SLOW_APPEND); + AtomicReference readBack = new AtomicReference<>(); + FunctionNode counter = + node( + "counter", + (ctx, message) -> { + ctx.state().put("n", 1); + ctx.emit(text("first")); + ctx.state().put("n", 2); + // "first" is stored by now, so its n=1 can no longer land over n=2. + readBack.set(ctx.state().get("n") + " " + store.storedTexts); + ctx.emit(text("second")); + ctx.state().put("after_second", 3); + return ctx.state().get("n"); + }); + + RunResult result = run(chain(counter), scheduler, store); + + assertThat(readBack.get()).isEqualTo("2 [first]"); + result.assertCompleted(); + assertThat( + result.eventsAt("wf@1/counter@1").stream() + .map(e -> describe(e) + e.actions().stateDelta())) + .containsExactly("first{n=1}", "second{n=2}", "2{after_second=3}") + .inOrder(); + }); + } + + @Test + public void emit_fromTheBodyOrAHelperStillWaitingAtReturn_returnsOnceStored() throws Exception { + onEveryScheduler(scheduler -> { + // Arrange: the body returns no output while its helper still waits in emit. + SlowStore store = new SlowStore(scheduler, SLOW_APPEND); + AtomicReference> storedWhenHelperReturned = new AtomicReference<>(); + ExecutorService helper = Executors.newSingleThreadExecutor(); + AtomicReference> helperEmit = new AtomicReference<>(); + FunctionNode reporter = + node( + "reporter", + (ctx, message) -> { + ctx.emit(text("from the body")); + helperEmit.set( + helper.submit(() -> { + ctx.emit(text("from a helper")); + storedWhenHelperReturned.set(ImmutableList.copyOf(store.storedTexts)); + })); + assertThat(store.appendsStarted.tryAcquire(2, 10, SECONDS)).isTrue(); + return null; + }); + + try { + run(chain(reporter), scheduler, store).assertCompleted(); + helperEmit.get().get(10, SECONDS); + } finally { + helper.shutdownNow(); + } + + assertThat(storedWhenHelperReturned.get()) + .containsExactly("from the body", "from a helper") + .inOrder(); + }); + } + + @Test + public void emit_runDisposedWhileWaiting_throwsCancellationOnThisAndEveryLaterCall() + throws Exception { + onEveryScheduler(scheduler -> { + // Arrange: the store never finishes a node event, so emit waits until the cancel. + SlowStore store = new SlowStore(scheduler, Duration.ofHours(1)); + // The canceled run drops what the body returns, so the body reports through this. + CompletableFuture> outcomes = new CompletableFuture<>(); + FunctionNode stuck = + node( + "stuck", + (ctx, message) -> { + String first = emitOutcome(ctx, "stuck"); + outcomes.complete(ImmutableList.of(first, emitOutcome(ctx, "after the cancel"))); + return "dropped"; + }); + Runner runner = runner(chain(stuck), scheduler, store); + Session session = runner.sessionService().createSession(APP, USER).blockingGet(); + Disposable run = runner.runAsync(USER, session.id(), text("go")).subscribe(); + assertThat(store.appendsStarted.tryAcquire(10, SECONDS)).isTrue(); + + run.dispose(); + + assertThat(outcomes.get(10, SECONDS)) + .containsExactly("CancellationException", "CancellationException"); + }); + } + + @Test + public void emit_runDisposedWhileTheBodyWorks_everyLaterCallThrowsCancellation() + throws Exception { + // Arrange: the body works between calls and learns of the cancel only from its interrupt. + CountDownLatch working = new CountDownLatch(1); + CompletableFuture> outcomes = new CompletableFuture<>(); + FunctionNode importer = + node( + "importer", + (ctx, message) -> { + working.countDown(); + try { + Thread.sleep(10_000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); // The cancel reached the body first. + } + String first = emitOutcome(ctx, "batch 1"); + outcomes.complete(ImmutableList.of(first, emitOutcome(ctx, "batch 2"))); + return "dropped"; + }); + Runner runner = runner(chain(importer), null); + Session session = runner.sessionService().createSession(APP, USER).blockingGet(); + Disposable run = runner.runAsync(USER, session.id(), text("go")).subscribe(); + assertThat(working.await(10, SECONDS)).isTrue(); + + run.dispose(); + + assertThat(outcomes.get(10, SECONDS)) + .containsExactly("CancellationException", "CancellationException"); + } + + @Test + public void emit_nodeTimeoutWhileWaiting_throwsCancellationBeforeTheStoreAndFailsTheRun() + throws Exception { + // Arrange: the store takes far longer than the node's timeout to store a node event. + SlowStore store = new SlowStore(Schedulers.io(), Duration.ofSeconds(2)); + // The canceled run drops what the body returns, so the body reports through these. + CompletableFuture outcome = new CompletableFuture<>(); + CompletableFuture> storedWhenEmitEnded = new CompletableFuture<>(); + FunctionNode stuck = + FunctionNodes.builder(Content.class, Object.class) + .name("stuck") + .config(NodeConfig.builder().timeout(Duration.ofMillis(200)).build()) + .body((ctx, message) -> { + outcome.complete(emitOutcome(ctx, "stuck")); + storedWhenEmitEnded.complete(ImmutableList.copyOf(store.storedTexts)); + return "dropped"; + }) + .build(); + + RunResult result = run(chain(stuck), null, store); + + assertThat(outcome.get(10, SECONDS)).isEqualTo("CancellationException"); + assertThat(storedWhenEmitEnded.get(10, SECONDS)).isEmpty(); + assertThat(result.error()).isInstanceOf(NodeTimeoutException.class); + } + + @Test + public void emit_nodeTimeoutWhileWaitingThenRetry_retryStartsAfterTheBodyReturnsAndEmits() { + // Arrange: only the first attempt's event is stored late, after the node's 200 ms timeout. + SlowStore store = + new SlowStore( + Schedulers.io(), + Duration.ofMillis(600), + event -> event.content().map(Content::text).orElse("").equals("first")); + AtomicInteger attempts = new AtomicInteger(); + List timeline = Collections.synchronizedList(new ArrayList<>()); + FunctionNode reporter = + FunctionNodes.builder(Content.class, Object.class) + .name("reporter") + .config( + NodeConfig.builder() + .timeout(Duration.ofMillis(200)) + .retryConfig( + RetryConfig.builder() + .maxAttempts(2) + .initialDelay(Duration.ZERO) + .jitter(0) + .build()) + .build()) + .body((ctx, message) -> { + int attempt = attempts.incrementAndGet(); + timeline.add("start " + attempt); + if (attempt == 1) { + Content first = text("first"); + assertThrows(CancellationException.class, () -> ctx.emit(first)); + // Outlasts the 600 ms save, so only the wait for the body keeps the retry after it. + Uninterruptibles.sleepUninterruptibly(1000, MILLISECONDS); + timeline.add("end 1"); + return "late"; + } + ctx.emit(text("second")); + return "retried"; + }) + .build(); + + RunResult result = run(chain(reporter), /* scheduler= */ null, store); + + result.assertCompleted(); + assertThat(timeline).containsExactly("start 1", "end 1", "start 2").inOrder(); + assertThat(result.outputAt("wf@1/reporter@1")).isEqualTo("retried"); + assertThat(result.eventsAt("wf@1/reporter@1").stream().map(FunctionNodeTest::describe)) + .contains("second"); + } + + @Test + public void runNode_ofAnotherNodeUnderTheBodysContext_failsWithIllegalState() { + // Arrange: "outer" runs "inner" itself, under its own context, instead of through ctx.runNode. + AtomicInteger innerRuns = new AtomicInteger(); + FunctionNode inner = + FunctionNodes.of( + "inner", + Object.class, + String.class, + (ctx, in) -> { + innerRuns.incrementAndGet(); + return "inner out"; + }); + AtomicReference error = new AtomicReference<>(); + FunctionNode outer = + node( + "outer", + (ctx, message) -> { + error.set( + assertThrows( + IllegalStateException.class, + () -> inner.runNode(ctx, null).ignoreElements().blockingAwait())); + return "done"; + }); + + run(chain(outer)).assertCompleted(); + + assertThat(error.get()).hasMessageThat().contains("call ctx.runNode"); + assertThat(innerRuns.get()).isEqualTo(0); + } + + @Test + public void runNode_ofItselfUnderItsBodysContext_failsWithIllegalState() { + AtomicReference> self = new AtomicReference<>(); + AtomicInteger runs = new AtomicInteger(); + AtomicReference error = new AtomicReference<>(); + FunctionNode recursive = + node( + "recursive", + (ctx, message) -> { + if (runs.incrementAndGet() > 1) { + return "nested"; // Reached only if the nested run starts. + } + error.set( + assertThrows( + IllegalStateException.class, + () -> self.get().runNode(ctx, message).ignoreElements().blockingAwait())); + return "done"; + }); + self.set(recursive); + + run(chain(recursive)).assertCompleted(); + + assertThat(error.get()).hasMessageThat().contains("call ctx.runNode"); + assertThat(runs.get()).isEqualTo(1); + } + + @Test + public void runNode_fromACustomNode_failsWithIllegalState() { + AtomicInteger innerRuns = new AtomicInteger(); + FunctionNode inner = + node( + "inner", + (ctx, message) -> { + innerRuns.incrementAndGet(); + return "done"; + }); + + RunResult result = run(chain(new TestNode("custom", inner::runNode))); + + assertThat(result.errorMessage("IllegalStateException")).contains("call ctx.runNode"); + assertThat(innerRuns.get()).isEqualTo(0); + } + + @Test + public void emit_partialEvent_isNotStoredAndReleasesNoLaterEventWithItsId() throws Exception { + // Arrange: a helper's event keeps the run busy, so the partial event waits in line. + SlowStore store = new SlowStore(Schedulers.io(), SLOW_APPEND); + ExecutorService helper = Executors.newSingleThreadExecutor(); + AtomicReference> storedWhenEmitReturned = new AtomicReference<>(); + FunctionNode typist = + node( + "typist", + (ctx, message) -> { + Future first = helper.submit(() -> ctx.emit(text("first"))); + assertThat(store.appendsStarted.tryAcquire(10, SECONDS)).isTrue(); + ctx.state().put("k", 1); + ctx.emit(Event.builder().id("x").partial(true).content(text("typing")).build()); + ctx.emit(Event.builder().id("x").content(text("typed")).build()); + storedWhenEmitReturned.set(ImmutableList.copyOf(store.storedTexts)); + first.get(10, SECONDS); + return "done"; + }); + + RunResult result; + try { + result = run(chain(typist), Schedulers.io(), store); + } finally { + helper.shutdownNow(); + } + + result.assertCompleted(); + ImmutableList events = result.eventsAt("wf@1/typist@1"); + assertThat(events.stream().map(e -> describe(e) + e.actions().stateDelta())) + .containsExactly("first{}", "typing{}", "typed{k=1}", "done{}") + .inOrder(); + assertThat(storedWhenEmitReturned.get()).containsExactly("first", "typed").inOrder(); + } + + @Test + public void emit_manyHelpersAtOnce_sendsEveryEventAndReturns() throws Exception { + int helpers = 8; + int callsPerHelper = 1000; + ExecutorService pool = Executors.newFixedThreadPool(helpers); + FunctionNode fan = + node( + "fan", + (ctx, message) -> { + CountDownLatch start = new CountDownLatch(1); + List> calls = new ArrayList<>(); + for (int h = 0; h < helpers; h++) { + calls.add( + pool.submit(() -> { + start.await(); + for (int i = 0; i < callsPerHelper; i++) { + // Partial calls do not wait, so they contend; the last one waits. + boolean partial = i < callsPerHelper - 1; + ctx.emit(Event.builder().partial(partial).content(text("e")).build()); + } + return null; + })); + } + start.countDown(); + for (Future call : calls) { + call.get(10, SECONDS); + } + return "out"; + }); + + RunResult result; + try { + result = run(chain(fan)); + } finally { + pool.shutdownNow(); + } + + result.assertCompleted(); + assertThat(result.eventsAt("wf@1/fan@1")).hasSize(helpers * callsPerHelper + 1); + } + + @Test + public void bodyRunClose_whileACallHandsItsEventOn_waitsForTheCall() throws Exception { + // Arrange: the subscriber holds the helper's event inside onNext, so its call is mid-send. + FunctionNode.BodyRun run = new FunctionNode.BodyRun(); + CountDownLatch inOnNext = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + List received = new CopyOnWriteArrayList<>(); + Disposable unused = + Flowable.create( + emitter -> run.open(emitter.serialize()), BackpressureStrategy.BUFFER) + .subscribe(item -> { + received.add(item); + inOnNext.countDown(); + Uninterruptibles.awaitUninterruptibly(release); + }); + Event helperEvent = Event.builder().id("h").partial(true).content(text("h")).build(); + ExecutorService threads = Executors.newFixedThreadPool(2); + try { + Future call = threads.submit(() -> run.emit(helperEvent)); + assertThat(inOnNext.await(10, SECONDS)).isTrue(); + + CountDownLatch closeStarted = new CountDownLatch(1); + Future closing = + threads.submit(() -> { + closeStarted.countDown(); + run.close(); + }); + + assertThat(closeStarted.await(10, SECONDS)).isTrue(); + assertThrows(TimeoutException.class, () -> closing.get(200, MILLISECONDS)); + release.countDown(); + closing.get(10, SECONDS); + assertThat(call.get(10, SECONDS)).isTrue(); + } finally { + release.countDown(); + threads.shutdownNow(); + } + assertThat(received).containsExactly(helperEvent); + } + + @Test + public void emit_helpersStillCallingAsTheBodyReturns_sendEveryAcceptedEventBeforeTheOutput() + throws Exception { + // No test hook can stop a call between admission and send, so many calls race the return. + int helperCount = 4; + for (int round = 0; round < 20; round++) { + ExecutorService helpers = Executors.newFixedThreadPool(helperCount); + Set accepted = ConcurrentHashMap.newKeySet(); + CountDownLatch started = new CountDownLatch(helperCount); + List> calls = new CopyOnWriteArrayList<>(); + FunctionNode racer = + node( + "racer", + (ctx, message) -> { + for (int h = 0; h < helperCount; h++) { + String prefix = "h" + h + "-"; + calls.add( + helpers.submit(() -> { + started.countDown(); + for (int i = 0; ; i++) { + String sent = prefix + i; + try { + ctx.emit(Event.builder().partial(true).content(text(sent)).build()); + } catch (IllegalStateException bodyReturned) { + return; + } + accepted.add(sent); + } + })); + } + assertThat(started.await(10, SECONDS)).isTrue(); + return "out"; + }); + + RunResult result; + try { + result = run(chain(racer)); + for (Future call : calls) { + call.get(10, SECONDS); + } + } finally { + helpers.shutdownNow(); + } + + List sent = new ArrayList<>(); + result.eventsAt("wf@1/racer@1").forEach(event -> sent.add(describe(event))); + assertThat(sent.get(sent.size() - 1)).isEqualTo("out"); + assertThat(sent).containsAtLeastElementsIn(accepted); + } + } + + @Test + public void emit_bodyThrowsWhileAHelperEventWaitsToBeStored_sendsItBeforeTheFailure() + throws Exception { + // Arrange: "first" is being stored when the body throws. + SlowStore store = new SlowStore(Schedulers.io(), Duration.ofMillis(300)); + ExecutorService helper = Executors.newSingleThreadExecutor(); + AtomicReference> first = new AtomicReference<>(); + FunctionNode failing = + node( + "failing", + (ctx, message) -> { + first.set(helper.submit(() -> ctx.emit(text("first")))); + assertThat(store.appendsStarted.tryAcquire(10, SECONDS)).isTrue(); + throw new IOException("down"); + }); + + RunResult result; + try { + result = run(chain(failing), Schedulers.single(), store); + first.get().get(5, SECONDS); + } finally { + helper.shutdownNow(); + } + + assertThat(result.eventsAt("wf@1/failing@1").stream().map(FunctionNodeTest::describe)) + .contains("first"); + assertThat(result.errorMessage("IOException")).isEqualTo("down"); + } + + @Test + public void emit_afterTheBodyReturned_throwsIllegalState() { + // A helper makes the late call, so an admitted call times out instead of hanging the test. + ExecutorService helper = Executors.newSingleThreadExecutor(); + AtomicReference bodyContext = new AtomicReference<>(); + FunctionNode reporter = + node( + "reporter", + (ctx, message) -> { + bodyContext.set(ctx); + return null; + }); + run(chain(reporter)).assertCompleted(); + + ExecutionException error; + try { + Future late = helper.submit(() -> bodyContext.get().emit(text("late"))); + error = assertThrows(ExecutionException.class, () -> late.get(10, SECONDS)); + } finally { + helper.shutdownNow(); + } + + assertThat(error).hasCauseThat().isInstanceOf(IllegalStateException.class); + assertThat(error) + .hasCauseThat() + .hasMessageThat() + .contains("works only while the FunctionNode body of this context's node runs"); + } + + @Test + public void runNode_underItsContextAfterTheBodyReturned_failsWithIllegalState() { + // A helper makes the late run, so a run that hangs times out instead of hanging the test. + ExecutorService helper = Executors.newSingleThreadExecutor(); + AtomicReference bodyContext = new AtomicReference<>(); + AtomicInteger runs = new AtomicInteger(); + FunctionNode reporter = + node( + "reporter", + (ctx, message) -> { + if (runs.incrementAndGet() == 1) { + bodyContext.set(ctx); + return null; + } + ctx.emit(text("late")); + return "late out"; + }); + run(chain(reporter)).assertCompleted(); + + ExecutionException error; + try { + Future late = + helper.submit(() -> + reporter.runNode(bodyContext.get(), text("again")).ignoreElements().blockingAwait()); + error = assertThrows(ExecutionException.class, () -> late.get(10, SECONDS)); + } finally { + helper.shutdownNow(); + } + + assertThat(error).hasCauseThat().isInstanceOf(IllegalStateException.class); + assertThat(error) + .hasCauseThat() + .hasMessageThat() + .contains("runs only through the workflow runner"); + } + + @Test + public void runNode_underTheContextOfARunThatAskedForConfirmation_failsWithIllegalState() { + // The first run asks and never starts its body. A late run under its context must not run. + ExecutorService helper = Executors.newSingleThreadExecutor(); + AtomicReference askingContext = new AtomicReference<>(); + FunctionNode pay = + FunctionNodes.builder(Content.class, Object.class) + .name("pay") + .requireConfirmation((ctx, message) -> { + boolean first = askingContext.get() == null; + askingContext.compareAndSet(null, ctx); + return first; + }) + .body((ctx, message) -> { + ctx.emit(text("charging")); + return "paid"; + }) + .build(); + RunResult asked = run(chain(pay)); + asked.assertCompleted(); + assertThat(confirmationRequests(asked.events())).hasSize(1); + + ExecutionException error; + try { + Future late = + helper.submit(() -> + pay.runNode(askingContext.get(), text("again")).ignoreElements().blockingAwait()); + error = assertThrows(ExecutionException.class, () -> late.get(10, SECONDS)); + } finally { + helper.shutdownNow(); + } + + assertThat(error).hasCauseThat().isInstanceOf(IllegalStateException.class); + assertThat(error) + .hasCauseThat() + .hasMessageThat() + .contains("runs only through the workflow runner"); + } + + @Test + public void runNode_fromItsConfirmationPredicate_failsAndTheNodeRunsOn() { + // Arrange: the predicate runs the node itself before the node's own body starts. + AtomicReference> self = new AtomicReference<>(); + AtomicReference nestedError = new AtomicReference<>(); + AtomicInteger predicateCalls = new AtomicInteger(); + FunctionNode pay = + FunctionNodes.builder(Content.class, Object.class) + .name("pay") + .requireConfirmation((ctx, message) -> { + if (predicateCalls.incrementAndGet() > 1) { + return false; // Reached only if the nested run starts. + } + nestedError.set( + assertThrows( + IllegalStateException.class, + () -> + self.get() + .runNode(ctx, text("nested")) + .ignoreElements() + .blockingAwait())); + return false; + }) + .body((ctx, message) -> { + // A nested run that ran anyway would wait here for a store that never comes. + if (!message.text().equals("nested")) { + ctx.emit(text("charging")); + } + return "paid"; + }) + .build(); + self.set(pay); + + RunResult result = run(chain(pay)); result.assertCompleted(); - assertThat(result.outputAt("wf@1/orders@1")) - .isEqualTo(ImmutableList.of(ImmutableMap.of("id", "a1", "quantity", 2))); + assertThat(nestedError.get()).hasMessageThat().contains("call ctx.runNode"); + assertThat(result.eventsAt("wf@1/pay@1").stream().map(FunctionNodeTest::describe)) + .containsExactly("charging", "paid") + .inOrder(); } @Test - public void of_genericOutputOfATypeJacksonCannotRead_isWritten() { - // Arrange: an output port need not be readable. - FunctionNode> boxed = - FunctionNodes.of( - "boxed", - Content.class, - new TypeReference>() {}, - (ctx, m) -> new Box<>(new NoCreator("x", 1))); + public void emit_fromAChildRunWithCtxRunNode_sendsUnderTheChildsPath() { + FunctionNode child = + node( + "child", + (ctx, message) -> { + ctx.emit(text("child working")); + return "child out"; + }); + FunctionNode parent = + FunctionNodes.builder(Content.class, Object.class) + .name("parent") + .rerunOnResume(true) + .body((ctx, message) -> { + ctx.emit(text("parent working")); + return ctx.runNode(child, message).blockingGet(); + }) + .build(); - RunResult result = run(chain(boxed)); + RunResult result = run(chain(parent)); result.assertCompleted(); - assertThat(result.outputAt("wf@1/boxed@1")) - .isEqualTo(ImmutableMap.of("item", ImmutableMap.of("label", "x"))); + assertThat(result.eventsAt("wf@1/parent@1/child@1").stream().map(FunctionNodeTest::describe)) + .containsExactly("child working", "child out") + .inOrder(); + assertThat(result.eventsAt("wf@1/parent@1").stream().map(FunctionNodeTest::describe)) + .containsExactly("parent working", "child out") + .inOrder(); } @Test - public void of_typeReadThroughADelegatingMapCreator_roundTrips() { - FunctionNode makeBag = - FunctionNodes.of( - "make_bag", - Content.class, - Bag.class, - (ctx, m) -> new Bag(ImmutableMap.of("name", "x"))); - FunctionNode readBag = - FunctionNodes.of("read_bag", Bag.class, String.class, (ctx, bag) -> "bag " + bag.getName()); + public void emit_fromTheConfirmationPredicate_throwsIllegalState() { + FunctionNode transfer = + transfer(new AtomicInteger()) + .requireConfirmation((ctx, message) -> { + ctx.emit(text("checking")); + return false; + }) + .build(); - RunResult result = run(chain(makeBag, readBag)); + RunResult result = run(chain(transfer)); - result.assertCompleted(); - assertThat(result.outputAt("wf@1/make_bag@1")).isEqualTo(ImmutableMap.of("name", "x")); - assertThat(result.outputAt("wf@1/read_bag@1")).isEqualTo("bag x"); + assertThat(result.errorMessage("IllegalStateException")) + .contains("works only while the FunctionNode body of this context's node runs"); } @Test - public void of_interfaceOutputWithoutACreator_failsAtBuild() { - WorkflowConfigurationException error = - assertThrows( - WorkflowConfigurationException.class, - () -> FunctionNodes.of("n", String.class, Decision.class, (ctx, in) -> null)); + public void emit_helperInterruptedWhileWaiting_throwsCancellationAndStaysInterrupted() + throws Exception { + // Arrange: the store never finishes a node event, so the helper's call waits. + SlowStore store = new SlowStore(Schedulers.io(), Duration.ofHours(1)); + ExecutorService helper = Executors.newSingleThreadExecutor(); + AtomicReference bodyContext = new AtomicReference<>(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch released = new CountDownLatch(1); + AtomicReference thrown = new AtomicReference<>(); + AtomicBoolean interruptedAfter = new AtomicBoolean(); + FunctionNode waiter = + node( + "waiter", + (ctx, message) -> { + bodyContext.set(ctx); + started.countDown(); + assertThat(Uninterruptibles.awaitUninterruptibly(released, 10, SECONDS)).isTrue(); + return "dropped"; + }); + Runner runner = runner(chain(waiter), /* scheduler= */ null, store); + Session session = runner.sessionService().createSession(APP, USER).blockingGet(); + Disposable run = runner.runAsync(USER, session.id(), text("go")).subscribe(); + try { + assertThat(started.await(10, SECONDS)).isTrue(); + Future call = + helper.submit(() -> { + try { + bodyContext.get().emit(text("h")); + } catch (Throwable t) { + thrown.set(t); + interruptedAfter.set(Thread.currentThread().isInterrupted()); + } + }); + assertThat(store.appendsStarted.tryAcquire(10, SECONDS)).isTrue(); + + helper.shutdownNow(); // Interrupts the waiting helper. + + call.get(10, SECONDS); + } finally { + released.countDown(); + run.dispose(); + helper.shutdownNow(); + } + assertThat(thrown.get()).isInstanceOf(CancellationException.class); + assertThat(interruptedAfter.get()).isTrue(); + } + + @Test + public void emit_duplicateIdWhileTheFirstCallWaits_throwsIllegalArgument() throws Exception { + // Arrange: the store never finishes a node event, so the first call waits until the cancel. + SlowStore store = new SlowStore(Schedulers.io(), Duration.ofHours(1)); + ExecutorService helper = Executors.newSingleThreadExecutor(); + AtomicReference bodyContext = new AtomicReference<>(); + FunctionNode stuck = + node( + "stuck", + (ctx, message) -> { + bodyContext.set(ctx); + ctx.emit(Event.builder().id("x").content(text("stuck")).build()); + return "dropped"; + }); + Runner runner = runner(chain(stuck), /* scheduler= */ null, store); + Session session = runner.sessionService().createSession(APP, USER).blockingGet(); + Disposable run = runner.runAsync(USER, session.id(), text("go")).subscribe(); + ExecutionException error; + try { + assertThat(store.appendsStarted.tryAcquire(10, SECONDS)).isTrue(); + Future duplicate = + helper.submit( + () -> bodyContext.get().emit(Event.builder().id("x").content(text("again")).build())); + error = assertThrows(ExecutionException.class, () -> duplicate.get(10, SECONDS)); + } finally { + run.dispose(); + helper.shutdownNow(); + } + assertThat(error).hasCauseThat().isInstanceOf(IllegalArgumentException.class); assertThat(error) + .hasCauseThat() .hasMessageThat() - .contains("interface or abstract class without a Jackson creator"); + .contains("still waits on an event with this ID"); } @Test - public void of_untypedAndAdkPorts_needNoJacksonShape() { - FunctionNode echo = - FunctionNodes.of("echo", Content.class, Object.class, (ctx, message) -> message.text()); - FunctionNode sink = - FunctionNodes.of("sink", Object.class, Void.class, (ctx, in) -> null); + public void emit_eventWithOutput_throwsIllegalArgument() { + AtomicReference error = new AtomicReference<>(); + FunctionNode eager = + node( + "eager", + (ctx, message) -> { + error.set( + assertThrows( + IllegalArgumentException.class, + () -> ctx.emit(Event.builder().output("too soon").build()))); + return "done"; + }); - RunResult result = run(chain(echo, sink)); + run(chain(eager)).assertCompleted(); - result.assertCompleted(); - assertThat(result.outputAt("wf@1/echo@1")).isEqualTo("go"); + assertThat(error.get()).hasMessageThat().contains("return the value from the body"); } @Test @@ -3706,8 +4818,239 @@ public void of_bodyReturningARequestInput_failsTheNode() { assertThat(result.errorMessage("IllegalArgumentException")) .isEqualTo( - "node 'ask' returned a RequestInput, which a NodeBody cannot send; return the output" - + " instead."); + "node 'ask' returned a RequestInput, which a NodeBody cannot send; call" + + " FunctionNodes.requestInput(...) instead."); + } + + @Test + public void requestInput_caughtAsException_stillPausesTheNode() { + FunctionNode asker = + FunctionNodes.of( + "asker", + Content.class, + String.class, + (ctx, message) -> { + try { + return FunctionNodes.requestInput("ok_1", "ok?"); + } catch (Exception e) { + return "swallowed"; + } + }); + WorkflowSession session = new WorkflowSession(chain(asker, captureInput("reply"))); + + ImmutableList paused = session.turn(go()); + ImmutableList events = session.turn(requestResult("ok_1", "yes")); + + assertThat(WorkflowResumeTesting.pathsOf(paused)).doesNotContain("wf@1/reply@1"); + assertThat(replyInput(events)).isEqualTo("yes"); + } + + @Test + public void requestInput_fromACustomNode_asksAndOutputsTheAnswer() { + Node asker = + new TestNode( + "asker", + (ctx, input) -> Flowable.fromCallable(() -> FunctionNodes.requestInput("ok_1"))); + WorkflowSession session = new WorkflowSession(chain(asker, captureInput("reply"))); + + ImmutableList paused = session.turn(go()); + ImmutableList events = session.turn(requestResult("ok_1", "yes")); + + assertThat(WorkflowResumeTesting.pathsOf(paused)).doesNotContain("wf@1/reply@1"); + assertThat(replyInput(events)).isEqualTo("yes"); + } + + @Test + public void requestInput_caughtAsThrowable_doesNotAskAndOutputsTheFallback() { + FunctionNode asker = + FunctionNodes.of( + "asker", + Content.class, + String.class, + (ctx, message) -> { + try { + return FunctionNodes.requestInput("ok_1", "ok?"); + } catch (Throwable t) { + return "fallback"; + } + }); + + ImmutableList events = + new WorkflowSession(chain(asker, captureInput("reply"))).turn(go()); + + assertThat(WorkflowResumeTesting.requests(events)).isEmpty(); + assertThat(replyInput(events)).isEqualTo("fallback"); + } + + @Test + public void requestInput_recordPayloadWithADate_sendsTheDateAsText() { + FunctionNode asker = + FunctionNodes.of( + "asker", + Content.class, + String.class, + (ctx, message) -> + FunctionNodes.requestInput( + "day_1", "Approve?", new Leave(LocalDate.of(2026, 12, 24)))); + + ImmutableList paused = new WorkflowSession(chain(asker)).turn(go()); + + assertThat(onlyRequest(paused).args().orElseThrow()) + .containsEntry(RequestInput.PAYLOAD_KEY, ImmutableMap.of("day", "2026-12-24")); + } + + @Test + public void requestInput_messageAndResponseSchema_sendsTheMessageAndChecksTheAnswer() { + FunctionNode asker = + FunctionNodes.of( + "asker", + Content.class, + Object.class, + (ctx, message) -> + FunctionNodes.requestInput( + "age_1", "How old?", /* payload= */ null, WorkflowResumeTesting.AGE_SCHEMA)); + WorkflowSession session = new WorkflowSession(chain(asker, captureInput("reply"))); + + ImmutableList paused = session.turn(go()); + WorkflowResumeTesting.FailedTurn turn = + session.failingTurn(requestAnswer("age_1", ImmutableMap.of("age", "old"))); + + assertThat(onlyRequest(paused).args().orElseThrow()) + .containsEntry(RequestInput.MESSAGE_KEY, "How old?"); + assertThat(turn.error()).hasMessageThat().startsWith("Validation failed for interrupt age_1:"); + assertThat(turn.error()).hasMessageThat().doesNotContain("old"); + } + + @Test + public void requestInput_payloadWithNoJsonForm_failsTheAskingNode() { + FunctionNode asker = + FunctionNodes.of( + "asker", + Content.class, + String.class, + (ctx, message) -> FunctionNodes.requestInput("ok_1", "ok?", new Object())); + + WorkflowResumeTesting.FailedTurn turn = new WorkflowSession(chain(asker)).failingTurn(go()); + + assertThat(turn.error()) + .hasMessageThat() + .isEqualTo("payload of requestInput has no JSON form as java.lang.Object."); + assertThat( + turn.events().stream() + .filter(event -> event.errorCode().isPresent()) + .map(WorkflowResumeTesting::pathOf)) + .containsExactly("wf@1/asker@1"); + } + + @Test + public void requestInput_responseSchemaWithNoJsonForm_failsTheAskingNode() { + Schema noJsonForm = Schema.builder().default_(new Object()).build(); + FunctionNode asker = + FunctionNodes.of( + "asker", + Content.class, + String.class, + (ctx, message) -> + FunctionNodes.requestInput("ok_1", "ok?", /* payload= */ null, noJsonForm)); + + WorkflowResumeTesting.FailedTurn turn = new WorkflowSession(chain(asker)).failingTurn(go()); + + assertThat(turn.error()).isInstanceOf(IllegalArgumentException.class); + assertThat( + turn.events().stream() + .filter(event -> event.errorCode().isPresent()) + .map(WorkflowResumeTesting::pathOf)) + .containsExactly("wf@1/asker@1"); + } + + @Test + public void runNode_childPauseCaughtAsException_stillPausesTheParent() { + FunctionNode parent = + FunctionNodes.builder(Content.class, Object.class) + .name("parent") + .rerunOnResume(true) + .body((ctx, message) -> { + try { + return ctx.runNode( + longRunningAsk("asker", "name_1"), + null, + RunNodeConfig.builder().runId("b").build()) + .blockingGet(); + } catch (Exception e) { + return "swallowed"; + } + }) + .build(); + WorkflowSession session = new WorkflowSession(chain(parent, captureInput("reply"))); + ImmutableMap details = ImmutableMap.of("name", "John"); + + ImmutableList paused = session.turn(go()); + ImmutableList events = session.turn(answer("name_1", details)); + + assertThat(WorkflowResumeTesting.pathsOf(paused)).doesNotContain("wf@1/parent@1"); + assertThat(WorkflowResumeTesting.pathsOf(paused)).doesNotContain("wf@1/reply@1"); + assertThat(replyInput(events)).isEqualTo(details); + } + + @Test + public void builder_rerunOnResumeBodyAsking_readsTheAnswerFromResumeInputs() { + FunctionNode asker = + FunctionNodes.builder(Content.class, String.class) + .name("asker") + .rerunOnResume(true) + .body((ctx, message) -> + ctx.resumeInputs().containsKey("ok_1") + ? "answered " + ctx.resumeInputs().get("ok_1") + : FunctionNodes.requestInput("ok_1", "ok?")) + .build(); + WorkflowSession session = new WorkflowSession(chain(asker, captureInput("reply"))); + session.advance(go()); + + ImmutableList events = session.turn(requestResult("ok_1", "yes")); + + assertThat(replyInput(events)).isEqualTo("answered yes"); + } + + @Test + public void of_requestInputAnsweredWithAMapOnARecordOutput_keepsKeysTheRecordLacks() { + // As in ADK Kotlin, the answer is not read as an Order, so it keeps the key Order lacks. + WorkflowSession session = + new WorkflowSession(chain(asker(Order.class, "o_1"), captureInput("reply"))); + session.advance(go()); + ImmutableMap answer = ImmutableMap.of("id", "a1", "quantity", 3L, "gift", true); + + ImmutableList events = session.turn(requestResult("o_1", answer)); + + assertThat(WorkflowResumeTesting.outputEventAt(events, "wf@1/asker@1").output()) + .hasValue(answer); + assertThat(replyInput(events)).isEqualTo(answer); + } + + @Test + public void of_requestInputAnsweredWithAnotherType_outputsItAndTheSuccessorRejectsIt() { + FunctionNode twice = + FunctionNodes.of("twice", Integer.class, Integer.class, (ctx, n) -> n * 2); + WorkflowSession session = new WorkflowSession(chain(asker(Integer.class, "n_1"), twice)); + session.advance(go()); + + WorkflowResumeTesting.FailedTurn turn = session.failingTurn(requestResult("n_1", "abc")); + + assertThat(WorkflowResumeTesting.outputEventAt(turn.events(), "wf@1/asker@1").output()) + .hasValue("abc"); + assertThat(turn.error()) + .hasMessageThat() + .isEqualTo("validation error: input of node 'twice' does not match java.lang.Integer."); + } + + @Test + public void of_requestInputAnsweredOnAVoidOutput_outputsTheAnswer() { + WorkflowSession session = + new WorkflowSession(chain(asker(Void.class, "p_1"), captureInput("reply"))); + session.advance(go()); + + ImmutableList events = session.turn(requestResult("p_1", "yes")); + + assertThat(replyInput(events)).isEqualTo("yes"); } @Test @@ -4501,7 +5844,8 @@ public void resume_answerWithAHint_bodySeesTheHint() { public void runNode_answerThatIsNotAMap_throwsWorkflowDataException() { FunctionNode transfer = transfer(new AtomicInteger()).requireConfirmation(true).build(); - Context context = nodeContext(ImmutableMap.of("wf_tool_confirmation:transfer@1", "approved")); + Context context = + nodeContext(transfer, ImmutableMap.of("wf_tool_confirmation:transfer@1", "approved")); Flowable run = transfer.runNode(context, null); WorkflowDataException error = assertThrows(WorkflowDataException.class, run::blockingFirst); @@ -4518,6 +5862,7 @@ public void runNode_answerWithConfirmedThatIsNotABoolean_throwsWorkflowDataExcep transfer(new AtomicInteger()).requireConfirmation(true).build(); Context context = nodeContext( + transfer, ImmutableMap.of( "wf_tool_confirmation:transfer@1", ImmutableMap.of("confirmed", "yes"))); Flowable run = transfer.runNode(context, null); @@ -4536,6 +5881,7 @@ public void runNode_answerWithHintThatIsNotAString_throwsWorkflowDataException() transfer(new AtomicInteger()).requireConfirmation(true).build(); Context context = nodeContext( + transfer, ImmutableMap.of( "wf_tool_confirmation:transfer@1", ImmutableMap.of("confirmed", true, "hint", 5))); Flowable run = transfer.runNode(context, null); @@ -4554,6 +5900,7 @@ public void runNode_answerWithAKeyOutsideTheSchema_throwsWorkflowDataException() transfer(new AtomicInteger()).requireConfirmation(true).build(); Context context = nodeContext( + transfer, ImmutableMap.of( "wf_tool_confirmation:transfer@1", ImmutableMap.of("confirmed", true, "approved", true))); @@ -4778,11 +6125,11 @@ private static FunctionNode delegatingToASilentChild( } /** Returns the context of a run of node {@code transfer} that resumes with {@code answers}. */ - private static Context nodeContext(Map answers) { + private static Context nodeContext(Node node, Map answers) { return new Context( TestUtils.createInvocationContext(TestUtils.createRootAgent()), NodeExecutionState.builder() - .node(TestNode.of("transfer")) + .node(node) .eventSink((event, nodeFailure) -> Completable.complete()) .runId("1") .resumeInputs(answers) @@ -4890,6 +6237,12 @@ private static Workflow chain(Node... nodes) { return Workflow.builder().name("wf").edges(edges.toArray(new Edge[0])).build(); } + /** Returns a node {@code asker} with a {@code type} output that asks for input as {@code id}. */ + private static FunctionNode asker(Class type, String id) { + return FunctionNodes.of( + "asker", Content.class, type, (ctx, message) -> FunctionNodes.requestInput(id, "?")); + } + /** * Returns a node {@code router} whose body gives its output to a child {@code child} that outputs * {@code childOutput}, and returns nothing itself. @@ -4915,9 +6268,14 @@ private static RunResult run(Node root) { return run(root, null); } - /** Runs {@code root} as an app's root node through a {@link Runner} on a "go" message. */ private static RunResult run(Node root, @Nullable Scheduler scheduler) { - Runner runner = runner(root, scheduler); + return run(root, scheduler, new InMemorySessionService()); + } + + /** Runs {@code root} as an app's root node through a {@link Runner} on a "go" message. */ + private static RunResult run( + Node root, @Nullable Scheduler scheduler, BaseSessionService sessionService) { + Runner runner = runner(root, scheduler, sessionService); Session session = runner.sessionService().createSession(APP, USER).blockingGet(); List events = new ArrayList<>(); AtomicReference error = new AtomicReference<>(); @@ -4949,12 +6307,138 @@ private static Future runInline( } private static Runner runner(Node root, @Nullable Scheduler scheduler) { + return runner(root, scheduler, new InMemorySessionService()); + } + + private static Runner runner( + Node root, @Nullable Scheduler scheduler, BaseSessionService sessionService) { return Runner.builder() .app(App.builder().name(APP).rootNode(root).build()) + .sessionService(sessionService) .scheduler(scheduler) .build(); } + /** Runs {@code check} on its own thread, for a bounded time, with each Runner scheduler. */ + private static void onEveryScheduler(SchedulerCheck check) throws Exception { + ExecutorService pool = Executors.newFixedThreadPool(2); + ExecutorService checks = Executors.newCachedThreadPool(); + try { + for (Scheduler scheduler : + ImmutableList.of( + Schedulers.io(), + Schedulers.trampoline(), + Schedulers.single(), + Schedulers.from(pool), + Schedulers.from(Runnable::run))) { + Future done = + checks.submit(() -> { + check.run(scheduler); + return null; + }); + try { + done.get(20, SECONDS); + } catch (ExecutionException | TimeoutException e) { + throw new AssertionError("on " + scheduler + ": " + e.getMessage(), e); + } + } + } finally { + checks.shutdownNow(); + pool.shutdownNow(); + } + } + + private interface SchedulerCheck { + void run(Scheduler scheduler) throws Exception; + } + + /** Returns a node named {@code name} that runs {@code body} on the user's message. */ + private static FunctionNode node(String name, NodeBody body) { + return FunctionNodes.of(name, Content.class, Object.class, body); + } + + private static Content text(String text) { + return Content.fromParts(Part.fromText(text)); + } + + /** Emits {@code text} and returns "returned", or the simple name of what the call threw. */ + private static String emitOutcome(Context ctx, String text) { + try { + ctx.emit(text(text)); + return "returned"; + } catch (RuntimeException e) { + return e.getClass().getSimpleName(); + } + } + + /** Returns the event's text or output, whichever it carries. */ + private static String describe(Event event) { + return event.content().map(Content::text).orElse("") + event.output().orElse(""); + } + + /** A session service that stores each slow node event {@code delay} late, on {@code appendOn}. */ + private static final class SlowStore implements BaseSessionService { + private final InMemorySessionService delegate = new InMemorySessionService(); + private final Scheduler appendOn; + private final Duration delay; + private final Predicate slow; + final List storedTexts = new CopyOnWriteArrayList<>(); + final Semaphore appendsStarted = new Semaphore(0); + + SlowStore(Scheduler appendOn, Duration delay) { + this(appendOn, delay, event -> true); + } + + SlowStore(Scheduler appendOn, Duration delay, Predicate slow) { + this.appendOn = appendOn; + this.delay = delay; + this.slow = slow; + } + + @Override + public Single appendEvent(Session session, Event event) { + if (event.nodeInfo().isEmpty() || !slow.test(event)) { + return delegate.appendEvent(session, event); + } + appendsStarted.release(); + return Single.timer(delay.toMillis(), MILLISECONDS, appendOn) + .flatMap(unused -> delegate.appendEvent(session, event)) + .doOnSuccess(unused -> storedTexts.add(describe(event))); + } + + // BaseSessionService's only abstract createSession overload is deprecated. + @SuppressWarnings("deprecation") + @Override + public Single createSession( + String appName, + String userId, + @Nullable ConcurrentMap state, + @Nullable String sessionId) { + return delegate.createSession(appName, userId, state, sessionId); + } + + @Override + public Maybe getSession( + String appName, String userId, String sessionId, Optional config) { + return delegate.getSession(appName, userId, sessionId, config); + } + + @Override + public Single listSessions(String appName, String userId) { + return delegate.listSessions(appName, userId); + } + + @Override + public Completable deleteSession(String appName, String userId, String sessionId) { + return delegate.deleteSession(appName, userId, sessionId); + } + + @Override + public Single listEvents(String appName, String userId, String sessionId) { + return delegate.listEvents(appName, userId, sessionId); + } + } + /** The events a run streamed and the error it failed with, if any. */ private record RunResult(ImmutableList events, @Nullable Throwable error) { diff --git a/core/src/test/java/com/google/adk/workflow/TimeOffWorkflowTest.java b/core/src/test/java/com/google/adk/workflow/TimeOffWorkflowTest.java new file mode 100644 index 000000000..acceaea6b --- /dev/null +++ b/core/src/test/java/com/google/adk/workflow/TimeOffWorkflowTest.java @@ -0,0 +1,145 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.adk.workflow; + +import static com.google.adk.workflow.WorkflowResumeTesting.go; +import static com.google.adk.workflow.WorkflowResumeTesting.onlyRequest; +import static com.google.adk.workflow.WorkflowResumeTesting.outputEventAt; +import static com.google.adk.workflow.WorkflowResumeTesting.requestResult; +import static com.google.adk.workflow.WorkflowResumeTesting.requests; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.truth.Truth.assertThat; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.adk.events.Event; +import com.google.adk.workflow.WorkflowResumeTesting.WorkflowSession; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.genai.types.Content; +import com.google.genai.types.FunctionCall; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Runs a time-off workflow through a {@code Runner}: each requested day is checked in parallel, a + * day short on cover asks the team lead, and a manager decides on the days the checks allow. Each + * ask is a {@link FunctionNodes#requestInput} call in a body, and its answer is the node's output. + */ +@RunWith(JUnit4.class) +public final class TimeOffWorkflowTest { + + /** The day the team is short on cover, which the team lead must approve. */ + private static final String SHORT_DAY = "2026-12-24"; + + private static final String MANAGER_APPROVAL_ID = "manager_approval"; + + record TimeOffRequest(String employee, ImmutableList days) {} + + record DayRequest(String employee, String day) {} + + record DayCheck(String day, boolean approved) {} + + record TimeOffDecision(boolean approved, int days) {} + + @Test + public void timeOffWorkflow_dayItemAndManagerAnswered_finishesWithTheManagersDecision() { + // The short day pauses its parallel item, then the manager decides on the rest. + WorkflowSession session = + new WorkflowSession( + timeOffWorkflow( + new TimeOffRequest( + "ana", ImmutableList.of("2026-12-23", SHORT_DAY, "2026-12-28")))); + + ImmutableList paused = session.turn(go()); + ImmutableList checked = + session.turn( + requestResult( + "cover_" + SHORT_DAY, + ImmutableMap.of("day", SHORT_DAY, "approved", false, "by", "lead"))); + ImmutableList decided = + session.turn( + requestResult( + MANAGER_APPROVAL_ID, + ImmutableMap.of("approved", true, "days", 2L, "note", "enjoy"))); + + FunctionCall coverAsk = onlyRequest(paused); + assertThat(coverAsk.id()).hasValue("cover_" + SHORT_DAY); + assertThat(coverAsk.args().orElseThrow()) + .containsEntry( + RequestInput.PAYLOAD_KEY, ImmutableMap.of("employee", "ana", "day", SHORT_DAY)); + // As in ADK Kotlin, an answer is not read as the output type, so it keeps keys DayCheck lacks. + assertThat(outputEventAt(checked, "wf@1/check_day@1").output()) + .hasValue( + ImmutableList.of( + ImmutableMap.of("day", "2026-12-23", "approved", true), + ImmutableMap.of("day", SHORT_DAY, "approved", false, "by", "lead"), + ImmutableMap.of("day", "2026-12-28", "approved", true))); + FunctionCall managerAsk = onlyRequest(checked); + assertThat(managerAsk.id()).hasValue(MANAGER_APPROVAL_ID); + assertThat(managerAsk.args().orElseThrow()) + .containsEntry(RequestInput.PAYLOAD_KEY, ImmutableList.of("2026-12-23", "2026-12-28")); + assertThat(outputEventAt(decided, "wf@1/manager_approval@1").output()) + .hasValue(ImmutableMap.of("approved", true, "days", 2L, "note", "enjoy")); + assertThat(requests(decided)).isEmpty(); + } + + /** Builds the workflow for {@code request}; a node stands in for an agent that extracts it. */ + private static Workflow timeOffWorkflow(TimeOffRequest request) { + FunctionNode> processRequest = + FunctionNodes.of( + "process_request", + Content.class, + new TypeReference>() {}, + (ctx, message) -> + request.days().stream() + .map(day -> new DayRequest(request.employee(), day)) + .collect(toImmutableList())); + FunctionNode checkDay = + FunctionNodes.of( + "check_day", + DayRequest.class, + DayCheck.class, + (ctx, dayRequest) -> + dayRequest.day().equals(SHORT_DAY) + ? FunctionNodes.requestInput( + "cover_" + dayRequest.day(), + "The team is short on cover that day. Approve it?", + dayRequest) + : new DayCheck(dayRequest.day(), true)); + FunctionNode, TimeOffDecision> managerApproval = + FunctionNodes.of( + "manager_approval", + new TypeReference>() {}, + TimeOffDecision.class, + (ctx, checks) -> + FunctionNodes.requestInput( + MANAGER_APPROVAL_ID, + "Please review these days off.", + checks.stream() + .filter(DayCheck::approved) + .map(DayCheck::day) + .collect(toImmutableList()))); + return Workflow.builder() + .name("wf") + .edges(g -> + g.fromStart() + .chain(processRequest, ParallelWorkers.parallelNode(checkDay), managerApproval)) + .build(); + } +} diff --git a/core/src/test/java/com/google/adk/workflow/WorkflowResumeTesting.java b/core/src/test/java/com/google/adk/workflow/WorkflowResumeTesting.java index 62d2bc397..1fd4189c1 100644 --- a/core/src/test/java/com/google/adk/workflow/WorkflowResumeTesting.java +++ b/core/src/test/java/com/google/adk/workflow/WorkflowResumeTesting.java @@ -117,6 +117,23 @@ static Content requestAnswer(String interruptId, Map response) { return answer(interruptId, response, RequestInput.FUNCTION_CALL_NAME); } + /** Returns the answer {@code {"result": result}} to the request {@code interruptId}. */ + static Content requestResult(String interruptId, Object result) { + return requestAnswer(interruptId, ImmutableMap.of("result", result)); + } + + /** Returns the {@code RequestInput} calls among {@code events}. */ + static ImmutableList requests(List events) { + return events.stream() + .flatMap(event -> event.functionCalls().stream()) + .filter(call -> call.name().orElse("").equals(RequestInput.FUNCTION_CALL_NAME)) + .collect(toImmutableList()); + } + + static FunctionCall onlyRequest(List events) { + return requests(events).stream().collect(onlyElement()); + } + static Content callContent(String name, String id) { return Content.builder() .role("model")