diff --git a/core/src/main/java/com/google/adk/workflow/NodeRunner.java b/core/src/main/java/com/google/adk/workflow/NodeRunner.java index 14f004d27..cf5755785 100644 --- a/core/src/main/java/com/google/adk/workflow/NodeRunner.java +++ b/core/src/main/java/com/google/adk/workflow/NodeRunner.java @@ -16,6 +16,7 @@ package com.google.adk.workflow; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.isNullOrEmpty; @@ -37,6 +38,7 @@ import io.reactivex.rxjava3.core.Completable; import io.reactivex.rxjava3.core.Flowable; import io.reactivex.rxjava3.core.Maybe; +import io.reactivex.rxjava3.core.Scheduler; import io.reactivex.rxjava3.core.Single; import io.reactivex.rxjava3.disposables.CompositeDisposable; import io.reactivex.rxjava3.processors.FlowableProcessor; @@ -201,12 +203,32 @@ public static Maybe runChild( checkNotNull(parent); checkNotNull(node); checkNotNull(config); - return Maybe.defer(() -> startChild(parent, node, nodeInput, config)); + return Maybe.defer( + () -> startChild(parent, node, nodeInput, config, /* reservedRunId= */ null)); + } + + /** + * Runs {@code node} as {@link #runChild(Context, Node, Object, RunNodeConfig)} does, using the + * {@code runId} that the caller reserved with {@link NodeExecutionState#nextChildRunId}. {@code + * config} must leave its run ID unset. + */ + static Maybe runChild( + Context parent, Node node, @Nullable Object nodeInput, RunNodeConfig config, String runId) { + checkNotNull(parent); + checkNotNull(node); + checkNotNull(config); + checkNotNull(runId); + checkArgument(config.runId().isEmpty(), "The config sets its own run ID."); + return Maybe.defer(() -> startChild(parent, node, nodeInput, config, runId)); } /** Runs {@code node} as a child of {@code parent} and turns the child's run into its result. */ private static Maybe startChild( - Context parent, Node node, @Nullable Object nodeInput, RunNodeConfig config) { + Context parent, + Node node, + @Nullable Object nodeInput, + RunNodeConfig config, + @Nullable String reservedRunId) { NodeExecutionState parentState = parent.requireNodeState(); checkState( parentState.node().rerunOnResume(), @@ -220,7 +242,10 @@ private static Maybe startChild( if (delegatesOutput) { parentState.claimOutputDelegation(); } - String runId = config.runId().orElseGet(() -> parentState.nextChildRunId(node.name())); + String runId = + reservedRunId != null + ? reservedRunId + : config.runId().orElseGet(() -> parentState.nextChildRunId(node.name())); return sharedChildRun(parent, node, nodeInput, runId, config) .doOnError( error -> { @@ -536,8 +561,7 @@ private Completable withTimeout(Completable run) { .andThen(Completable.error(() -> new NodeTimeoutException(node.name(), limit))); io.opentelemetry.context.Context traceContext = io.opentelemetry.context.Context.current(); // Off the caller's thread, so a body that blocks does not block the caller. - return Completable.timer(0, NANOSECONDS, invocationContext.scheduler()) - // Unlike subscribeOn, canceling this hop does not interrupt the handling of node events. + return completeOn(invocationContext.scheduler()) .andThen(run.compose(Tracing.withContext(traceContext))) // convert saturates, so a timeout too long to count in nanoseconds waits about 292 years. .timeout(NANOSECONDS.convert(limit), NANOSECONDS, timedOut) @@ -802,6 +826,15 @@ static String segment(String name, String runId) { return name + "@" + runId; } + /** + * Returns a {@link Completable} that completes on {@code scheduler}, so work chained after it + * starts on that scheduler instead of the caller's thread. Unlike {@code subscribeOn}, canceling + * the chained work does not interrupt the thread it runs on. + */ + static Completable completeOn(Scheduler scheduler) { + return Completable.timer(0, NANOSECONDS, scheduler); + } + /** Returns whether {@code event} is a final event whose content is the node's output. */ private static boolean isMessageAsOutput(Event event) { return !event.partial().orElse(false) diff --git a/core/src/main/java/com/google/adk/workflow/ParallelWorkers.java b/core/src/main/java/com/google/adk/workflow/ParallelWorkers.java index 129d4f5c6..6ef6af115 100644 --- a/core/src/main/java/com/google/adk/workflow/ParallelWorkers.java +++ b/core/src/main/java/com/google/adk/workflow/ParallelWorkers.java @@ -22,12 +22,14 @@ import com.google.adk.JsonBaseModel; import com.google.adk.agents.Context; import com.google.adk.annotations.Experimental; +import com.google.adk.telemetry.Tracing; import com.google.common.collect.ImmutableList; import com.google.genai.types.Content; import com.google.genai.types.Part; import com.google.genai.types.Schema; import com.google.genai.types.Type; import io.reactivex.rxjava3.core.Flowable; +import io.reactivex.rxjava3.core.Maybe; import io.reactivex.rxjava3.core.Single; import java.util.ArrayList; import java.util.Collections; @@ -84,7 +86,7 @@ private static final class ParallelWorkerNode extends BaseNode { JsonBaseModel.getMapper() .readerFor(Object.class) .with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); - // No run ID: items subscribe in input order, so each gets the same generated ID on resume. + // No run ID: FanOut reserves IDs in input order, so each item keeps its ID on resume. private static final RunNodeConfig ITEM_CONFIG = RunNodeConfig.builder().useSubBranch(true).build(); @@ -153,6 +155,11 @@ private static boolean acceptsString(Schema schema) { private final class FanOut { private final Context context; private final List items; + // Reserved up front, because the items begin on other threads in no fixed order. + private final ImmutableList runIds; + // The items start on other threads, so each restores the node's trace context. + private final io.opentelemetry.context.Context traceContext = + io.opentelemetry.context.Context.current(); // Written by the item runs, which may finish on different threads. private final AtomicReferenceArray outputs; private final AtomicReferenceArray failures; @@ -161,6 +168,13 @@ private final class FanOut { FanOut(Context context, List items) { this.context = context; this.items = items; + NodeExecutionState state = context.requireNodeState(); + ImmutableList.Builder reserved = + ImmutableList.builderWithExpectedSize(items.size()); + for (int i = 0; i < items.size(); i++) { + reserved.add(state.nextChildRunId(worker.name())); + } + this.runIds = reserved.build(); this.outputs = new AtomicReferenceArray<>(items.size()); this.failures = new AtomicReferenceArray<>(items.size()); } @@ -178,11 +192,17 @@ Flowable run() { /** Runs one item, or replays its result on resume, and emits whether it failed. */ private Single runItem(int index) { - return context - .runNode(worker, items.get(index), ITEM_CONFIG) - .doOnSuccess(output -> outputs.set(index, output)) - .ignoreElement() - .toSingleDefault(false) + Maybe run = + NodeRunner.runChild(context, worker, items.get(index), ITEM_CONFIG, runIds.get(index)); + // Off the caller's thread, so an item that blocks its thread cannot hold up the others. + return NodeRunner.completeOn(context.invocationContext().scheduler()) + .andThen(run.compose(Tracing.withContext(traceContext))) + // Not doOnSuccess, which reports a canceled item's late failure as undeliverable. + .map(output -> { + outputs.set(index, output); + return false; + }) + .defaultIfEmpty(false) .onErrorResumeNext(error -> settle(index, error)); } diff --git a/core/src/test/java/com/google/adk/workflow/ParallelWorkersTest.java b/core/src/test/java/com/google/adk/workflow/ParallelWorkersTest.java index 90af56815..2fa51865a 100644 --- a/core/src/test/java/com/google/adk/workflow/ParallelWorkersTest.java +++ b/core/src/test/java/com/google/adk/workflow/ParallelWorkersTest.java @@ -28,23 +28,35 @@ import static com.google.adk.workflow.WorkflowResumeTesting.pathsOf; import static com.google.adk.workflow.WorkflowResumeTesting.replyInput; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertThrows; import com.google.adk.agents.Context; +import com.google.adk.agents.LlmAgent; +import com.google.adk.apps.App; import com.google.adk.events.Event; +import com.google.adk.models.LlmResponse; +import com.google.adk.runner.Runner; +import com.google.adk.sessions.Session; +import com.google.adk.testing.TestLlm; import com.google.adk.testing.TestNode; import com.google.adk.testing.TestUtils; import com.google.adk.workflow.WorkflowResumeTesting.WorkflowSession; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import com.google.genai.types.Content; import com.google.genai.types.Part; import com.google.genai.types.Schema; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.context.Scope; import io.reactivex.rxjava3.core.Completable; import io.reactivex.rxjava3.core.Flowable; +import io.reactivex.rxjava3.core.Scheduler; import io.reactivex.rxjava3.core.Single; import io.reactivex.rxjava3.observers.TestObserver; import io.reactivex.rxjava3.schedulers.Schedulers; @@ -57,6 +69,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; @@ -172,14 +186,16 @@ public void runNode_listInput_runsEachItemOnItsOwnBranchAndOutputsInInputOrder() Run run = run(ParallelWorkers.parallelNode(doubler()), ImmutableList.of(1, 2, 3)); assertThat(run.output()).isEqualTo(ImmutableList.of(2, 4, 6)); - ImmutableList itemEvents = run.eventsUnder("worker@1/"); - assertThat(pathsOf(itemEvents)) - .containsExactly("worker@1/worker@1", "worker@1/worker@2", "worker@1/worker@3") - .inOrder(); + // The items run at once, so their events arrive in no fixed order. assertThat( - itemEvents.stream().map(event -> event.branch().orElse("")).collect(toImmutableList())) - .containsExactly("worker@1", "worker@2", "worker@3") - .inOrder(); + run.eventsUnder("worker@1/").stream() + .collect( + toImmutableMap( + WorkflowResumeTesting::pathOf, event -> event.branch().orElse("")))) + .containsExactly( + "worker@1/worker@1", "worker@1", + "worker@1/worker@2", "worker@2", + "worker@1/worker@3", "worker@3"); } @Test @@ -249,7 +265,7 @@ public void runNode_maxParallelWorkers_startsNoMoreItemsUntilOneFinishes() { return release.andThen(Flowable.just(item)); }); TestObserver observer = - start( + startInline( ParallelWorkers.parallelNode(worker, /* maxParallelWorkers= */ 2), ImmutableList.of(1, 2, 3, 4, 5, 6)) .test(); @@ -273,7 +289,7 @@ public void runNode_noMaxParallelWorkers_startsEveryItemAtOnce() { return release.andThen(Flowable.just(item)); }); TestObserver observer = - start(ParallelWorkers.parallelNode(worker), ImmutableList.of(1, 2, 3)).test(); + startInline(ParallelWorkers.parallelNode(worker), ImmutableList.of(1, 2, 3)).test(); int startedBeforeRelease = started.get(); release.onComplete(); @@ -283,6 +299,118 @@ public void runNode_noMaxParallelWorkers_startsEveryItemAtOnce() { .hasValue(ImmutableList.of(1, 2, 3)); } + @Test + public void runNode_itemsThatBlockTheirThreads_runAtOnce() { + CountDownLatch allRunning = new CountDownLatch(3); + // Each item holds the thread it runs on until all three are running. + Node worker = + node(item -> + Flowable.fromCallable(() -> { + allRunning.countDown(); + awaitRelease(allRunning); + return item; + })); + + Run run = run(ParallelWorkers.parallelNode(worker), ImmutableList.of(1, 2, 3)); + + assertThat(run.output()).isEqualTo(ImmutableList.of(1, 2, 3)); + } + + @Test + public void runAsync_agentWorkersBlockedOnTheirModel_callItAtOnce() { + CountDownLatch allCalling = new CountDownLatch(3); + // Each call holds its thread until all three are in flight, like Gemini's non-streaming call. + TestLlm model = + new TestLlm(() -> + Flowable.fromCallable(() -> { + allCalling.countDown(); + awaitRelease(allCalling); + return LlmResponse.builder() + .content(Content.fromParts(Part.fromText("done"))) + .build(); + })); + LlmAgent worker = LlmAgent.builder().name("worker").model(model).build(); + + ImmutableList events = + runThroughRunner(ParallelWorkers.parallelNode(worker), "[1, 2, 3]", /* scheduler= */ null); + + assertThat(model.getRequests()).hasSize(3); + assertThat(events.stream().filter(event -> event.errorCode().isPresent())).isEmpty(); + } + + @Test + public void runAsync_itemsRunOnTheRunnersScheduler() { + ExecutorService executor = + Executors.newSingleThreadExecutor(task -> new Thread(task, "app-scheduler")); + Node worker = node(item -> Flowable.fromCallable(() -> Thread.currentThread().getName())); + + ImmutableList events; + try { + events = + runThroughRunner( + ParallelWorkers.parallelNode(worker), "[1, 2]", Schedulers.from(executor)); + } finally { + executor.shutdownNow(); + } + + assertThat(Iterables.getLast(events).output()) + .hasValue(ImmutableList.of("app-scheduler", "app-scheduler")); + } + + @Test + public void runNode_itemsStartOnOtherThreads_keepTheCallersTraceContext() { + ContextKey key = ContextKey.named("caller"); + Map seenByItem = new ConcurrentHashMap<>(); + Node worker = + node(item -> { + seenByItem.put(item, String.valueOf(io.opentelemetry.context.Context.current().get(key))); + return Flowable.just(item); + }); + + Run run; + try (Scope unused = + io.opentelemetry.context.Context.current().with(key, "parent").makeCurrent()) { + run = run(ParallelWorkers.parallelNode(worker), ImmutableList.of(1, 2, 3)); + } + + assertThat(run.output()).isEqualTo(ImmutableList.of(1, 2, 3)); + assertThat(seenByItem).containsExactly(1, "parent", 2, "parent", 3, "parent"); + } + + @Test + public void runNode_itemsStartingInReverse_keepTheirRunIdsInInputOrder() throws Exception { + List starts = new CopyOnWriteArrayList<>(); + CountDownLatch allQueued = new CountDownLatch(3); + Scheduler queueing = + Schedulers.from(start -> { + starts.add(start); + allQueued.countDown(); + }); + Map pathByItem = new ConcurrentHashMap<>(); + Node worker = + new TestNode( + "worker", + (context, item) -> { + pathByItem.put(item, context.requireNodeState().nodePath()); + return Flowable.just(item); + }); + TestObserver observer = + start( + ParallelWorkers.parallelNode(worker), + ImmutableList.of(1, 2, 3), + new CopyOnWriteArrayList<>(), + queueing) + .test(); + assertThat(allQueued.await(10, SECONDS)).isTrue(); + + // Starts the items last-in, first-out, so the third item begins first. + Lists.reverse(starts).forEach(Runnable::run); + + observer.awaitDone(10, SECONDS).assertComplete(); + assertThat(pathByItem) + .containsExactly(1, "worker@1/worker@1", 2, "worker@1/worker@2", 3, "worker@1/worker@3"); + } + @Test public void runNode_itemConfigWithRetries_retriesOnlyTheFailedItem() { Map attemptsByItem = new ConcurrentHashMap<>(); @@ -354,37 +482,60 @@ public void runNode_itemFailsUnderCap_neverStartsQueuedItems() { } @Test - public void runNode_itemFails_cancelsRunningSiblings() { - AtomicBoolean siblingCancelled = new AtomicBoolean(); - Node worker = - node( - item -> - item.equals("slow") - ? Flowable.timer(10, SECONDS).doOnCancel(() -> siblingCancelled.set(true)) - : Flowable.error(new IllegalStateException("boom"))); + public void runNode_itemFails_cancelsRunningSiblings() throws Exception { + CountDownLatch siblingCancelled = new CountDownLatch(1); + Node worker = failsOnceSlowStarts(new IllegalStateException("boom"), siblingCancelled); Run run = run(ParallelWorkers.parallelNode(worker), ImmutableList.of("slow", "bad")); assertThat(run.failure().orElseThrow()).hasMessageThat().isEqualTo("boom"); - assertThat(siblingCancelled.get()).isTrue(); + assertThat(siblingCancelled.await(10, SECONDS)).isTrue(); } @Test - public void runNode_itemThrowsAnError_propagatesItAndCancelsRunningSiblings() { - AtomicBoolean siblingCancelled = new AtomicBoolean(); - Node worker = - node( - item -> - item.equals("slow") - ? Flowable.timer(10, SECONDS).doOnCancel(() -> siblingCancelled.set(true)) - : Flowable.error(new AssertionError("fatal"))); + public void runNode_itemThrowsAnError_propagatesItAndCancelsRunningSiblings() throws Exception { + CountDownLatch siblingCancelled = new CountDownLatch(1); + Node worker = failsOnceSlowStarts(new AssertionError("fatal"), siblingCancelled); Node node = ParallelWorkers.parallelNode(worker); AssertionError thrown = assertThrows(AssertionError.class, () -> run(node, ImmutableList.of("slow", "fatal"))); assertThat(thrown).hasMessageThat().isEqualTo("fatal"); - assertThat(siblingCancelled.get()).isTrue(); + assertThat(siblingCancelled.await(10, SECONDS)).isTrue(); + } + + @Test + public void runNode_itemFails_doesNotInterruptASiblingThatBlocksItsThread() throws Exception { + CountDownLatch blockedStarted = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch blockedDone = new CountDownLatch(1); + AtomicBoolean siblingInterrupted = new AtomicBoolean(); + Node worker = + node(item -> + item.equals("blocked") + ? Flowable.fromCallable(() -> { + blockedStarted.countDown(); + try { + boolean unused = release.await(10, SECONDS); + } catch (InterruptedException e) { + siblingInterrupted.set(true); + } finally { + blockedDone.countDown(); + } + return "late"; + }) + : Flowable.defer(() -> { + awaitRelease(blockedStarted); + return Flowable.error(new IllegalStateException("boom")); + })); + + Run run = run(ParallelWorkers.parallelNode(worker), ImmutableList.of("blocked", "bad")); + release.countDown(); + + assertThat(run.failure().orElseThrow()).hasMessageThat().isEqualTo("boom"); + assertThat(blockedDone.await(10, SECONDS)).isTrue(); + assertThat(siblingInterrupted.get()).isFalse(); } @Test @@ -493,7 +644,8 @@ public void resume_underCap_startsTheQueuedItemOnceTheAskingItemIsAnswered() { return item.equals(1) ? Flowable.just(item) : Flowable.just(interrupt("ask-" + item)); }); Node items = emitter("items", ImmutableList.of(1, 2, 3)); - Node fanOut = ParallelWorkers.parallelNode(worker, /* maxParallelWorkers= */ 2); + // One at a time, so the second item asks before a slot frees for the third. + Node fanOut = ParallelWorkers.parallelNode(worker, /* maxParallelWorkers= */ 1); WorkflowSession session = new WorkflowSession( workflow( @@ -661,14 +813,47 @@ private static Single start(Node node, @Nullable Object nodeInput) { * events}. */ private static Single start(Node node, @Nullable Object nodeInput, List events) { + return start(node, nodeInput, events, /* scheduler= */ null); + } + + private static Single start( + Node node, @Nullable Object nodeInput, List events, @Nullable Scheduler scheduler) { return NodeRunner.builder() .node(node) - .invocationContext(TestUtils.createInvocationContext(TestUtils.createRootAgent())) + .invocationContext( + TestUtils.createInvocationContext(TestUtils.createRootAgent()).toBuilder() + .scheduler(scheduler) + .build()) .eventSink((event, nodeFailure) -> Completable.fromAction(() -> events.add(event))) .build() .run(nodeInput); } + /** + * Returns the run of {@code node} as the workflow root, ignoring its events. It runs on the + * trampoline scheduler, so each item starts on the subscribing thread. + */ + private static Single startInline(Node node, @Nullable Object nodeInput) { + return start(node, nodeInput, new CopyOnWriteArrayList<>(), Schedulers.trampoline()); + } + + /** Runs {@code root} as an app's root node through a {@link Runner} on {@code message}. */ + private static ImmutableList runThroughRunner( + Node root, String message, @Nullable Scheduler scheduler) { + Runner runner = + Runner.builder() + .app(App.builder().name("app").rootNode(root).build()) + .scheduler(scheduler) + .build(); + Session session = runner.sessionService().createSession("app", "user").blockingGet(); + return ImmutableList.copyOf( + runner + .runAsync("user", session.id(), Content.fromParts(Part.fromText(message))) + .timeout(30, SECONDS) + .toList() + .blockingGet()); + } + /** Returns a plain node named "worker", not a {@link BaseNode}, that runs {@code body}. */ private static Node node(Function> body) { return new TestNode("worker", (context, item) -> Flowable.defer(() -> body.apply(item))); @@ -730,6 +915,23 @@ private static Node failsOn(Object bad) { : Flowable.just(item)); } + /** + * Returns a node named "worker" whose "slow" item runs until canceled and then counts down {@code + * cancelled}. Its other items fail with {@code error} once the slow item has started. + */ + private static Node failsOnceSlowStarts(Throwable error, CountDownLatch cancelled) { + CountDownLatch slowStarted = new CountDownLatch(1); + return node(item -> + item.equals("slow") + ? Flowable.timer(10, SECONDS) + .doOnSubscribe(unused -> slowStarted.countDown()) + .doOnCancel(cancelled::countDown) + : Flowable.defer(() -> { + awaitRelease(slowStarted); + return Flowable.error(error); + })); + } + private static Node interruptsOn(Object asking) { return node( item ->