Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions apps/ai-studio/src/adapters/execution-stream-adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,13 @@ import type { ExecutionEvent, ExecutionSnapshot } from '@workflow-builder/types/
import { BACKEND_URL } from '../config';
import { applyConnectionLost, applyEvent, applySnapshot } from '../stores/use-execution-store';

const TERMINAL_TYPES = new Set(['execution_completed', 'execution_failed', 'execution_cancelled']);
const TERMINAL_STATUSES = new Set(['completed', 'failed', 'cancelled']);
const TERMINAL_TYPES = new Set([
'execution_completed',
'execution_incomplete',
'execution_failed',
'execution_cancelled',
]);
const TERMINAL_STATUSES = new Set(['completed', 'incomplete', 'failed', 'cancelled']);
const MAX_RETRIES = 5;

export function connectExecutionStream(executionId: string, streamUrl: string): () => void {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ export function AiStudioControls() {
}, [executeFromCanvas]);

const isRunning = status === 'pending' || status === 'running';
const isDone = status === 'completed' || status === 'failed' || status === 'cancelled';
const isDone = status === 'completed' || status === 'incomplete' || status === 'failed' || status === 'cancelled';

return (
<div
Expand Down
3 changes: 3 additions & 0 deletions apps/ai-studio/src/components/execution/highlighting.css
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
html[data-theme='light'] {
--ai-studio-edge-color--active: var(--ax-colors-green-400);
--ai-studio-status-color--completed: var(--ax-colors-green-400);
--ai-studio-status-color--incomplete: var(--ax-colors-orange-400);
}

html[data-theme='dark'] {
--ai-studio-edge-color--active: var(--ax-colors-green-200);
--ai-studio-status-color--completed: var(--ax-colors-green-200);
--ai-studio-status-color--incomplete: var(--ax-colors-orange-300);
}

:root {
--ai-studio-status-color--failed: var(--ax-txt-error-default);
--ai-studio-status-bg--completed: color-mix(in srgb, var(--ai-studio-status-color--completed), transparent 85%);
--ai-studio-status-bg--incomplete: color-mix(in srgb, var(--ai-studio-status-color--incomplete), transparent 85%);
--ai-studio-status-bg--failed: color-mix(in srgb, var(--ai-studio-status-color--failed), transparent 85%);

--ai-studio-node-shadow-color--active: var(--ai-studio-edge-color--active);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@
color: var(--ax-txt-secondary, #888);
}

.badge--execution_incomplete {
color: var(--ai-studio-status-color--incomplete);
}

.badge--node_failed,
.badge--execution_failed {
color: var(--ai-studio-status-color--failed);
Expand Down Expand Up @@ -159,6 +163,11 @@
background: var(--ai-studio-status-bg--completed);
}

.status--incomplete {
color: var(--ai-studio-status-color--incomplete);
background: var(--ai-studio-status-bg--incomplete);
}

.status--failed {
color: var(--ai-studio-status-color--failed);
background: var(--ai-studio-status-bg--failed);
Expand Down
7 changes: 7 additions & 0 deletions apps/ai-studio/src/components/execution/log-panel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,13 @@ function EventRow({ event, selectedNodeId }: { event: ExecutionEvent; selectedNo

break;
}
case 'execution_incomplete': {
detail = event.payload.deadEnds
.map(({ nodeId, port }) => `${nodeId} routed to "${port}" — nothing connected to that handle`)
.join('\n');

break;
}
// No default
}

Expand Down
3 changes: 3 additions & 0 deletions apps/ai-studio/src/stores/use-execution-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ function eventToExecutionStatus(event: ExecutionEvent): ExecutionStatus | undefi
case 'execution_completed': {
return 'completed';
}
case 'execution_incomplete': {
return 'incomplete';
}
case 'execution_failed': {
return 'failed';
}
Expand Down
9 changes: 9 additions & 0 deletions apps/backend/src/events/drain-events.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,15 @@ describe('drainEventsSince', () => {
expect(result.lastSequence).toBe(2);
});

it('flags terminal when the last fetched event is execution_incomplete', async () => {
const fetch = fixedFetcher([makeEvent('exec-1', 1), makeEvent('exec-1', 2, 'execution_incomplete')]);

const result = await drainEventsSince('exec-1', 0, fetch, async () => {});

expect(result.reachedTerminal).toBe(true);
expect(result.lastSequence).toBe(2);
});

it('write failure stops the drain and pins the cursor at the last successful write', async () => {
const fetch = fixedFetcher([makeEvent('exec-1', 1), makeEvent('exec-1', 2), makeEvent('exec-1', 3)]);
let calls = 0;
Expand Down
7 changes: 6 additions & 1 deletion apps/backend/src/events/drain-events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,12 @@ import type { ExecutionEventRow } from './fetch-events-after';
export type EventFetcher = (executionId: string, afterSequence: number) => Promise<ExecutionEventRow[]>;
type EventWriter = (event: ExecutionEventRow) => Promise<void>;

const TERMINAL_EVENT_TYPES = new Set(['execution_completed', 'execution_failed', 'execution_cancelled']);
const TERMINAL_EVENT_TYPES = new Set([
'execution_completed',
'execution_incomplete',
'execution_failed',
'execution_cancelled',
]);

export type DrainResult = {
lastSequence: number;
Expand Down
2 changes: 1 addition & 1 deletion apps/backend/src/routes/executions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import type { TenantVariables } from '../tenant';

const logger = backendLogger.child({ component: 'executions-route' });

const TERMINAL_STATUSES = new Set(['completed', 'failed', 'cancelled']);
const TERMINAL_STATUSES = new Set(['completed', 'incomplete', 'failed', 'cancelled']);

export function createExecutionsRoutes(
assertAuthorized: AssertAuthorized,
Expand Down
2 changes: 1 addition & 1 deletion apps/execution-worker/src/database.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export const database = {
},

async updateExecutionStatus(executionId: string, status: string, errorMessage?: string) {
const isTerminal = ['completed', 'failed', 'cancelled'].includes(status);
const isTerminal = ['completed', 'incomplete', 'failed', 'cancelled'].includes(status);

await sql`
UPDATE executions SET
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ export async function runWorkflow(input: WorkflowExecutionInput<AiStudioNode>):
// tells Temporal to close the run as Failed rather than Completed. It has to be a
// TemporalFailure (anything else fails the workflow *task* and retries forever), and
// non-retryable since replaying a deterministic graph failure would re-run LLM activities.
//
// Only 'failed' throws. An 'incomplete' outcome — a branch that routed to a port with
// nothing wired to it — falls through on purpose: nothing errored, so the Workflow
// Execution closes as Completed and the run's own 'incomplete' status carries the
// detail. Do not add it to this check.
if (outcome.status === 'failed') {
throw ApplicationFailure.nonRetryable(outcome.error.message, outcome.error.code ?? 'WorkflowExecutionFailed');
}
Expand Down
27 changes: 24 additions & 3 deletions packages/execution-core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ const node: MyNode = {
};
```

If a node with `'errorRoute'` policy fails but has no outgoing edge tagged `'errorRoute'`, the run completes cleanly — the failure is recorded as `node_failed` and nothing else fires. That makes `'errorRoute'` usable as a silent DLQ when paired with downstream observability on `node_failed` events.
If a node with `'errorRoute'` policy fails but has no outgoing edge tagged `'errorRoute'`, the failure is recorded as `node_failed` and the run ends **incomplete** — the policy named a port and nothing was wired to it. See [Incomplete runs](#incomplete-runs). For deliberate absorption, use `'continue'` on a node with no downstream edges: it records `node_failed` and ends the branch without claiming a route it does not have.

## Skipped nodes

Expand All @@ -149,6 +149,27 @@ The reason does not depend on the order a node's predecessors happen to resolve

Events are emitted once the whole wave has propagated, after that wave's `node_completed` events and before the next wave's `node_started`. A skipped node emits exactly one `node_skipped` and no `node_started`/`node_completed`, so it stays absent from `nodeOutputs` — downstream joins see only the live predecessors' outputs.

## Incomplete runs

A run is **incomplete** when a node returned an explicit `nextPort` and no outgoing edge went live. Each occurrence is a _dead end_, recorded as `{ nodeId, port }`; the run finishes everything else it can reach, then emits a single terminal `execution_incomplete` event naming every one of them, sets the execution status to `'incomplete'`, and returns `{ status: 'incomplete', deadEnds }`.

It is deliberately **not** a failure. Nothing threw, so the engine closes the run normally — the Temporal adapter returns rather than raising an `ApplicationFailure`, and the Workflow Execution shows as Completed. What changes is the run's own status, so an operator can tell "the graph ran" from "the graph ran everything it was supposed to".

| Shape | Outcome |
| ----------------------------------------------------------- | ------------------------------------------------------------------------- |
| A decision routes to a handle no edge carries | **incomplete** |
| A decision node with no outgoing edges at all | **incomplete** |
| An `'errorRoute'` failure with no `'errorRoute'` edge | **incomplete** |
| A plain leaf (returns no `nextPort`) | completed |
| A decision routes to a wired handle; other branches pruned | completed, others `node_skipped` |
| A successful node whose only outgoing edges are error edges | completed — `nextPort` is undefined, so the success path is simply a leaf |
| A `'continue'` failure on a leaf | completed |
| Nodes that never became ready (a cycle) | **failed** — see below |

Failure always wins. An unhandled node failure returns before the terminal check, and the stall check runs ahead of it, so a run reports incomplete only where it would otherwise have reported completed.

`Workflow stalled: nodes never became ready` stays a **failure** and keeps its own name. A stall is the scheduler genuinely unable to proceed — a cycle, or a dangling-edge bug — where an incomplete run has finished and simply did not reach everything. Two different conditions, two different words.

## Template references

`resolveTemplate(template, context)` (in `src/templates/`) interpolates `{{namespace.path}}` references against the live `ExecutionContext`. Three forms are supported - **strict by default**, with two opt-in modifiers for missing values:
Expand Down Expand Up @@ -184,7 +205,7 @@ In practice this means:
- **Positional `Promise.all`.** Concurrent waves use `Promise.all`, which resolves with results in input order regardless of completion order. The runner reads positionally and never branches on which promise finished first; `Promise.race` and `Promise.any` are not used.
- **No top-level side effects.** `graph-runner.ts` only exports function declarations. Nothing reads the environment or instantiates dated objects at import time.

A regression test (`graph-runner.replay-determinism.test.ts`) runs each canonical topology (linear, fan-out, diamond, decision, failure, stall) ten times against an identical deterministic port mock and asserts the resulting sequence of `EventEmitterPort` calls, statuses, and activity invocations is byte-equivalent across runs.
A regression test (`graph-runner.replay-determinism.test.ts`) runs each canonical topology (linear, fan-out, diamond, decision, skip, dead end, failure, stall) ten times against an identical deterministic port mock and asserts the resulting sequence of `EventEmitterPort` calls, statuses, and activity invocations is byte-equivalent across runs.

A full audit — every potential source of non-determinism enumerated with a verdict, plus maintenance rules for future contributors — lives in [`replay-audit.md`](./replay-audit.md). Read it before adding code that runs inside `runGraph`.

Expand All @@ -206,7 +227,7 @@ export interface LoggerPort {

### Where logger lives

`LoggerPort` is **not** passed into `runGraph`, and `runGraph` does **not** import it. The runner is re-exported from the sandbox-safe entry (`@workflow-builder/execution-core/workflow`) and runs inside Temporal's V8 workflow context, where every call to `new Date()` poisons history replay. Lifecycle signals (`execution_started/completed/failed`, `node_started/completed/failed`, `node_skipped`) already flow through `EventEmitterPort` — operators tail those for run-time observability of a workflow.
`LoggerPort` is **not** passed into `runGraph`, and `runGraph` does **not** import it. The runner is re-exported from the sandbox-safe entry (`@workflow-builder/execution-core/workflow`) and runs inside Temporal's V8 workflow context, where every call to `new Date()` poisons history replay. Lifecycle signals (`execution_started/completed/incomplete/failed`, `node_started/completed/failed`, `node_skipped`) already flow through `EventEmitterPort` — operators tail those for run-time observability of a workflow.

Use `LoggerPort` outside the sandbox — in HTTP routes, in activity executors (LLM calls, HTTP retries), at app startup.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,35 @@ describe('runGraph — replay determinism (re-execution equivalence)', () => {
]);
});

it('dead ends — the terminal incomplete payload is identical across runs', async () => {
// `deadEnds` accumulates across waves in propagation order, then ships as one payload.
// An accidental Set, or emitting per-wave instead of once at the end, would re-order
// the list without changing any other recorded call.
const input = makeInput(
[start('S'), trigger('D1'), trigger('D2'), trigger('Live')],
[edge('e1', 'S', 'D1'), edge('e2', 'S', 'D2'), edge('e3', 'S', 'Live')],
);

const records = await runNTimes(
input,
{ D1: { output: 'd1', nextPort: 'gone-1' }, D2: { output: 'd2', nextPort: 'gone-2' } },
RUNS,
);
expectAllRunsIdentical(records);

expect(records[0]!.events.at(-1)).toEqual({
type: 'execution_incomplete',
nodeId: undefined,
payload: {
deadEnds: [
{ nodeId: 'D1', port: 'gone-1' },
{ nodeId: 'D2', port: 'gone-2' },
],
},
});
expect(records[0]!.statuses.at(-1)?.status).toBe('incomplete');
});

it('node failure — failure path is deterministic too (same error code, same event sequence)', async () => {
// The catch branch builds errorPayload from the thrown error. Across
// replays the activity returns the same error (cached in history), so
Expand Down
Loading