diff --git a/packages/workflow-executor/src/errors.ts b/packages/workflow-executor/src/errors.ts index 530c97405c..10173ea78a 100644 --- a/packages/workflow-executor/src/errors.ts +++ b/packages/workflow-executor/src/errors.ts @@ -270,6 +270,17 @@ export class AiInvokeTimeoutError extends WorkflowExecutorError { } } +// Wraps any AI-call failure (timeout, outage, malformed/missing tool call, out-of-range pick) so +// handlers degrade to the Manual path. Caught internally — never HTTP-mapped; not exported from index. +export class AiAssistUnavailableError extends WorkflowExecutorError { + readonly reason: unknown; + + constructor(reason: unknown) { + super('AI assistance unavailable'); + this.reason = reason; + } +} + export class NoMcpToolsError extends WorkflowExecutorError { constructor(requestedMcpServerId: string) { super( diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 170f91b347..2f8483bcdd 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -16,12 +16,14 @@ import { DynamicStructuredTool, HumanMessage, SystemMessage } from '@forestadmin import { z } from 'zod'; import { + AiAssistUnavailableError, InvalidAIResponseError, InvalidPreRecordedArgsError, NoRelationshipFieldsError, RelatedRecordNotFoundError, RelationNotFoundError, StepStateError, + extractErrorMessage, } from '../errors'; import RecordStepExecutor from './record-step-executor'; import { StepExecutionMode } from '../types/validated/step-definition'; @@ -42,6 +44,10 @@ Choose the fields that are most useful for determining which record best matches const SELECT_RECORD_SYSTEM_PROMPT = `You are an AI agent selecting the most relevant related record from a list of candidates. Choose the record that best matches the user request based on the provided field values.`; +// Only appended when -1 is a valid answer. The base prompt tells the AI to always pick, so without +// this it forces a weak match on an impossible request instead of declining. +const SELECT_RECORD_NONE_ALLOWED_PROMPT = `If the request states a specific requirement that NO candidate satisfies, return -1 instead of forcing an arbitrary or loosely-related match. Do NOT return -1 merely because a match is imperfect — only when no candidate genuinely fits the request.`; + // Bound only what is sent to the AI in selectBestRecordIndex — the full candidate list is // always returned to the front via availableRecordIds. These cap the prompt size, not the data. const MAX_RELEVANT_FIELDS = 6; @@ -103,22 +109,40 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - if (!execution.pendingData) { + if (!execution.pendingData || !execution.selectedRecordRef) { throw new StepStateError(`Step at index ${this.context.stepIndex} has no pending data`); } + const useAi = this.context.stepDefinition.executionType !== StepExecutionMode.Manual; const schema = await this.getCollectionSchema(execution.selectedRecordRef.collectionName); const target = await this.buildTarget(schema, fieldName, execution.selectedRecordRef); - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target); + + // AI failure while re-listing → degrade to the no-AI list, not a run error (see AiAssistUnavailableError). + let aiSuggested = useAi; + let candidates; + + try { + candidates = await this.collectCandidateIds(target, useAi); + } catch (error) { + if (!(useAi && error instanceof AiAssistUnavailableError)) throw error; + this.logAiDegrade(error.reason); + aiSuggested = false; + candidates = await this.collectCandidateIds(target, false); + } + + const { availableRecordIds, suggestedRecord } = candidates; await this.context.runStore.saveStepExecution(this.context.runId, { ...execution, userConfirmation: undefined, + // Rebuild pendingData for the new relation from scratch (retain only the immutable field list) + // so no stale suggestion state — suggestedRecord or suggestNoRecord — survives the field switch. pendingData: { - ...execution.pendingData, + availableFields: execution.pendingData.availableFields, suggestedField: { name: target.name, displayName: target.displayName }, availableRecordIds, suggestedRecord, + ...(aiSuggested && !suggestedRecord && { suggestNoRecord: true }), }, }); @@ -127,23 +151,49 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { stepDefinition: step } = this.context; - const target = await this.resolveTarget(); + const useAi = step.executionType !== StepExecutionMode.Manual; + + try { + if (step.executionType === StepExecutionMode.FullyAutomated) { + return await this.resolveAndLoadAutomatic(); + } - // Branch B -- fully automated execution - if (step.executionType === StepExecutionMode.FullyAutomated) { - return this.resolveAndLoadAutomatic(target); + const target = await this.resolveTarget(useAi); + const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); + + return await this.saveAndAwaitInput(target, sourceSchema, useAi); + } catch (error) { + // AI failure in any AI mode → degrade to Manual (see AiAssistUnavailableError). + if (useAi && error instanceof AiAssistUnavailableError) { + this.logAiDegrade(error.reason); + + return this.degradeToManualAwaitInput(); + } + + throw error; } + } - // Branch C -- pre-fetch candidates, await user confirmation + private logAiDegrade(reason: unknown): void { + this.context.logger( + 'Warn', + 'load-related-record: AI unavailable, degrading to manual selection', + { ...this.logCtx, error: extractErrorMessage(reason) }, + ); + } + + // The re-run is AI-free (first eligible relation, no-AI list), so it can't re-trigger the failure. + private async degradeToManualAwaitInput(): Promise { + const target = await this.resolveTarget(false); const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); - return this.saveAndAwaitInput(target, sourceSchema); + return this.saveAndAwaitInput(target, sourceSchema, false); } // Picks the (record, relation) pair to follow. Unlike a separate record-then-relation choice, // this lets the AI decide by what each relation LEADS TO — so "load the dvd" follows // store→dvds rather than latching onto a previously-loaded dvd whose collection just matches. - private async resolveTarget(): Promise { + private async resolveTarget(useAi: boolean): Promise { const { preRecordedArgs } = this.context.stepDefinition; const sourceRecords = @@ -168,8 +218,10 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor 1 && useAi ? await this.selectRelationToFollow(eligible) : eligible[0]; return this.targetFromCandidate(chosen); } @@ -265,15 +317,36 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - const { selectedRecordRef, name, displayName } = target; + const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds( + target, + suggestViaAi, + ); - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target); + return this.persistAwaitInput(target, sourceSchema, { + availableRecordIds, + suggestedRecord, + // An AI pass that yields no record is a deliberate "nothing relevant" → pre-check "No X to load". + // A Manual pass with no suggestion just means the user picks, so no pre-check. + suggestNoRecord: suggestViaAi && !suggestedRecord, + }); + } + + private async persistAwaitInput( + target: RelationTarget, + sourceSchema: CollectionSchema, + pending: { + availableRecordIds: LoadRelatedRecordCandidate[]; + suggestedRecord?: LoadRelatedRecordCandidate; + suggestNoRecord: boolean; + }, + ): Promise { + const { selectedRecordRef, name, displayName } = target; const availableFields: RelationRef[] = sourceSchema.fields .filter(isFollowableRelation) @@ -285,8 +358,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { const candidate = await this.fetchXToOneCandidate(target); return candidate - ? { availableRecordIds: [candidate], suggestedRecord: candidate } - : { availableRecordIds: [] }; + ? { availableRecordIds: [candidate], suggestedRecord: candidate, ambiguous: false } + : { availableRecordIds: [], ambiguous: false }; } - const { relatedData, bestIndex, relatedSchema } = await this.selectBestFromRelatedData( - target, - 50, - ); + const { relatedData, bestIndex, confident, relatedSchema } = + await this.selectBestFromRelatedData( + target, + 50, + // allowNone: the AI may judge no candidate relevant (→ "No X to load"); only meaningful when + // ranking (Manual passes rank=false and never calls the AI). + suggestViaAi ? { rank: true, allowNone: true } : { rank: false }, + ); if (relatedData.length === 0) { - return { availableRecordIds: [] }; + return { availableRecordIds: [], ambiguous: false }; } const referenceField = relatedSchema.referenceField ?? null; @@ -325,7 +409,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor= 0 ? toCandidate(relatedData[bestIndex]) : undefined, + // -1 (none relevant) is not ambiguity; only a low-confidence positive pick is. + ambiguous: bestIndex >= 0 && !confident, }; } @@ -338,37 +424,33 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - const record = await this.fetchRecordForRelation(target); - - return this.persistAndReturn(record, target, undefined); - } - - private async fetchRecordForRelation(target: RelationTarget): Promise { - if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { - return this.fetchXToOneRecordRef(target); - } - - if (target.relationType === 'HasMany') { - return this.selectBestRelatedRecord(target); - } - - return this.fetchFirstCandidate(target); - } + private async resolveAndLoadAutomatic(): Promise { + // No source record throws (like Manual/AI-assisted) → the front offers "continue without". + const target = await this.resolveTarget(true); + const { availableRecordIds, suggestedRecord, ambiguous } = await this.collectCandidateIds( + target, + true, + ); - private async fetchXToOneRecordRef(target: RelationTarget): Promise { - const candidate = await this.fetchXToOneCandidate(target); + // Full AI auto-loads only a confident single pick; otherwise it degrades to an AI-assisted + // confirmation — "No X to load" pre-checked when nothing fits, else its best guess pre-selected. + if (!suggestedRecord || ambiguous) { + const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); - if (!candidate) { - throw new RelatedRecordNotFoundError(target.selectedRecordRef.collectionName, target.name); + return this.persistAwaitInput(target, sourceSchema, { + availableRecordIds, + suggestedRecord, + suggestNoRecord: !suggestedRecord, + }); } - return { + const record: RecordRef = { collectionName: target.relatedCollectionName, - recordId: candidate.recordId, + recordId: suggestedRecord.recordId, stepIndex: this.context.stepIndex, }; + + return this.persistAndReturn(record, target, undefined); } private async fetchXToOneCandidate( @@ -401,7 +483,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { selectedRecordRef, pendingData, userConfirmation } = execution; - if (!pendingData) { + if (!pendingData || !selectedRecordRef) { throw new StepStateError(`Step at index ${this.context.stepIndex} has no pending data`); } @@ -448,65 +530,66 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor, limit: number, + // allowNone is meaningful only when ranking, so it's absent from the rank:false variant. + opts: { rank: true; allowNone: boolean } | { rank: false } = { rank: true, allowNone: false }, ): Promise<{ relatedData: RecordData[]; bestIndex: number; + // Whether the suggested record is a confident pick. Deterministic/short-circuit paths (single + // candidate, no ranking) are confident by construction; the ranked path reflects the AI's flag. + confident: boolean; suggestedFields: string[]; relatedSchema: CollectionSchema; }> { const relatedSchema = await this.getCollectionSchema(target.relatedCollectionName); const relatedData = await this.fetchRelatedData(target, relatedSchema, limit); - // Empty (bestIndex unused — callers guard on length) or single → no ranking needed. - if (relatedData.length <= 1) { - return { relatedData, bestIndex: 0, suggestedFields: [], relatedSchema }; + // Empty lists have nothing to rank or pre-select. + if (relatedData.length === 0) { + return { relatedData, bestIndex: -1, confident: true, suggestedFields: [], relatedSchema }; } - // The final record stays AI-suggested + user-confirmed — only the source + relation are - // pinned deterministically. Index-based record pinning was removed (not revise-safe). - const suggestedFields = await this.selectRelevantFields( - relatedSchema, - this.context.stepDefinition.prompt, - ); - const bestIndex = await this.selectBestRecordIndex( - relatedData, - suggestedFields, - this.context.stepDefinition.prompt, - ); - - return { relatedData, bestIndex, suggestedFields, relatedSchema }; - } - - /** HasMany + fully automated execution: fetch top 50, then AI calls to select the best record. */ - private async selectBestRelatedRecord(target: RelationTarget): Promise { - const { relatedData, bestIndex } = await this.selectBestFromRelatedData(target, 50); - - if (relatedData.length === 0) { - throw new RelatedRecordNotFoundError(target.selectedRecordRef.collectionName, target.name); + // Manual: no AI. A sole candidate is pre-selected (the only possible choice, as for an xToOne + // relation); with several, the user picks from the list. + if (!opts.rank) { + return { + relatedData, + bestIndex: relatedData.length === 1 ? 0 : -1, + confident: true, + suggestedFields: [], + relatedSchema, + }; } - return this.toRecordRef(relatedData[bestIndex]); - } + // Ranking (AI-assisted / Full AI): even a lone candidate goes through the AI so Full AI can + // decline it (-1 → "No X to load") rather than auto-loading a possibly-irrelevant sole record. - private async fetchFirstCandidate(target: RelationTarget): Promise { - const candidates = await this.fetchCandidates(target, 1); + // The final record stays AI-suggested + user-confirmed (or AI-decided in Full AI): only the + // source and relation are pinned deterministically, not the record index (not revise-safe). + const suggestedFields = await this.withAiAssist(() => + this.selectRelevantFields(relatedSchema, this.context.stepDefinition.prompt), + ); + const { index: bestIndex, confident } = await this.withAiAssist(() => + this.selectBestRecordIndex( + relatedData, + suggestedFields, + this.context.stepDefinition.prompt, + opts.allowNone, + ), + ); - return candidates[0]; + return { relatedData, bestIndex, confident, suggestedFields, relatedSchema }; } - private async fetchCandidates( - target: Pick, - limit: number, - ): Promise { - const { selectedRecordRef, name } = target; - const relatedSchema = await this.getCollectionSchema(target.relatedCollectionName); - const relatedData = await this.fetchRelatedData(target, relatedSchema, limit); - - if (relatedData.length === 0) { - throw new RelatedRecordNotFoundError(selectedRecordRef.collectionName, name); + // Tags failures raised in an AI call as AiAssistUnavailableError; passing an already-tagged error + // through avoids double-wrapping when calls nest. + private async withAiAssist(call: () => Promise): Promise { + try { + return await call(); + } catch (error) { + if (error instanceof AiAssistUnavailableError) throw error; + throw new AiAssistUnavailableError(error); } - - return relatedData.map(r => this.toRecordRef(r)); } private async fetchRelatedData( @@ -577,20 +660,24 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor( - messages, - tool, - ); + // Wrap only the AI call + response validation: buildPreviousStepsMessages above reads the run + // store, and a store failure must surface, not be mistagged as an AI failure and degraded. + return this.withAiAssist(async () => { + const { relation } = await this.invokeWithTool<{ relation: string; reasoning: string }>( + messages, + tool, + ); - const index = labels.indexOf(relation); + const index = labels.indexOf(relation); - if (index === -1) { - throw new InvalidAIResponseError( - `AI selected relation "${relation}" which does not match any available option`, - ); - } + if (index === -1) { + throw new InvalidAIResponseError( + `AI selected relation "${relation}" which does not match any available option`, + ); + } - return candidates[index]; + return candidates[index]; + }); } /** AI call 1 for HasMany: selects the most relevant fields to compare candidates. */ @@ -650,12 +737,12 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor nonRelationFields.find(f => f.displayName === dn)?.fieldName ?? dn); } - /** AI call 2 for HasMany: selects the best record by index from the candidate list. */ private async selectBestRecordIndex( candidates: RecordData[], fieldNames: string[], prompt: string | undefined, - ): Promise { + allowNone = false, + ): Promise<{ index: number; confident: boolean }> { const filteredCandidates = candidates.map((c, i) => { const entries = Object.entries(c.values).filter( ([k]) => fieldNames.length === 0 || fieldNames.includes(k), @@ -681,7 +768,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor( - messages, - tool, - ); + const { recordIndex, confident: rawConfident } = await this.invokeWithTool<{ + recordIndex: number; + confident?: boolean; + reasoning: string; + }>(messages, tool); - // NOTE: The Zod schema's .min(0).max(maxIndex) shapes the tool prompt only — it is NOT - // validated against the AI response. This guard is the sole runtime enforcement. - if (!Number.isInteger(recordIndex) || recordIndex < 0 || recordIndex > maxIndex) { + // The Zod .min().max() shapes the tool prompt only — NOT validated against the AI response; this + // guard is the sole runtime enforcement. -1 is accepted only when noneAllowed (allowNone && !truncated). + if (!Number.isInteger(recordIndex) || recordIndex < minIndex || recordIndex > maxIndex) { throw new InvalidAIResponseError( - `AI selected record index ${recordIndex} which is out of range (0-${maxIndex}) or not an integer`, + `AI selected record index ${recordIndex} which is out of range (${minIndex}-${maxIndex}) or not an integer`, ); } - return recordIndex; - } + // A definitive -1 ("none relevant") is not ambiguity. For a positive pick, honour the AI's + // confidence flag, defaulting to confident when omitted so a missing flag still auto-loads. + const confident = recordIndex < 0 ? true : rawConfident !== false; - private toRecordRef(data: RecordData): RecordRef { - return { - collectionName: data.collectionName, - recordId: data.recordId, - stepIndex: this.context.stepIndex, - }; + return { index: recordIndex, confident }; } } diff --git a/packages/workflow-executor/src/http/step-serializer.ts b/packages/workflow-executor/src/http/step-serializer.ts index 7a8a7de619..908382643c 100644 --- a/packages/workflow-executor/src/http/step-serializer.ts +++ b/packages/workflow-executor/src/http/step-serializer.ts @@ -15,9 +15,12 @@ export default function serializeStepForWire(step: StepExecutionData): unknown { return { ...step, selectedRecordRef: serializeRecordRef(step.selectedRecordRef) }; case 'load-related-record': { + // Omit selectedRecordRef when absent — serializing undefined throws. const result: Record = { ...step, - selectedRecordRef: serializeRecordRef(step.selectedRecordRef), + ...(step.selectedRecordRef && { + selectedRecordRef: serializeRecordRef(step.selectedRecordRef), + }), }; if (step.pendingData) { diff --git a/packages/workflow-executor/src/types/step-execution-data.ts b/packages/workflow-executor/src/types/step-execution-data.ts index 0585c237d6..537a0ae9a3 100644 --- a/packages/workflow-executor/src/types/step-execution-data.ts +++ b/packages/workflow-executor/src/types/step-execution-data.ts @@ -176,6 +176,9 @@ export interface LoadRelatedRecordPendingData { availableRecordIds: LoadRelatedRecordCandidate[]; // Absent when the relation has no linked record(s): the list is empty and there's nothing to suggest. suggestedRecord?: LoadRelatedRecordCandidate; + // The AI actively judged no candidate relevant (incl. Full AI degrading to confirmation) → the front + // pre-checks "No X to load". Distinct from a plain absent suggestedRecord (Manual: the user picks). + suggestNoRecord?: boolean; } export interface LoadRelatedRecordStepExecutionData @@ -183,7 +186,8 @@ export interface LoadRelatedRecordStepExecutionData WithUserConfirmation { type: 'load-related-record'; pendingData?: LoadRelatedRecordPendingData; - selectedRecordRef: RecordRef; + // Set on every await/load path (and preserved through the user-initiated "continue without" skip). + selectedRecordRef?: RecordRef; executionParams?: RelationRef; executionResult?: { relation: RelationRef; record: RecordRef } | { skipped: true }; } diff --git a/packages/workflow-executor/src/types/validated/step-definition.ts b/packages/workflow-executor/src/types/validated/step-definition.ts index 6ac1a42c46..a2c6cda898 100644 --- a/packages/workflow-executor/src/types/validated/step-definition.ts +++ b/packages/workflow-executor/src/types/validated/step-definition.ts @@ -98,10 +98,10 @@ export type TriggerActionStepDefinition = z.infer { expect(bindTools.mock.calls[0][0][0].name).toBe('select-record-by-content'); }); - it('takes the single candidate directly without AI record-selection calls', async () => { + it('consults the AI for a lone candidate and loads it when the AI picks index 0', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -761,15 +761,37 @@ describe('LoadRelatedRecordStepExecutor', () => { const agentPort = makeMockAgentPort([ { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, ]); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); - const invoke = jest.fn(); + // A lone candidate in an AI mode still runs the AI so it can be declined (-1); here the AI + // picks index 0. Call 1: select-fields → ['City']; Call 2: select-record-by-content → 0. + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 0, reasoning: 'Paris is relevant' }, + id: 'c3', + }, + ], + }); const bindTools = jest.fn().mockReturnValue({ invoke }); const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); const context = makeContext({ model, agentPort, - workflowPort: makeMockWorkflowPort({ customers: hasManySchema }), + runStore, + workflowPort: makeMockWorkflowPort({ customers: hasManySchema, addresses: addressSchema }), stepDefinition: makeStep({ executionType: StepExecutionMode.FullyAutomated }), }); const executor = new LoadRelatedRecordStepExecutor(context); @@ -777,12 +799,19 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.status).toBe('success'); - // Single relation (auto-picked, no select-relation-to-follow) AND single related - // candidate (no field/record AI calls) → no AI calls at all. - expect(bindTools).not.toHaveBeenCalled(); + expect(bindTools).toHaveBeenCalledTimes(2); + expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + executionResult: expect.objectContaining({ + record: expect.objectContaining({ collectionName: 'addresses', recordId: [1] }), + }), + }), + ); }); - it('returns error outcome when AI selects an out-of-range record index', async () => { + it('degrades to manual selection when the AI returns an out-of-range record index', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -835,14 +864,15 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI made an unexpected choice. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + // Out-of-range index = AI failure → degrade to Manual (see AiAssistUnavailableError), not an error/auto-load. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); }); - it('returns error when AI returns empty fieldNames violating the min:1 constraint', async () => { + it('degrades to manual selection when the AI returns empty fieldNames (min:1 violation)', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -884,11 +914,12 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI made an unexpected choice. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + // Invalid field-selection response = AI failure → degrade to Manual (see AiAssistUnavailableError). + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); }); }); @@ -960,14 +991,39 @@ describe('LoadRelatedRecordStepExecutor', () => { }, ], }); - const agentPort = makeMockAgentPort([{ collectionName: 'tags', recordId: [7], values: {} }]); - const mockModel = makeMockModel({ relationName: 'Tags', reasoning: 'Load tags' }); + const agentPort = makeMockAgentPort([ + { collectionName: 'tags', recordId: [7], values: { name: 'urgent' } }, + ]); + const tagsSchema = makeCollectionSchema({ + collectionName: 'tags', + collectionDisplayName: 'Tags', + fields: [{ fieldName: 'name', displayName: 'Name', isRelationship: false }], + }); + // A lone toMany candidate is still ranked by the AI (so Full AI can decline it); here it picks + // index 0. Call 1: select-fields → ['Name']; Call 2: select-record-by-content → 0. + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['Name'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 0, reasoning: 'The only tag is relevant' }, + id: 'c3', + }, + ], + }); + const model = { + bindTools: jest.fn().mockReturnValue({ invoke }), + } as unknown as ExecutionContext['model']; const runStore = makeMockRunStore(); const context = makeContext({ - model: mockModel.model, + model, agentPort, runStore, - workflowPort: makeMockWorkflowPort({ customers: belongsToManySchema }), + workflowPort: makeMockWorkflowPort({ customers: belongsToManySchema, tags: tagsSchema }), stepDefinition: makeStep({ executionType: StepExecutionMode.FullyAutomated }), }); const executor = new LoadRelatedRecordStepExecutor(context); @@ -975,13 +1031,13 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.status).toBe('success'); - // To-many path: /relationships call with limit: 1, no parent-record projection. + // To-many path uses the candidate-list limit (50) and ranks even a lone candidate via the AI. expect(agentPort.getRelatedData).toHaveBeenCalledWith( expect.objectContaining({ collection: 'customers', id: [42], relation: 'tags', - limit: 1, + limit: 50, relatedSchema: expect.objectContaining({ collectionName: 'tags' }), }), expect.objectContaining({ id: 1 }), @@ -997,39 +1053,646 @@ describe('LoadRelatedRecordStepExecutor', () => { ); }); - // fetchCandidates throws RelatedRecordNotFoundError when the agent returns an - // empty list. Same user-facing message as the other empty-result paths. - it('returns error when getRelatedData returns an empty array', async () => { - const belongsToManySchema = makeCollectionSchema({ - fields: [ - { - fieldName: 'tags', - displayName: 'Tags', - isRelationship: true, - relationType: 'BelongsToMany', - relatedCollectionName: 'tags', - }, - ], - }); - const agentPort = makeMockAgentPort([]); - const mockModel = makeMockModel({ relationName: 'Tags', reasoning: 'Load tags' }); + // Empty list: Full AI hands off with "No X to load" pre-checked rather than auto-skipping. + it('falls back to awaiting-input (No X to load) when getRelatedData returns an empty array (Branch B)', async () => { + const belongsToManySchema = makeCollectionSchema({ + fields: [ + { + fieldName: 'tags', + displayName: 'Tags', + isRelationship: true, + relationType: 'BelongsToMany', + relatedCollectionName: 'tags', + }, + ], + }); + const agentPort = makeMockAgentPort([]); + const mockModel = makeMockModel({ relationName: 'Tags', reasoning: 'Load tags' }); + const runStore = makeMockRunStore(); + const context = makeContext({ + model: mockModel.model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ customers: belongsToManySchema }), + stepDefinition: makeStep({ executionType: StepExecutionMode.FullyAutomated }), + }); + const executor = new LoadRelatedRecordStepExecutor(context); + + const result = await executor.execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); + }); + }); + + describe('executionType=Manual: no AI pre-selection', () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + + it('lists the narrowed candidates with no suggestion and never invokes the AI model', async () => { + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.Manual, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + expect(bindTools).not.toHaveBeenCalled(); + + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls[0][1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + }); + + it('pre-fills a sole toMany candidate in Manual (the only option, not an AI pick)', async () => { + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + ]); + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.Manual, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + expect(bindTools).not.toHaveBeenCalled(); + + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls[0][1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1])]); + expect(saved.pendingData.suggestedRecord).toEqual(cand([1])); + }); + + it('still pre-fills the single xToOne record in Manual (the only option, not an AI pick)', async () => { + const agentPort = makeMockAgentPort(); // default: 1 related order #99 via getSingleRelatedData + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + stepDefinition: makeStep({ + executionType: StepExecutionMode.Manual, + preRecordedArgs: { relationName: 'order' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + expect(bindTools).not.toHaveBeenCalled(); + + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls[0][1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand(['99'])]); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); + }); + + it('switching relation (refreshCandidatesForField) in Manual makes no AI suggestion', async () => { + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'order', displayName: 'Order' }, + availableRecordIds: [cand(['99'])], + suggestedRecord: cand(['99']), + }, + }); + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]); + const addressesSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: addressesSchema, + }), + incomingPendingData: { fieldName: 'address' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.Manual }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + expect(bindTools).not.toHaveBeenCalled(); + + const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(finalSave.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(finalSave.pendingData.suggestedRecord).toBeUndefined(); + }); + }); + + describe('executionType=AutomatedWithConfirmation: AI may decline', () => { + it('pre-checks "No X to load" (suggestNoRecord) when the AI judges no candidate relevant', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: -1, reasoning: 'None of the candidates is relevant' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.AutomatedWithConfirmation, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + // AI-assisted lets the AI decline (-1): the user reviews with "No X to load" pre-checked rather + // than a misleading forced suggestion. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); + }); + + it('tells the AI it may return -1 in the record-selection prompt (Bug 2)', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 0, reasoning: 'match' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const context = makeContext({ + model, + agentPort, + runStore: makeMockRunStore(), + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.AutomatedWithConfirmation, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + await new LoadRelatedRecordStepExecutor(context).execute(); + + // 2nd AI call = record selection. Without the decline guidance the base prompt forces a pick, so + // an impossible request returns a weak match instead of -1. + const recordSelectionMessages = invoke.mock.calls[1][0] as Array<{ content: unknown }>; + const content = recordSelectionMessages.map(m => String(m.content)).join('\n'); + expect(content).toContain('-1'); + }); + }); + + describe('executionType=FullyAutomated: AI judges none relevant → confirmation', () => { + it('falls back to awaiting-input (No X to load) when select-record-by-content returns -1', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: -1, reasoning: 'None of the candidates is relevant' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + // Field + record-selection AI calls both run; the record call returns -1 → hand off with "No X + // to load" pre-checked. + expect(bindTools).toHaveBeenCalledTimes(2); + expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); + }); + + it('consults the AI for a lone candidate and hands off (No X to load) when it returns -1', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + ]); + // A lone candidate must not auto-load in Full AI: it still goes through the AI, which returns + // -1 to decline. Call 1: select-fields → ['City']; Call 2: select-record-by-content → -1. + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: -1, reasoning: 'The only candidate does not fit the request' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); + expect(saved.executionResult).toBeUndefined(); + }); + }); + + describe('executionType=FullyAutomated: low-confidence match → confirmation', () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + + it('degrades to awaiting-input with the best guess pre-selected when the AI is not confident', async () => { + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { + recordIndex: 0, + confident: false, + reasoning: 'Paris and Lyon both plausible', + }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + // Full AI can't confidently single one out → hand off with the AI's best guess (index 0 → + // recordId [1]) pre-selected, not an auto-load. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toEqual(cand([1])); + // A best guess, not "none relevant" — the "No X to load" box must stay unchecked. + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); + expect(saved.executionResult).toBeUndefined(); + }); + + it('auto-loads (unchanged) when the AI is confident about its pick', async () => { + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 1, confident: true, reasoning: 'Lyon clearly matches' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('success'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.executionResult).toEqual( + expect.objectContaining({ + record: expect.objectContaining({ collectionName: 'addresses', recordId: [2] }), + }), + ); + }); + }); + + describe('AI failure/timeout degrades to Manual (does not error the run)', () => { + it('Full AI: an AI invoke that throws degrades to awaiting-input with the candidate list', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest.fn().mockRejectedValue(new Error('AI provider timed out')); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + // Deterministic list presented for the user to pick — no error, no auto-skip, no suggestion. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); + expect(saved.executionResult).toBeUndefined(); + }); + + it('AI-assisted: an AI invoke that throws degrades to awaiting-input (no error)', async () => { + const agentPort = makeMockAgentPort(); + const invoke = jest.fn().mockRejectedValue(new Error('AI provider unavailable')); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; const runStore = makeMockRunStore(); + // Default 2-relation schema → relation selection calls the AI (which throws) → degrade. const context = makeContext({ - model: mockModel.model, + model, agentPort, runStore, - workflowPort: makeMockWorkflowPort({ customers: belongsToManySchema }), - stepDefinition: makeStep({ executionType: StepExecutionMode.FullyAutomated }), + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), }); - const executor = new LoadRelatedRecordStepExecutor(context); - const result = await executor.execute(); + const result = await new LoadRelatedRecordStepExecutor(context).execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - 'The related record could not be found. It may have been deleted.', - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + // Degrade picks the first eligible relation (Order) and pre-fills its single linked record. + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); }); @@ -1887,6 +2550,145 @@ describe('LoadRelatedRecordStepExecutor', () => { ); }); + it('clears a stale suggestNoRecord flag when switching to a relation that yields a suggestion', async () => { + // Previous field ended in "No X to load" (suggestNoRecord true, no suggestion). Switching to a + // relation that resolves a record must drop the stale flag — pendingData is rebuilt, not spread. + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'address', displayName: 'Address' }, + availableRecordIds: [cand([1]), cand([2])], + suggestedRecord: undefined, + suggestNoRecord: true, + }, + }); + const agentPort = makeMockAgentPort(); // default: order #99 via getSingleRelatedData + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const mockModel = makeMockModel({}); + const context = makeContext({ + model: mockModel.model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ customers: makeCollectionSchema() }), + incomingPendingData: { fieldName: 'order' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(finalSave.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(finalSave.pendingData.suggestedRecord).toEqual(cand(['99'])); + expect(finalSave.pendingData.suggestNoRecord).toBeUndefined(); + }); + + it('degrades to the no-AI candidate list when the AI fails during a field switch', async () => { + // HasMany field switch; the ranking AI call throws → falls back to the deterministic list, no + // suggestion (degrade, not error). + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'order', displayName: 'Order' }, + availableRecordIds: [cand(['99'])], + suggestedRecord: cand(['99']), + }, + }); + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]); + const addressesSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const invoke = jest.fn().mockRejectedValue(new Error('AI provider timed out')); + const model = { + bindTools: jest.fn().mockReturnValue({ invoke }), + } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: addressesSchema, + }), + incomingPendingData: { fieldName: 'address' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(finalSave.pendingData.suggestedField).toEqual({ + name: 'address', + displayName: 'Address', + }); + expect(finalSave.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(finalSave.pendingData.suggestedRecord).toBeUndefined(); + expect(finalSave.pendingData.suggestNoRecord).toBeUndefined(); + }); + + it('rethrows a non-AI failure during a field switch instead of degrading', async () => { + // A data-fetch failure is not a tagged AI failure, so it surfaces as a step error instead of + // degrading. + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'order', displayName: 'Order' }, + availableRecordIds: [cand(['99'])], + suggestedRecord: cand(['99']), + }, + }); + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + ]); + (agentPort.getRelatedData as jest.Mock).mockRejectedValue( + new AgentPortError('getRelatedData', new Error('db down')), + ); + const addressesSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const mockModel = makeMockModel({}); + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const context = makeContext({ + model: mockModel.model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: addressesSchema, + }), + incomingPendingData: { fieldName: 'address' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('error'); + }); + it('reruns xToOne candidate lookup when previewing a BelongsTo relation', async () => { // Same setup but switching to Order (BelongsTo). Verifies the xToOne path is // used inside refreshCandidatesForField — no AI calls, single candidate from @@ -2347,6 +3149,36 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(saved.pendingData.availableRecordIds).toHaveLength(50); }); + // Safety core (noneAllowed = allowNone && !truncated): a truncated list must NOT offer the AI the + // -1 "none relevant" option, or it could decline while a match hides in the unseen tail. + it('withholds the -1 (none) option from the AI when the candidate list is truncated', async () => { + const big = 'y'.repeat(80); + const values = Object.fromEntries(Array.from({ length: 6 }, (_, i) => [`f${i}`, big])); + const relatedData: RecordData[] = Array.from({ length: 50 }, (_, i) => ({ + collectionName: 'addresses', + recordId: [i + 1], + values, + })); + const { invoke, model } = buildModel(Array.from({ length: 6 }, (_, i) => `F${i}`)); + const context = makeContext({ + model, + logger: jest.fn(), + agentPort: makeMockAgentPort(relatedData), + runStore: makeMockRunStore(), + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: wideSchema(6), + }), + // AI-assisted so allowNone is true at the call site — the truncation guard, not the mode, + // suppresses the -1 option. + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(selectRecordPrompt(invoke)).not.toContain('-1'); + }); + // The AI sees only the budgeted prefix, but its index maps back into the FULL list — so a // non-zero pick must resolve to the correct full-list record (cap the prompt, not the data). it('maps a non-zero AI index back into the full candidate list after truncation', async () => { @@ -2523,8 +3355,8 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('RelatedRecordNotFoundError', () => { - it('returns error when BelongsTo getRelatedData returns empty array (Branch B)', async () => { + describe('empty candidate list → awaiting-input (No X to load)', () => { + it('falls back to awaiting-input when BelongsTo has no linked record (Branch B)', async () => { const agentPort = makeMockAgentPort([]); const mockModel = makeMockModel({ relation: relationOption({ @@ -2545,14 +3377,13 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - 'The related record could not be found. It may have been deleted.', - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([]); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); - it('returns error when HasMany getRelatedData returns empty array (Branch B)', async () => { + it('falls back to awaiting-input when HasMany returns an empty array (Branch B)', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -2578,11 +3409,10 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - 'The related record could not be found. It may have been deleted.', - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([]); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); }); @@ -2637,9 +3467,9 @@ describe('LoadRelatedRecordStepExecutor', () => { }); describe('select-relation-to-follow failure', () => { - // With >=2 candidates the AI must echo back one of the offered labels. A label that - // matches no option (here a fabricated relation) is rejected as an invalid AI response. - it('returns error when AI selects a relation label not among the offered options', async () => { + // With >=2 candidates the AI must echo back an offered label; a fabricated one is an invalid + // response → degrade to Manual (see AiAssistUnavailableError). + it('degrades to manual selection when AI selects a relation label not among the offered options', async () => { const agentPort = makeMockAgentPort(); // Default schema has two relations (Order, Address) → select-relation-to-follow IS invoked. const mockModel = makeMockModel({ @@ -2661,16 +3491,17 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI made an unexpected choice. Try rephrasing the step's prompt.", - ); - expect(agentPort.getRelatedData).not.toHaveBeenCalled(); + // Degrade picks the first eligible relation (Order) and presents its linked record — no auto-load. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); }); describe('AI malformed/missing tool call', () => { - it('returns error on malformed tool call', async () => { + // A malformed/missing tool call is an AI failure → degrade to Manual (see AiAssistUnavailableError). + it('degrades to manual selection on a malformed tool call', async () => { const invoke = jest.fn().mockResolvedValue({ tool_calls: [], invalid_tool_calls: [ @@ -2688,16 +3519,13 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.type).toBe('record'); - expect(result.stepOutcome.stepId).toBe('load-1'); - expect(result.stepOutcome.stepIndex).toBe(0); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI returned an unexpected response. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); - it('returns error when AI returns no tool call', async () => { + it('degrades to manual selection when AI returns no tool call', async () => { const invoke = jest.fn().mockResolvedValue({ tool_calls: [] }); const bindTools = jest.fn().mockReturnValue({ invoke }); const runStore = makeMockRunStore(); @@ -2710,13 +3538,10 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.type).toBe('record'); - expect(result.stepOutcome.stepId).toBe('load-1'); - expect(result.stepOutcome.stepIndex).toBe(0); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI couldn't decide what to do. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); }); @@ -3120,16 +3945,43 @@ describe('LoadRelatedRecordStepExecutor', () => { collectionDisplayName: 'Addresses', }), }); - const mockModel = makeMockModel({ - relation: relationOption({ - recordId: [42], - relationDisplayName: 'Address', - relatedCollectionName: 'addresses', - }), - reasoning: 'Load address', - }); + // 3 AI calls (2 relations → follow; lone candidate → field + record ranking). The ranking + // reuses the already-fetched 'addresses' schema, so no extra getCollectionSchema call. + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-relation-to-follow', + args: { + relation: relationOption({ + recordId: [42], + relationDisplayName: 'Address', + relatedCollectionName: 'addresses', + }), + reasoning: 'Load address', + }, + id: 'c1', + }, + ], + }) + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['Email'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 0, reasoning: 'The only candidate is relevant' }, + id: 'c3', + }, + ], + }); + const model = { + bindTools: jest.fn().mockReturnValue({ invoke }), + } as unknown as ExecutionContext['model']; const context = makeContext({ - model: mockModel.model, + model, workflowPort, stepDefinition: makeStep({ executionType: StepExecutionMode.FullyAutomated }), }); @@ -3610,8 +4462,9 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(result.stepOutcome.error).toBe('The pre-configured step parameters are invalid'); }); - it('surfaces a distinct "no source record" message when the source step loaded nothing', async () => { - // The source step exists in the live path but its run-store execution has no record. + it('errors (Full AI) when the source step loaded no record', async () => { + // Source step exists but its run-store execution has no record. Full AI surfaces the error (the + // front offers "continue without"). const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore, @@ -3628,6 +4481,24 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(result.stepOutcome.error).toContain("didn't load any record"); }); + it('surfaces a "no source record" error in await modes when the source step loaded nothing', async () => { + // Await modes surface the same error as Full AI so the front can offer "continue without". + const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); + const context = makeContext({ + runStore, + previousSteps: [makeLoadRelatedPreviousStep(1)], + stepDefinition: makeStep({ + executionType: StepExecutionMode.AutomatedWithConfirmation, + preRecordedArgs: { selectedRecordStepId: 'load-1', relationName: 'customer' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('error'); + expect(result.stepOutcome.error).toContain("didn't load any record"); + }); + it('errors with the pre-recorded-args message when the pinned relation matches nothing', async () => { const context = makeContext({ stepDefinition: makeStep({ diff --git a/packages/workflow-executor/test/http/step-serializer.test.ts b/packages/workflow-executor/test/http/step-serializer.test.ts index 1ff16a3c6c..3dc5cc8021 100644 --- a/packages/workflow-executor/test/http/step-serializer.test.ts +++ b/packages/workflow-executor/test/http/step-serializer.test.ts @@ -108,6 +108,19 @@ describe('serializeStepForWire', () => { expect(result.executionResult).toEqual({ skipped: true }); }); + + it('does not throw when selectedRecordRef is absent (Full AI no-source skip)', () => { + const step: LoadRelatedRecordStepExecutionData = { + type: 'load-related-record', + stepIndex: 2, + executionResult: { skipped: true }, + }; + + const result = serializeStepForWire(step) as Record; + + expect(result.executionResult).toEqual({ skipped: true }); + expect('selectedRecordRef' in result).toBe(false); + }); }); it('returns non-record steps unchanged', () => { diff --git a/packages/workflow-executor/test/types/step-definition.test.ts b/packages/workflow-executor/test/types/step-definition.test.ts new file mode 100644 index 0000000000..04fdd1c6aa --- /dev/null +++ b/packages/workflow-executor/test/types/step-definition.test.ts @@ -0,0 +1,43 @@ +import { + LoadRelatedRecordStepDefinitionSchema, + StepExecutionMode, + StepType, +} from '../../src/types/validated/step-definition'; + +describe('LoadRelatedRecordStepDefinitionSchema executionType', () => { + const base = { type: StepType.LoadRelatedRecord as const }; + + it('parses each valid execution mode to its own value', () => { + expect( + LoadRelatedRecordStepDefinitionSchema.parse({ ...base, executionType: 'manual' }) + .executionType, + ).toBe(StepExecutionMode.Manual); + expect( + LoadRelatedRecordStepDefinitionSchema.parse({ + ...base, + executionType: 'automated-with-confirmation', + }).executionType, + ).toBe(StepExecutionMode.AutomatedWithConfirmation); + expect( + LoadRelatedRecordStepDefinitionSchema.parse({ ...base, executionType: 'fully-automated' }) + .executionType, + ).toBe(StepExecutionMode.FullyAutomated); + }); + + it('defaults a missing executionType to AutomatedWithConfirmation', () => { + expect(LoadRelatedRecordStepDefinitionSchema.parse(base).executionType).toBe( + StepExecutionMode.AutomatedWithConfirmation, + ); + }); + + // No `.catch` on the enum: an invalid value must be rejected, not silently coerced to + // AutomatedWithConfirmation (which would turn AI prefill back on for a `manual` typo). + it('rejects an invalid executionType instead of coercing it', () => { + const result = LoadRelatedRecordStepDefinitionSchema.safeParse({ + ...base, + executionType: 'not-a-mode', + }); + + expect(result.success).toBe(false); + }); +});