From 527aa7ede109090f8ab8afe1e13de4d7be258088 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 30 Jun 2026 14:51:59 +0200 Subject: [PATCH 01/14] feat(workflow-executor): add 3-way execution mode to the load related record step (PRD-148) Manual presents the narrowed list with no AI; AI-assisted suggests a record; Full AI selects + loads and auto-skips on no source record, empty list, or AI judging none relevant. selectedRecordRef is optional now (no-source skip); the wire serializer guards it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../load-related-record-step-executor.ts | 177 ++++++++---- .../src/http/step-serializer.ts | 6 +- .../src/types/step-execution-data.ts | 4 +- .../src/types/validated/step-definition.ts | 8 +- .../load-related-record-step-executor.test.ts | 265 ++++++++++++++++-- .../test/http/step-serializer.test.ts | 13 + 6 files changed, 396 insertions(+), 77 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 170f91b347..60ea709e01 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -21,6 +21,7 @@ import { NoRelationshipFieldsError, RelatedRecordNotFoundError, RelationNotFoundError, + SourceRecordMissingError, StepStateError, } from '../errors'; import RecordStepExecutor from './record-step-executor'; @@ -103,13 +104,18 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - if (!execution.pendingData) { + if (!execution.pendingData || !execution.selectedRecordRef) { throw new StepStateError(`Step at index ${this.context.stepIndex} has no pending data`); } + // Switching the relation in Manual mode must still not pre-select a record. + const suggestViaAi = this.context.stepDefinition.executionType !== StepExecutionMode.Manual; const schema = await this.getCollectionSchema(execution.selectedRecordRef.collectionName); const target = await this.buildTarget(schema, fieldName, execution.selectedRecordRef); - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target); + const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds( + target, + suggestViaAi, + ); await this.context.runStore.saveStepExecution(this.context.runId, { ...execution, @@ -127,23 +133,26 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { stepDefinition: step } = this.context; - const target = await this.resolveTarget(); + // Manual mode never invokes AI — neither to pick the relation nor to suggest a record. + const useAi = step.executionType !== StepExecutionMode.Manual; - // Branch B -- fully automated execution + // Branch B -- Full AI: AI selects and the record is loaded with no user input (may auto-skip). if (step.executionType === StepExecutionMode.FullyAutomated) { - return this.resolveAndLoadAutomatic(target); + return this.resolveAndLoadAutomatic(useAi); } - // Branch C -- pre-fetch candidates, await user confirmation + // Branches C & D -- pre-fetch candidates, await user confirmation. AI-assisted pre-selects a + // record (suggestedRecord); Manual presents the narrowed list with no AI pick. + const target = await this.resolveTarget(useAi); const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); - return this.saveAndAwaitInput(target, sourceSchema); + return this.saveAndAwaitInput(target, sourceSchema, useAi); } // Picks the (record, relation) pair to follow. Unlike a separate record-then-relation choice, // this lets the AI decide by what each relation LEADS TO — so "load the dvd" follows // store→dvds rather than latching onto a previously-loaded dvd whose collection just matches. - private async resolveTarget(): Promise { + private async resolveTarget(useAi: boolean): Promise { const { preRecordedArgs } = this.context.stepDefinition; const sourceRecords = @@ -168,8 +177,10 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor 1 && useAi ? await this.selectRelationToFollow(eligible) : eligible[0]; return this.targetFromCandidate(chosen); } @@ -265,15 +276,20 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { selectedRecordRef, name, displayName } = target; - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target); + const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds( + target, + suggestViaAi, + ); const availableFields: RelationRef[] = sourceSchema.fields .filter(isFollowableRelation) @@ -294,21 +310,28 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { const candidate = await this.fetchXToOneCandidate(target); + // The lone xToOne record pre-fills in every mode — it's the only option, not an AI pick. return candidate ? { availableRecordIds: [candidate], suggestedRecord: candidate } : { availableRecordIds: [] }; } + // Rank (AI-suggest) only when AI-assisted. allowNone is Full-AI-only, never on the await path — + // here the human is the one who decides "none relevant" via the checkbox. const { relatedData, bestIndex, relatedSchema } = await this.selectBestFromRelatedData( target, 50, + { rank: suggestViaAi, allowNone: false }, ); if (relatedData.length === 0) { @@ -325,7 +348,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor= 0 ? toCandidate(relatedData[bestIndex]) : undefined, }; } @@ -338,14 +362,32 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + /** + * Branch B: Full AI. xToOne loads the linked record; HasMany ranks candidates via AI; + * BelongsToMany takes the first. Auto-skips (persists `skipped` + success) when there is no source + * record, no candidate, or the AI judges none relevant — the run then advances with nothing loaded. + */ + private async resolveAndLoadAutomatic(useAi: boolean): Promise { + let target: RelationTarget; + + try { + target = await this.resolveTarget(useAi); + } catch (error) { + // No source record to load from → Full AI auto-continues. (Manual/AI-assisted let this error + // surface so the front can offer "continue without" — PRD-550.) + if (error instanceof SourceRecordMissingError) return this.persistSkip(); + throw error; + } + const record = await this.fetchRecordForRelation(target); + // Empty candidate list or AI judged none relevant → skip and move on. + if (record === null) return this.persistSkip(target.selectedRecordRef); + return this.persistAndReturn(record, target, undefined); } - private async fetchRecordForRelation(target: RelationTarget): Promise { + private async fetchRecordForRelation(target: RelationTarget): Promise { if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { return this.fetchXToOneRecordRef(target); } @@ -357,12 +399,11 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + // Returns null (→ Full AI skip) when the relation has no linked record. + private async fetchXToOneRecordRef(target: RelationTarget): Promise { const candidate = await this.fetchXToOneCandidate(target); - if (!candidate) { - throw new RelatedRecordNotFoundError(target.selectedRecordRef.collectionName, target.name); - } + if (!candidate) return null; return { collectionName: target.relatedCollectionName, @@ -401,7 +442,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { selectedRecordRef, pendingData, userConfirmation } = execution; - if (!pendingData) { + if (!pendingData || !selectedRecordRef) { throw new StepStateError(`Step at index ${this.context.stepIndex} has no pending data`); } @@ -448,6 +489,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor, limit: number, + opts: { rank: boolean; allowNone: boolean } = { rank: true, allowNone: false }, ): Promise<{ relatedData: RecordData[]; bestIndex: number; @@ -457,13 +499,20 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - const { relatedData, bestIndex } = await this.selectBestFromRelatedData(target, 50); + /** + * HasMany + Full AI: fetch top 50, then AI selects the best record. Returns null (→ skip) when + * there is no candidate or the AI judges none of them relevant. + */ + private async selectBestRelatedRecord(target: RelationTarget): Promise { + const { relatedData, bestIndex } = await this.selectBestFromRelatedData(target, 50, { + rank: true, + allowNone: true, + }); - if (relatedData.length === 0) { - throw new RelatedRecordNotFoundError(target.selectedRecordRef.collectionName, target.name); - } + if (relatedData.length === 0 || bestIndex < 0) return null; return this.toRecordRef(relatedData[bestIndex]); } - private async fetchFirstCandidate(target: RelationTarget): Promise { - const candidates = await this.fetchCandidates(target, 1); - - return candidates[0]; - } - - private async fetchCandidates( - target: Pick, - limit: number, - ): Promise { - const { selectedRecordRef, name } = target; + // BelongsToMany + Full AI: take the first candidate. Returns null (→ skip) when there is none. + private async fetchFirstCandidate(target: RelationTarget): Promise { const relatedSchema = await this.getCollectionSchema(target.relatedCollectionName); - const relatedData = await this.fetchRelatedData(target, relatedSchema, limit); + const relatedData = await this.fetchRelatedData(target, relatedSchema, 1); - if (relatedData.length === 0) { - throw new RelatedRecordNotFoundError(selectedRecordRef.collectionName, name); - } - - return relatedData.map(r => this.toRecordRef(r)); + return relatedData.length > 0 ? this.toRecordRef(relatedData[0]) : null; } private async fetchRelatedData( @@ -544,6 +585,20 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + await this.context.runStore.saveStepExecution(this.context.runId, { + type: 'load-related-record', + stepIndex: this.context.stepIndex, + selectedRecordRef, + executionResult: { skipped: true }, + }); + + return this.buildOutcomeResult({ status: 'success' }); + } + private relationOptionLabel(candidate: RelationCandidate): string { const { record, schema, field } = candidate; @@ -650,11 +705,15 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor nonRelationFields.find(f => f.displayName === dn)?.fieldName ?? dn); } - /** AI call 2 for HasMany: selects the best record by index from the candidate list. */ + /** + * AI call 2 for HasMany: selects the best record by index from the candidate list. When + * `allowNone` (Full AI), the AI may return -1 to signal that none of the candidates is relevant. + */ private async selectBestRecordIndex( candidates: RecordData[], fieldNames: string[], prompt: string | undefined, + allowNone = false, ): Promise { const filteredCandidates = candidates.map((c, i) => { const entries = Object.entries(c.values).filter( @@ -690,6 +749,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor maxIndex) { + // NOTE: The Zod schema's .min().max() shapes the tool prompt only — it is NOT validated against + // the AI response. This guard is the sole runtime enforcement. -1 (none relevant) is accepted + // only when allowNone. + if (!Number.isInteger(recordIndex) || recordIndex < minIndex || recordIndex > maxIndex) { throw new InvalidAIResponseError( - `AI selected record index ${recordIndex} which is out of range (0-${maxIndex}) or not an integer`, + `AI selected record index ${recordIndex} which is out of range (${minIndex}-${maxIndex}) or not an integer`, ); } diff --git a/packages/workflow-executor/src/http/step-serializer.ts b/packages/workflow-executor/src/http/step-serializer.ts index 7a8a7de619..f5c057c8fb 100644 --- a/packages/workflow-executor/src/http/step-serializer.ts +++ b/packages/workflow-executor/src/http/step-serializer.ts @@ -15,9 +15,13 @@ export default function serializeStepForWire(step: StepExecutionData): unknown { return { ...step, selectedRecordRef: serializeRecordRef(step.selectedRecordRef) }; case 'load-related-record': { + // selectedRecordRef is absent on a Full AI no-source auto-skip — omit it rather than + // dereferencing undefined (which would throw and 503 the run-fetch). const result: Record = { ...step, - selectedRecordRef: serializeRecordRef(step.selectedRecordRef), + ...(step.selectedRecordRef && { + selectedRecordRef: serializeRecordRef(step.selectedRecordRef), + }), }; if (step.pendingData) { diff --git a/packages/workflow-executor/src/types/step-execution-data.ts b/packages/workflow-executor/src/types/step-execution-data.ts index 0585c237d6..59448737ee 100644 --- a/packages/workflow-executor/src/types/step-execution-data.ts +++ b/packages/workflow-executor/src/types/step-execution-data.ts @@ -183,7 +183,9 @@ export interface LoadRelatedRecordStepExecutionData WithUserConfirmation { type: 'load-related-record'; pendingData?: LoadRelatedRecordPendingData; - selectedRecordRef: RecordRef; + // Absent only for the Full AI auto-skip when there is no source record to load from + // (nothing was resolved). Always set on the await/load paths. + selectedRecordRef?: RecordRef; executionParams?: RelationRef; executionResult?: { relation: RelationRef; record: RecordRef } | { skipped: true }; } diff --git a/packages/workflow-executor/src/types/validated/step-definition.ts b/packages/workflow-executor/src/types/validated/step-definition.ts index 6ac1a42c46..089f9856da 100644 --- a/packages/workflow-executor/src/types/validated/step-definition.ts +++ b/packages/workflow-executor/src/types/validated/step-definition.ts @@ -98,10 +98,12 @@ export type TriggerActionStepDefinition = z.infer { // fetchCandidates throws RelatedRecordNotFoundError when the agent returns an // empty list. Same user-facing message as the other empty-result paths. - it('returns error when getRelatedData returns an empty array', async () => { + it('auto-skips when getRelatedData returns an empty array (Branch B)', async () => { const belongsToManySchema = makeCollectionSchema({ fields: [ { @@ -1025,11 +1025,213 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - 'The related record could not be found. It may have been deleted.', + // Full AI: no candidate to load → skip so the run can advance (no error). + expect(result.stepOutcome.status).toBe('success'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + type: 'load-related-record', + executionResult: { skipped: true }, + }), + ); + }); + }); + + describe('executionType=Manual: no AI pre-selection (PRD-148)', () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + + it('lists the narrowed candidates with no suggestion and never invokes the AI model', async () => { + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.Manual, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + // Manual mode performs zero AI calls — no relation chooser, no field/record ranking. + expect(bindTools).not.toHaveBeenCalled(); + + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls[0][1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + }); + + it('still pre-fills the single xToOne record in Manual (the only option, not an AI pick)', async () => { + // BelongsTo 'order' → one linked record; pre-filled even in Manual, with no AI call. + const agentPort = makeMockAgentPort(); // default: 1 related order #99 via getSingleRelatedData + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + stepDefinition: makeStep({ + executionType: StepExecutionMode.Manual, + preRecordedArgs: { relationName: 'order' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + expect(bindTools).not.toHaveBeenCalled(); + + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls[0][1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand(['99'])]); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); + }); + + it('switching relation (refreshCandidatesForField) in Manual makes no AI suggestion', async () => { + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'order', displayName: 'Order' }, + availableRecordIds: [cand(['99'])], + suggestedRecord: cand(['99']), + }, + }); + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]); + const addressesSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: addressesSchema, + }), + incomingPendingData: { fieldName: 'address' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.Manual }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + // Manual: switching to a multi-record relation must neither call the AI nor pre-select. + expect(bindTools).not.toHaveBeenCalled(); + + const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(finalSave.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(finalSave.pendingData.suggestedRecord).toBeUndefined(); + }); + }); + + describe('executionType=FullyAutomated: AI judges none relevant → skip (PRD-148)', () => { + it('skips (no record loaded) when select-record-by-content returns -1', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: -1, reasoning: 'None of the candidates is relevant' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('success'); + // Two AI calls run (field + record selection); the record call returns -1 → skip. + expect(bindTools).toHaveBeenCalledTimes(2); + expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + type: 'load-related-record', + executionResult: { skipped: true }, + }), ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); }); }); @@ -2524,7 +2726,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); describe('RelatedRecordNotFoundError', () => { - it('returns error when BelongsTo getRelatedData returns empty array (Branch B)', async () => { + it('auto-skips when BelongsTo getRelatedData returns empty array (Branch B)', async () => { const agentPort = makeMockAgentPort([]); const mockModel = makeMockModel({ relation: relationOption({ @@ -2545,14 +2747,18 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - 'The related record could not be found. It may have been deleted.', + // Full AI: linked record absent → skip (no error). + expect(result.stepOutcome.status).toBe('success'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + type: 'load-related-record', + executionResult: { skipped: true }, + }), ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); }); - it('returns error when HasMany getRelatedData returns empty array (Branch B)', async () => { + it('auto-skips when HasMany getRelatedData returns empty array (Branch B)', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -2578,11 +2784,15 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - 'The related record could not be found. It may have been deleted.', + // Full AI: empty candidate list → skip (no error). + expect(result.stepOutcome.status).toBe('success'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + type: 'load-related-record', + executionResult: { skipped: true }, + }), ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); }); }); @@ -3610,7 +3820,7 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(result.stepOutcome.error).toBe('The pre-configured step parameters are invalid'); }); - it('surfaces a distinct "no source record" message when the source step loaded nothing', async () => { + it('auto-skips (Full AI) when the source step loaded nothing', async () => { // The source step exists in the live path but its run-store execution has no record. const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ @@ -3624,6 +3834,29 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); + // Full AI: no source record to load from → skip and let the run advance. + expect(result.stepOutcome.status).toBe('success'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ executionResult: { skipped: true } }), + ); + }); + + it('surfaces a "no source record" error in await modes when the source step loaded nothing', async () => { + // Only Full AI auto-skips; Manual / AI-assisted surface the error so the front can offer + // "continue without" (PRD-550). + const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); + const context = makeContext({ + runStore, + previousSteps: [makeLoadRelatedPreviousStep(1)], + stepDefinition: makeStep({ + executionType: StepExecutionMode.AutomatedWithConfirmation, + preRecordedArgs: { selectedRecordStepId: 'load-1', relationName: 'customer' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + expect(result.stepOutcome.status).toBe('error'); expect(result.stepOutcome.error).toContain("didn't load any record"); }); diff --git a/packages/workflow-executor/test/http/step-serializer.test.ts b/packages/workflow-executor/test/http/step-serializer.test.ts index 1ff16a3c6c..3dc5cc8021 100644 --- a/packages/workflow-executor/test/http/step-serializer.test.ts +++ b/packages/workflow-executor/test/http/step-serializer.test.ts @@ -108,6 +108,19 @@ describe('serializeStepForWire', () => { expect(result.executionResult).toEqual({ skipped: true }); }); + + it('does not throw when selectedRecordRef is absent (Full AI no-source skip)', () => { + const step: LoadRelatedRecordStepExecutionData = { + type: 'load-related-record', + stepIndex: 2, + executionResult: { skipped: true }, + }; + + const result = serializeStepForWire(step) as Record; + + expect(result.executionResult).toEqual({ skipped: true }); + expect('selectedRecordRef' in result).toBe(false); + }); }); it('returns non-record steps unchanged', () => { From ea0b5f83320aeb73c58f445a0f9c35a771144700 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 30 Jun 2026 15:06:07 +0200 Subject: [PATCH 02/14] docs(workflow-executor): clarify the Manual source-record default in resolveTarget (PRD-148) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/executors/load-related-record-step-executor.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 60ea709e01..be91d73e94 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -177,8 +177,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor 1 && useAi ? await this.selectRelationToFollow(eligible) : eligible[0]; From 15793400ce3ce894a736194576386012cb9a165b Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 30 Jun 2026 15:13:58 +0200 Subject: [PATCH 03/14] fix(workflow-executor): don't allow none-relevant skip on a truncated candidate list (PRD-148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the HasMany candidate list is truncated to fit the AI prompt budget, -1 (none relevant) is no longer offered — the AI picks from what it saw, so a match in the unseen tail is not silently skipped. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../executors/load-related-record-step-executor.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index be91d73e94..9b0e3b40e5 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -741,7 +741,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor maxIndex) { throw new InvalidAIResponseError( `AI selected record index ${recordIndex} which is out of range (${minIndex}-${maxIndex}) or not an integer`, From 4f8b3981705cbf2015530c79ceca2c29731f7c46 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 30 Jun 2026 15:39:24 +0200 Subject: [PATCH 04/14] style(workflow-executor): trim comments to the non-obvious why only (PRD-148) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../load-related-record-step-executor.ts | 39 +++---------------- .../src/http/step-serializer.ts | 3 +- .../src/types/step-execution-data.ts | 3 +- .../src/types/validated/step-definition.ts | 4 +- 4 files changed, 8 insertions(+), 41 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 9b0e3b40e5..63ea394f17 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -108,7 +108,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { stepDefinition: step } = this.context; - // Manual mode never invokes AI — neither to pick the relation nor to suggest a record. const useAi = step.executionType !== StepExecutionMode.Manual; - // Branch B -- Full AI: AI selects and the record is loaded with no user input (may auto-skip). if (step.executionType === StepExecutionMode.FullyAutomated) { return this.resolveAndLoadAutomatic(useAi); } - // Branches C & D -- pre-fetch candidates, await user confirmation. AI-assisted pre-selects a - // record (suggestedRecord); Manual presents the narrowed list with no AI pick. const target = await this.resolveTarget(useAi); const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); @@ -277,9 +272,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor= 0 ? toCandidate(relatedData[bestIndex]) : undefined, }; } @@ -363,11 +352,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { let target: RelationTarget; @@ -382,7 +366,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const { relatedData, bestIndex } = await this.selectBestFromRelatedData(target, 50, { rank: true, @@ -543,7 +520,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const relatedSchema = await this.getCollectionSchema(target.relatedCollectionName); const relatedData = await this.fetchRelatedData(target, relatedSchema, 1); @@ -586,9 +562,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { await this.context.runStore.saveStepExecution(this.context.runId, { type: 'load-related-record', @@ -706,10 +681,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor nonRelationFields.find(f => f.displayName === dn)?.fieldName ?? dn); } - /** - * AI call 2 for HasMany: selects the best record by index from the candidate list. When - * `allowNone` (Full AI), the AI may return -1 to signal that none of the candidates is relevant. - */ private async selectBestRecordIndex( candidates: RecordData[], fieldNames: string[], diff --git a/packages/workflow-executor/src/http/step-serializer.ts b/packages/workflow-executor/src/http/step-serializer.ts index f5c057c8fb..198fcff7e3 100644 --- a/packages/workflow-executor/src/http/step-serializer.ts +++ b/packages/workflow-executor/src/http/step-serializer.ts @@ -15,8 +15,7 @@ export default function serializeStepForWire(step: StepExecutionData): unknown { return { ...step, selectedRecordRef: serializeRecordRef(step.selectedRecordRef) }; case 'load-related-record': { - // selectedRecordRef is absent on a Full AI no-source auto-skip — omit it rather than - // dereferencing undefined (which would throw and 503 the run-fetch). + // Omit selectedRecordRef when absent (Full AI no-source skip) — serializing undefined throws. const result: Record = { ...step, ...(step.selectedRecordRef && { diff --git a/packages/workflow-executor/src/types/step-execution-data.ts b/packages/workflow-executor/src/types/step-execution-data.ts index 59448737ee..626c435e28 100644 --- a/packages/workflow-executor/src/types/step-execution-data.ts +++ b/packages/workflow-executor/src/types/step-execution-data.ts @@ -183,8 +183,7 @@ export interface LoadRelatedRecordStepExecutionData WithUserConfirmation { type: 'load-related-record'; pendingData?: LoadRelatedRecordPendingData; - // Absent only for the Full AI auto-skip when there is no source record to load from - // (nothing was resolved). Always set on the await/load paths. + // Absent only on the Full AI no-source auto-skip; always set on the await/load paths. selectedRecordRef?: RecordRef; executionParams?: RelationRef; executionResult?: { relation: RelationRef; record: RecordRef } | { skipped: true }; diff --git a/packages/workflow-executor/src/types/validated/step-definition.ts b/packages/workflow-executor/src/types/validated/step-definition.ts index 089f9856da..a2c6cda898 100644 --- a/packages/workflow-executor/src/types/validated/step-definition.ts +++ b/packages/workflow-executor/src/types/validated/step-definition.ts @@ -98,9 +98,7 @@ export type TriggerActionStepDefinition = z.infer Date: Wed, 1 Jul 2026 17:36:37 +0200 Subject: [PATCH 05/14] feat(workflow-executor): degrade Full AI to a confirmation and let the AI decline (PRD-148) Full AI no longer silently skips when it can't load a relevant record. With nothing relevant it degrades to an AI-assisted confirmation (a human decides, "No X to load" pre-checked); no source record now surfaces an error like the await modes instead of skipping. AI-assisted may decline too (allowNone), and the record-selection prompt tells the AI to return -1 on an impossible request rather than forcing a weak match. pendingData carries a suggestNoRecord flag so the front distinguishes an active "nothing relevant" from a plain no-suggestion (Manual). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../load-related-record-step-executor.ts | 130 ++++------ .../src/http/step-serializer.ts | 2 +- .../src/types/step-execution-data.ts | 5 +- .../load-related-record-step-executor.test.ts | 223 +++++++++++++----- 4 files changed, 222 insertions(+), 138 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 63ea394f17..6f7b9bc124 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -21,7 +21,6 @@ import { NoRelationshipFieldsError, RelatedRecordNotFoundError, RelationNotFoundError, - SourceRecordMissingError, StepStateError, } from '../errors'; import RecordStepExecutor from './record-step-executor'; @@ -43,6 +42,10 @@ Choose the fields that are most useful for determining which record best matches const SELECT_RECORD_SYSTEM_PROMPT = `You are an AI agent selecting the most relevant related record from a list of candidates. Choose the record that best matches the user request based on the provided field values.`; +// Only appended when -1 is a valid answer. The base prompt tells the AI to always pick, so without +// this it forces a weak match on an impossible request instead of declining. +const SELECT_RECORD_NONE_ALLOWED_PROMPT = `If the request states a specific requirement that NO candidate satisfies, return -1 instead of forcing an arbitrary or loosely-related match. Do NOT return -1 merely because a match is imperfect — only when no candidate genuinely fits the request.`; + // Bound only what is sent to the AI in selectBestRecordIndex — the full candidate list is // always returned to the front via availableRecordIds. These cap the prompt size, not the data. const MAX_RELEVANT_FIELDS = 6; @@ -135,7 +138,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - const { selectedRecordRef, name, displayName } = target; - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds( target, suggestViaAi, ); + return this.persistAwaitInput(target, sourceSchema, { + availableRecordIds, + suggestedRecord, + // An AI pass that yields no record is a deliberate "nothing relevant" → pre-check "No X to load". + // A Manual pass with no suggestion just means the user picks, so no pre-check. + suggestNoRecord: suggestViaAi && !suggestedRecord, + }); + } + + private async persistAwaitInput( + target: RelationTarget, + sourceSchema: CollectionSchema, + pending: { + availableRecordIds: LoadRelatedRecordCandidate[]; + suggestedRecord?: LoadRelatedRecordCandidate; + suggestNoRecord: boolean; + }, + ): Promise { + const { selectedRecordRef, name, displayName } = target; + const availableFields: RelationRef[] = sourceSchema.fields .filter(isFollowableRelation) .map(f => ({ name: f.fieldName, displayName: f.displayName })); @@ -295,8 +316,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - let target: RelationTarget; - - try { - target = await this.resolveTarget(useAi); - } catch (error) { - // No source record to load from → Full AI auto-continues. (Manual/AI-assisted let this error - // surface so the front can offer "continue without" — PRD-550.) - if (error instanceof SourceRecordMissingError) return this.persistSkip(); - throw error; - } + private async resolveAndLoadAutomatic(): Promise { + // No source record throws (like Manual/AI-assisted) → the front offers "continue without" (PRD-550). + const target = await this.resolveTarget(true); + const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target, true); - const record = await this.fetchRecordForRelation(target); + // Full AI auto-loads a confident pick and advances. With nothing relevant to load it degrades to + // an AI-assisted confirmation (a human decides, "No X to load" pre-checked) instead of skipping — + // mirrors the Trigger Action automated→confirmation fallback. + if (!suggestedRecord) { + const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); - if (record === null) return this.persistSkip(target.selectedRecordRef); - - return this.persistAndReturn(record, target, undefined); - } - - private async fetchRecordForRelation(target: RelationTarget): Promise { - if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { - return this.fetchXToOneRecordRef(target); - } - - if (target.relationType === 'HasMany') { - return this.selectBestRelatedRecord(target); + return this.persistAwaitInput(target, sourceSchema, { + availableRecordIds, + suggestNoRecord: true, + }); } - return this.fetchFirstCandidate(target); - } - - // Returns null (→ Full AI skip) when the relation has no linked record. - private async fetchXToOneRecordRef(target: RelationTarget): Promise { - const candidate = await this.fetchXToOneCandidate(target); - - if (!candidate) return null; - - return { + const record: RecordRef = { collectionName: target.relatedCollectionName, - recordId: candidate.recordId, + recordId: suggestedRecord.recordId, stepIndex: this.context.stepIndex, }; + + return this.persistAndReturn(record, target, undefined); } private async fetchXToOneCandidate( @@ -509,24 +515,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - const { relatedData, bestIndex } = await this.selectBestFromRelatedData(target, 50, { - rank: true, - allowNone: true, - }); - - if (relatedData.length === 0 || bestIndex < 0) return null; - - return this.toRecordRef(relatedData[bestIndex]); - } - - private async fetchFirstCandidate(target: RelationTarget): Promise { - const relatedSchema = await this.getCollectionSchema(target.relatedCollectionName); - const relatedData = await this.fetchRelatedData(target, relatedSchema, 1); - - return relatedData.length > 0 ? this.toRecordRef(relatedData[0]) : null; - } - private async fetchRelatedData( target: Pick, relatedSchema: CollectionSchema, @@ -562,19 +550,6 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - await this.context.runStore.saveStepExecution(this.context.runId, { - type: 'load-related-record', - stepIndex: this.context.stepIndex, - selectedRecordRef, - executionResult: { skipped: true }, - }); - - return this.buildOutcomeResult({ status: 'success' }); - } - private relationOptionLabel(candidate: RelationCandidate): string { const { record, schema, field } = candidate; @@ -749,6 +724,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor = { ...step, ...(step.selectedRecordRef && { diff --git a/packages/workflow-executor/src/types/step-execution-data.ts b/packages/workflow-executor/src/types/step-execution-data.ts index 626c435e28..537a0ae9a3 100644 --- a/packages/workflow-executor/src/types/step-execution-data.ts +++ b/packages/workflow-executor/src/types/step-execution-data.ts @@ -176,6 +176,9 @@ export interface LoadRelatedRecordPendingData { availableRecordIds: LoadRelatedRecordCandidate[]; // Absent when the relation has no linked record(s): the list is empty and there's nothing to suggest. suggestedRecord?: LoadRelatedRecordCandidate; + // The AI actively judged no candidate relevant (incl. Full AI degrading to confirmation) → the front + // pre-checks "No X to load". Distinct from a plain absent suggestedRecord (Manual: the user picks). + suggestNoRecord?: boolean; } export interface LoadRelatedRecordStepExecutionData @@ -183,7 +186,7 @@ export interface LoadRelatedRecordStepExecutionData WithUserConfirmation { type: 'load-related-record'; pendingData?: LoadRelatedRecordPendingData; - // Absent only on the Full AI no-source auto-skip; always set on the await/load paths. + // Set on every await/load path (and preserved through the user-initiated "continue without" skip). selectedRecordRef?: RecordRef; executionParams?: RelationRef; executionResult?: { relation: RelationRef; record: RecordRef } | { skipped: true }; diff --git a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts index dd6c8d82c1..4f7b5145c0 100644 --- a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts +++ b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts @@ -975,13 +975,14 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.status).toBe('success'); - // To-many path: /relationships call with limit: 1, no parent-record projection. + // To-many path: /relationships call with the candidate-list limit (a lone record short-circuits + // ranking, so it loads without an AI call). expect(agentPort.getRelatedData).toHaveBeenCalledWith( expect.objectContaining({ collection: 'customers', id: [42], relation: 'tags', - limit: 1, + limit: 50, relatedSchema: expect.objectContaining({ collectionName: 'tags' }), }), expect.objectContaining({ id: 1 }), @@ -997,9 +998,9 @@ describe('LoadRelatedRecordStepExecutor', () => { ); }); - // fetchCandidates throws RelatedRecordNotFoundError when the agent returns an - // empty list. Same user-facing message as the other empty-result paths. - it('auto-skips when getRelatedData returns an empty array (Branch B)', async () => { + // Empty list → nothing to load: Full AI degrades to an AI-assisted confirmation ("No X to load" + // pre-checked) instead of skipping, so a human decides. + it('falls back to awaiting-input (No X to load) when getRelatedData returns an empty array (Branch B)', async () => { const belongsToManySchema = makeCollectionSchema({ fields: [ { @@ -1025,15 +1026,12 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Full AI: no candidate to load → skip so the run can advance (no error). - expect(result.stepOutcome.status).toBe('success'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ - type: 'load-related-record', - executionResult: { skipped: true }, - }), - ); + // Full AI with no candidate → hands off to the user with "No X to load" pre-checked. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); }); @@ -1165,8 +1163,135 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('executionType=FullyAutomated: AI judges none relevant → skip (PRD-148)', () => { - it('skips (no record loaded) when select-record-by-content returns -1', async () => { + describe('executionType=AutomatedWithConfirmation: AI may decline (PRD-148)', () => { + it('pre-checks "No X to load" (suggestNoRecord) when the AI judges no candidate relevant', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: -1, reasoning: 'None of the candidates is relevant' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.AutomatedWithConfirmation, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + // AI-assisted no longer forces a pick: the AI may return -1, and the user reviews with + // "No X to load" pre-checked (rather than a potentially misleading forced suggestion). + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); + }); + + it('tells the AI it may return -1 in the record-selection prompt (Bug 2)', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 0, reasoning: 'match' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const context = makeContext({ + model, + agentPort, + runStore: makeMockRunStore(), + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.AutomatedWithConfirmation, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + await new LoadRelatedRecordStepExecutor(context).execute(); + + // 2nd AI call = record selection. Without the decline guidance the base prompt forces a pick, + // so an impossible request would return a weak match instead of -1 (Bug 2). + const recordSelectionMessages = invoke.mock.calls[1][0] as Array<{ content: unknown }>; + const content = recordSelectionMessages.map(m => String(m.content)).join('\n'); + expect(content).toContain('-1'); + }); + }); + + describe('executionType=FullyAutomated: AI judges none relevant → confirmation (PRD-148)', () => { + it('falls back to awaiting-input (No X to load) when select-record-by-content returns -1', async () => { const customersWithAddress = makeCollectionSchema({ fields: [ { @@ -1221,17 +1346,15 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); - expect(result.stepOutcome.status).toBe('success'); - // Two AI calls run (field + record selection); the record call returns -1 → skip. + expect(result.stepOutcome.status).toBe('awaiting-input'); + // Two AI calls run (field + record selection); the record call returns -1 → hand off to the + // user with the candidates listed and "No X to load" pre-checked. expect(bindTools).toHaveBeenCalledTimes(2); expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ - type: 'load-related-record', - executionResult: { skipped: true }, - }), - ); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); }); @@ -2725,8 +2848,8 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('RelatedRecordNotFoundError', () => { - it('auto-skips when BelongsTo getRelatedData returns empty array (Branch B)', async () => { + describe('empty candidate list → awaiting-input (No X to load)', () => { + it('falls back to awaiting-input when BelongsTo has no linked record (Branch B)', async () => { const agentPort = makeMockAgentPort([]); const mockModel = makeMockModel({ relation: relationOption({ @@ -2747,18 +2870,14 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Full AI: linked record absent → skip (no error). - expect(result.stepOutcome.status).toBe('success'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ - type: 'load-related-record', - executionResult: { skipped: true }, - }), - ); + // Full AI: linked record absent → hand off to the user with "No X to load" pre-checked. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([]); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); - it('auto-skips when HasMany getRelatedData returns empty array (Branch B)', async () => { + it('falls back to awaiting-input when HasMany returns an empty array (Branch B)', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -2784,15 +2903,11 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Full AI: empty candidate list → skip (no error). - expect(result.stepOutcome.status).toBe('success'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ - type: 'load-related-record', - executionResult: { skipped: true }, - }), - ); + // Full AI: empty candidate list → hand off to the user with "No X to load" pre-checked. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([]); + expect(saved.pendingData.suggestNoRecord).toBe(true); }); }); @@ -3820,8 +3935,10 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(result.stepOutcome.error).toBe('The pre-configured step parameters are invalid'); }); - it('auto-skips (Full AI) when the source step loaded nothing', async () => { - // The source step exists in the live path but its run-store execution has no record. + it('errors (Full AI) when the source step loaded nothing — no longer auto-skips', async () => { + // The source step exists in the live path but its run-store execution has no record. Full AI + // now surfaces the error like the await modes (no silent skip); the front offers "continue + // without" (PRD-550). const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore, @@ -3834,16 +3951,12 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); - // Full AI: no source record to load from → skip and let the run advance. - expect(result.stepOutcome.status).toBe('success'); - expect(runStore.saveStepExecution).toHaveBeenCalledWith( - 'run-1', - expect.objectContaining({ executionResult: { skipped: true } }), - ); + expect(result.stepOutcome.status).toBe('error'); + expect(result.stepOutcome.error).toContain("didn't load any record"); }); it('surfaces a "no source record" error in await modes when the source step loaded nothing', async () => { - // Only Full AI auto-skips; Manual / AI-assisted surface the error so the front can offer + // Same as Full AI now — every mode surfaces the error so the front can offer // "continue without" (PRD-550). const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ From 2b2251d03ea345bd1a0ea8243bcaae82b91ae35d Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Thu, 2 Jul 2026 15:33:38 +0000 Subject: [PATCH 06/14] test(workflow-executor): cover load-related executionType schema parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin LoadRelatedRecordStepDefinitionSchema behaviour: each valid mode parses to its own value (incl. 'manual' → 'manual'), a missing mode defaults to AutomatedWithConfirmation, and an invalid mode is rejected rather than silently coerced (no `.catch`, so a `manual` typo can't flip AI prefill back on). Co-Authored-By: Claude Opus 4.8 --- .../test/types/step-definition.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 packages/workflow-executor/test/types/step-definition.test.ts diff --git a/packages/workflow-executor/test/types/step-definition.test.ts b/packages/workflow-executor/test/types/step-definition.test.ts new file mode 100644 index 0000000000..04fdd1c6aa --- /dev/null +++ b/packages/workflow-executor/test/types/step-definition.test.ts @@ -0,0 +1,43 @@ +import { + LoadRelatedRecordStepDefinitionSchema, + StepExecutionMode, + StepType, +} from '../../src/types/validated/step-definition'; + +describe('LoadRelatedRecordStepDefinitionSchema executionType', () => { + const base = { type: StepType.LoadRelatedRecord as const }; + + it('parses each valid execution mode to its own value', () => { + expect( + LoadRelatedRecordStepDefinitionSchema.parse({ ...base, executionType: 'manual' }) + .executionType, + ).toBe(StepExecutionMode.Manual); + expect( + LoadRelatedRecordStepDefinitionSchema.parse({ + ...base, + executionType: 'automated-with-confirmation', + }).executionType, + ).toBe(StepExecutionMode.AutomatedWithConfirmation); + expect( + LoadRelatedRecordStepDefinitionSchema.parse({ ...base, executionType: 'fully-automated' }) + .executionType, + ).toBe(StepExecutionMode.FullyAutomated); + }); + + it('defaults a missing executionType to AutomatedWithConfirmation', () => { + expect(LoadRelatedRecordStepDefinitionSchema.parse(base).executionType).toBe( + StepExecutionMode.AutomatedWithConfirmation, + ); + }); + + // No `.catch` on the enum: an invalid value must be rejected, not silently coerced to + // AutomatedWithConfirmation (which would turn AI prefill back on for a `manual` typo). + it('rejects an invalid executionType instead of coercing it', () => { + const result = LoadRelatedRecordStepDefinitionSchema.safeParse({ + ...base, + executionType: 'not-a-mode', + }); + + expect(result.success).toBe(false); + }); +}); From f90b8681c628345305435b4868bd8d5aec26692c Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Thu, 2 Jul 2026 15:33:51 +0000 Subject: [PATCH 07/14] feat(workflow-executor): degrade Full AI on low-confidence match and AI failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Load Related Record 3-way execution mode hardening. Bundled because the changes are interwoven in the executor and its test: - feat: Full AI now degrades to an AI-assisted confirmation when the AI returns a best guess but cannot confidently single one out among several viable candidates. selectBestRecordIndex gains a `confident` flag (distinct from the existing -1 "none relevant" sentinel, whose semantics are unchanged); an ambiguous positive pick pre-selects the best guess as suggestedRecord for a human to confirm instead of auto-loading. Confident picks still auto-load; a missing flag defaults to confident (backward compatible). - fix: AI failure/timeout in any AI mode (relation pick, field/record ranking, or a Full AI auto-load) now degrades to the deterministic Manual path — deterministic candidate list presented for the user to pick — instead of erroring the run or auto-skipping. AI calls are tagged via an internal marker so genuine config/data errors still surface. - fix: refreshCandidatesForField rebuilds pendingData from scratch on a field switch (keeping only the immutable relation list) so no stale suggestion state (suggestedRecord / suggestNoRecord) lingers. - style: drop ticket IDs from comments and test titles. - test: pin the truncated-list none-guard (noneAllowed = allowNone && !truncated — the AI is not offered -1 when the list was truncated). - tighten selectBestFromRelatedData options so allowNone only applies when rank is true. Co-Authored-By: Claude Opus 4.8 --- .../load-related-record-step-executor.ts | 196 +++++++--- .../load-related-record-step-executor.test.ts | 350 ++++++++++++++++-- 2 files changed, 459 insertions(+), 87 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 6f7b9bc124..c011f4e67d 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-classes-per-file */ import type { StepExecutionResult } from '../types/execution-context'; import type { LoadRelatedRecordCandidate, @@ -22,6 +23,7 @@ import { RelatedRecordNotFoundError, RelationNotFoundError, StepStateError, + extractErrorMessage, } from '../errors'; import RecordStepExecutor from './record-step-executor'; import { StepExecutionMode } from '../types/validated/step-definition'; @@ -80,6 +82,19 @@ function isFollowableRelation(field: FieldSchema): boolean { return field.isRelationship && Boolean(field.relatedCollectionName || field.polymorphicTypeField); } +// Internal marker: wraps any failure raised while calling the AI (timeout, provider outage, +// missing/malformed tool call, out-of-range choice) so the step handlers can degrade to the +// deterministic Manual path instead of erroring the run. Never leaves this module. +class AiAssistUnavailableError extends Error { + readonly reason: unknown; + + constructor(reason: unknown) { + super('AI assistance unavailable'); + this.name = 'AiAssistUnavailableError'; + this.reason = reason; + } +} + export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { protected async doExecute(): Promise { // Branch A -- Re-entry after pending execution found in RunStore @@ -111,22 +126,37 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + const target = await this.resolveTarget(false); const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); - return this.saveAndAwaitInput(target, sourceSchema, useAi); + return this.saveAndAwaitInput(target, sourceSchema, false); } // Picks the (record, relation) pair to follow. Unlike a separate record-then-relation choice, @@ -179,7 +238,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor 1 && useAi ? await this.selectRelationToFollow(eligible) : eligible[0]; + eligible.length > 1 && useAi + ? await this.withAiAssist(() => this.selectRelationToFollow(eligible)) + : eligible[0]; return this.targetFromCandidate(chosen); } @@ -332,25 +393,29 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { if (target.relationType === 'BelongsTo' || target.relationType === 'HasOne') { const candidate = await this.fetchXToOneCandidate(target); return candidate - ? { availableRecordIds: [candidate], suggestedRecord: candidate } - : { availableRecordIds: [] }; + ? { availableRecordIds: [candidate], suggestedRecord: candidate, ambiguous: false } + : { availableRecordIds: [], ambiguous: false }; } - const { relatedData, bestIndex, relatedSchema } = await this.selectBestFromRelatedData( - target, - 50, - // allowNone: the AI may judge no candidate relevant (→ "No X to load"); only meaningful when - // ranking (Manual passes rank=false and never calls the AI). - { rank: suggestViaAi, allowNone: true }, - ); + const { relatedData, bestIndex, confident, relatedSchema } = + await this.selectBestFromRelatedData( + target, + 50, + // allowNone: the AI may judge no candidate relevant (→ "No X to load"); only meaningful when + // ranking (Manual passes rank=false and never calls the AI). + suggestViaAi ? { rank: true, allowNone: true } : { rank: false }, + ); if (relatedData.length === 0) { - return { availableRecordIds: [] }; + return { availableRecordIds: [], ambiguous: false }; } const referenceField = relatedSchema.referenceField ?? null; @@ -364,6 +429,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor= 0 ? toCandidate(relatedData[bestIndex]) : undefined, + // -1 (none relevant) is not ambiguity; only a low-confidence positive pick is. + ambiguous: bestIndex >= 0 && !confident, }; } @@ -377,19 +444,26 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { - // No source record throws (like Manual/AI-assisted) → the front offers "continue without" (PRD-550). + // No source record throws (like Manual/AI-assisted) → the front offers "continue without". const target = await this.resolveTarget(true); - const { availableRecordIds, suggestedRecord } = await this.collectCandidateIds(target, true); + const { availableRecordIds, suggestedRecord, ambiguous } = await this.collectCandidateIds( + target, + true, + ); - // Full AI auto-loads a confident pick and advances. With nothing relevant to load it degrades to - // an AI-assisted confirmation (a human decides, "No X to load" pre-checked) instead of skipping — - // mirrors the Trigger Action automated→confirmation fallback. - if (!suggestedRecord) { + // Full AI auto-loads a confident pick and advances. It degrades to an AI-assisted confirmation + // (a human decides) instead of auto-loading when either: + // - nothing is relevant to load → "No X to load" pre-checked (suggestNoRecord); or + // - the AI cannot confidently single out one of several viable candidates → its best guess is + // pre-selected for the human to confirm (no "No X to load"). + // Mirrors the Trigger Action automated→confirmation fallback. + if (!suggestedRecord || ambiguous) { const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); return this.persistAwaitInput(target, sourceSchema, { availableRecordIds, - suggestNoRecord: true, + suggestedRecord, + suggestNoRecord: !suggestedRecord, }); } @@ -479,10 +553,15 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor, limit: number, - opts: { rank: boolean; allowNone: boolean } = { rank: true, allowNone: false }, + // allowNone is meaningful only when ranking, so it's absent from the rank:false variant. + opts: { rank: true; allowNone: boolean } | { rank: false } = { rank: true, allowNone: false }, ): Promise<{ relatedData: RecordData[]; bestIndex: number; + // Whether the suggested record is a confident pick. The deterministic/short-circuit paths + // (single candidate, no ranking) are confident by construction; the ranked path reflects the + // AI's own confidence flag. + confident: boolean; suggestedFields: string[]; relatedSchema: CollectionSchema; }> { @@ -492,27 +571,40 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor + this.selectRelevantFields(relatedSchema, this.context.stepDefinition.prompt), ); - const bestIndex = await this.selectBestRecordIndex( - relatedData, - suggestedFields, - this.context.stepDefinition.prompt, - opts.allowNone, + const { index: bestIndex, confident } = await this.withAiAssist(() => + this.selectBestRecordIndex( + relatedData, + suggestedFields, + this.context.stepDefinition.prompt, + opts.allowNone, + ), ); - return { relatedData, bestIndex, suggestedFields, relatedSchema }; + return { relatedData, bestIndex, confident, suggestedFields, relatedSchema }; + } + + // Tags any failure raised inside an AI call so the step handlers can degrade to the deterministic + // Manual path. Passing through an already-tagged error avoids double-wrapping when calls nest. + private async withAiAssist(call: () => Promise): Promise { + try { + return await call(); + } catch (error) { + if (error instanceof AiAssistUnavailableError) throw error; + throw new AiAssistUnavailableError(error); + } } private async fetchRelatedData( @@ -661,7 +753,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { + ): Promise<{ index: number; confident: boolean }> { const filteredCandidates = candidates.map((c, i) => { const entries = Object.entries(c.values).filter( ([k]) => fieldNames.length === 0 || fieldNames.includes(k), @@ -716,6 +808,12 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor( - messages, - tool, - ); + const { recordIndex, confident: rawConfident } = await this.invokeWithTool<{ + recordIndex: number; + confident?: boolean; + reasoning: string; + }>(messages, tool); // NOTE: The Zod schema's .min().max() shapes the tool prompt only — it is NOT validated against // the AI response. This guard is the sole runtime enforcement. -1 (none relevant) is accepted @@ -743,6 +842,11 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { expect(bindTools).not.toHaveBeenCalled(); }); - it('returns error outcome when AI selects an out-of-range record index', async () => { + it('degrades to manual selection when the AI returns an out-of-range record index', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -835,14 +835,16 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI made an unexpected choice. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + // An out-of-range index is an AI failure → degrade to the Manual path (deterministic list, + // user picks) rather than erroring the run or auto-loading a guess. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); }); - it('returns error when AI returns empty fieldNames violating the min:1 constraint', async () => { + it('degrades to manual selection when the AI returns empty fieldNames (min:1 violation)', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -884,11 +886,12 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI made an unexpected choice. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + // An invalid field-selection response is an AI failure → degrade to the Manual path. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); }); }); @@ -1035,7 +1038,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('executionType=Manual: no AI pre-selection (PRD-148)', () => { + describe('executionType=Manual: no AI pre-selection', () => { const customersWithAddress = makeCollectionSchema({ fields: [ { @@ -1163,7 +1166,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('executionType=AutomatedWithConfirmation: AI may decline (PRD-148)', () => { + describe('executionType=AutomatedWithConfirmation: AI may decline', () => { it('pre-checks "No X to load" (suggestNoRecord) when the AI judges no candidate relevant', async () => { const customersWithAddress = makeCollectionSchema({ fields: [ @@ -1290,7 +1293,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); - describe('executionType=FullyAutomated: AI judges none relevant → confirmation (PRD-148)', () => { + describe('executionType=FullyAutomated: AI judges none relevant → confirmation', () => { it('falls back to awaiting-input (No X to load) when select-record-by-content returns -1', async () => { const customersWithAddress = makeCollectionSchema({ fields: [ @@ -1358,6 +1361,201 @@ describe('LoadRelatedRecordStepExecutor', () => { }); }); + describe('executionType=FullyAutomated: low-confidence match → confirmation', () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + + it('degrades to awaiting-input with the best guess pre-selected when the AI is not confident', async () => { + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { + recordIndex: 0, + confident: false, + reasoning: 'Paris and Lyon both plausible', + }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + // Full AI can't confidently single one out → hand off to a human with the AI's best guess + // (index 0 → recordId [1]) pre-selected, rather than auto-loading an arbitrary pick. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toEqual(cand([1])); + // Not a "none relevant" outcome — the "No X to load" box must stay unchecked. + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); + // Nothing was auto-loaded. + expect(saved.executionResult).toBeUndefined(); + }); + + it('auto-loads (unchanged) when the AI is confident about its pick', async () => { + const agentPort = makeMockAgentPort(relatedData); + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 1, confident: true, reasoning: 'Lyon clearly matches' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('success'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.executionResult).toEqual( + expect.objectContaining({ + record: expect.objectContaining({ collectionName: 'addresses', recordId: [2] }), + }), + ); + }); + }); + + describe('AI failure/timeout degrades to Manual (does not error the run)', () => { + it('Full AI: an AI invoke that throws degrades to awaiting-input with the candidate list', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const relatedData: RecordData[] = [ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]; + const agentPort = makeMockAgentPort(relatedData); + // AI provider outage/timeout: the model invoke rejects. + const invoke = jest.fn().mockRejectedValue(new Error('AI provider timed out')); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + // Deterministic list presented for the user to pick — no error, no auto-skip, no suggestion. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBeUndefined(); + expect(saved.executionResult).toBeUndefined(); + }); + + it('AI-assisted: an AI invoke that throws degrades to awaiting-input (no error)', async () => { + const agentPort = makeMockAgentPort(); + const invoke = jest.fn().mockRejectedValue(new Error('AI provider unavailable')); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + // Default 2-relation schema → relation selection calls the AI (which throws) → degrade. + const context = makeContext({ + model, + agentPort, + runStore, + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + // Degrade picks the first eligible relation (Order) and pre-fills its single linked record. + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); + }); + }); + describe('operation activity log', () => { it('logs listRelatedData against the source record and its collection, not the trigger', async () => { const runStore = makeMockRunStore(); @@ -2212,6 +2410,46 @@ describe('LoadRelatedRecordStepExecutor', () => { ); }); + it('clears a stale suggestNoRecord flag when switching to a relation that yields a suggestion', async () => { + // Previous field ended in "No X to load" (suggestNoRecord true, no suggestion). Switching to a + // relation that resolves a record must drop the stale flag and surface the new suggestion — + // pendingData is rebuilt for the new field, not spread over the old one. + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'address', displayName: 'Address' }, + availableRecordIds: [cand([1]), cand([2])], + suggestedRecord: undefined, + suggestNoRecord: true, + }, + }); + const agentPort = makeMockAgentPort(); // default: order #99 via getSingleRelatedData + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const mockModel = makeMockModel({}); + const context = makeContext({ + model: mockModel.model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ customers: makeCollectionSchema() }), + incomingPendingData: { fieldName: 'order' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(finalSave.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(finalSave.pendingData.suggestedRecord).toEqual(cand(['99'])); + // The stale "No X to load" flag from the previous field must not linger. + expect(finalSave.pendingData.suggestNoRecord).toBeUndefined(); + }); + it('reruns xToOne candidate lookup when previewing a BelongsTo relation', async () => { // Same setup but switching to Order (BelongsTo). Verifies the xToOne path is // used inside refreshCandidatesForField — no AI calls, single candidate from @@ -2672,6 +2910,38 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(saved.pendingData.availableRecordIds).toHaveLength(50); }); + // Safety core (noneAllowed = allowNone && !truncated): when the candidate list is truncated the + // AI must NOT be offered the -1 "none relevant" option, so it can't decline based on a partial + // view — a match could be hiding in the unseen tail. + it('withholds the -1 (none) option from the AI when the candidate list is truncated', async () => { + const big = 'y'.repeat(80); + const values = Object.fromEntries(Array.from({ length: 6 }, (_, i) => [`f${i}`, big])); + const relatedData: RecordData[] = Array.from({ length: 50 }, (_, i) => ({ + collectionName: 'addresses', + recordId: [i + 1], + values, + })); + const { invoke, model } = buildModel(Array.from({ length: 6 }, (_, i) => `F${i}`)); + const context = makeContext({ + model, + logger: jest.fn(), + agentPort: makeMockAgentPort(relatedData), + runStore: makeMockRunStore(), + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: wideSchema(6), + }), + // AI-assisted so allowNone is true at the call site — the truncation guard is what suppresses + // the -1 option, not the mode. + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + await new LoadRelatedRecordStepExecutor(context).execute(); + + // Neither the decline guidance nor the "-1" index appears in the record-selection prompt. + expect(selectRecordPrompt(invoke)).not.toContain('-1'); + }); + // The AI sees only the budgeted prefix, but its index maps back into the FULL list — so a // non-zero pick must resolve to the correct full-list record (cap the prompt, not the data). it('maps a non-zero AI index back into the full candidate list after truncation', async () => { @@ -2962,9 +3232,10 @@ describe('LoadRelatedRecordStepExecutor', () => { }); describe('select-relation-to-follow failure', () => { - // With >=2 candidates the AI must echo back one of the offered labels. A label that - // matches no option (here a fabricated relation) is rejected as an invalid AI response. - it('returns error when AI selects a relation label not among the offered options', async () => { + // With >=2 candidates the AI must echo back one of the offered labels. A label that matches no + // option (here a fabricated relation) is an invalid AI response → the step degrades to the + // deterministic Manual path instead of erroring the run. + it('degrades to manual selection when AI selects a relation label not among the offered options', async () => { const agentPort = makeMockAgentPort(); // Default schema has two relations (Order, Address) → select-relation-to-follow IS invoked. const mockModel = makeMockModel({ @@ -2986,16 +3257,19 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI made an unexpected choice. Try rephrasing the step's prompt.", - ); - expect(agentPort.getRelatedData).not.toHaveBeenCalled(); + // Degrade picks the first eligible relation (Order, BelongsTo) deterministically and presents + // its single linked record for confirmation — no error, no auto-load off a bad relation pick. + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); }); describe('AI malformed/missing tool call', () => { - it('returns error on malformed tool call', async () => { + // A malformed or missing tool call is an AI failure. In an AI mode it degrades to the Manual + // path (first eligible relation + its candidate) rather than erroring the run. + it('degrades to manual selection on a malformed tool call', async () => { const invoke = jest.fn().mockResolvedValue({ tool_calls: [], invalid_tool_calls: [ @@ -3013,16 +3287,13 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.type).toBe('record'); - expect(result.stepOutcome.stepId).toBe('load-1'); - expect(result.stepOutcome.stepIndex).toBe(0); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI returned an unexpected response. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); - it('returns error when AI returns no tool call', async () => { + it('degrades to manual selection when AI returns no tool call', async () => { const invoke = jest.fn().mockResolvedValue({ tool_calls: [] }); const bindTools = jest.fn().mockReturnValue({ invoke }); const runStore = makeMockRunStore(); @@ -3035,13 +3306,10 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.type).toBe('record'); - expect(result.stepOutcome.stepId).toBe('load-1'); - expect(result.stepOutcome.stepIndex).toBe(0); - expect(result.stepOutcome.status).toBe('error'); - expect(result.stepOutcome.error).toBe( - "The AI couldn't decide what to do. Try rephrasing the step's prompt.", - ); - expect(runStore.saveStepExecution).not.toHaveBeenCalled(); + expect(result.stepOutcome.status).toBe('awaiting-input'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); + expect(saved.pendingData.suggestedRecord).toEqual(cand(['99'])); }); }); @@ -3938,7 +4206,7 @@ describe('LoadRelatedRecordStepExecutor', () => { it('errors (Full AI) when the source step loaded nothing — no longer auto-skips', async () => { // The source step exists in the live path but its run-store execution has no record. Full AI // now surfaces the error like the await modes (no silent skip); the front offers "continue - // without" (PRD-550). + // without". const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore, @@ -3957,7 +4225,7 @@ describe('LoadRelatedRecordStepExecutor', () => { it('surfaces a "no source record" error in await modes when the source step loaded nothing', async () => { // Same as Full AI now — every mode surfaces the error so the front can offer - // "continue without" (PRD-550). + // "continue without". const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore, From 71542cd52744b9674d0710263040d606cd9c250a Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Thu, 2 Jul 2026 15:57:58 +0000 Subject: [PATCH 08/14] test(workflow-executor): cover degrade and low-confidence branches Close the changed-lines coverage gap on the refreshCandidatesForField AI-failure catch: add a test where the ranking AI call throws during a field switch and the step degrades to the deterministic no-AI list (false branch), and one where a non-AI data-fetch failure is rethrown as a step error rather than degrading (true branch). Co-Authored-By: Claude Opus 4.8 --- .../load-related-record-step-executor.test.ts | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts index cc4cbda524..b1c0365b93 100644 --- a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts +++ b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts @@ -2450,6 +2450,109 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(finalSave.pendingData.suggestNoRecord).toBeUndefined(); }); + it('degrades to the no-AI candidate list when the AI fails during a field switch', async () => { + // AI mode + user switches to a HasMany relation; the ranking AI call throws → refresh must + // fall back to the deterministic list (no suggestion) instead of erroring the run. + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'order', displayName: 'Order' }, + availableRecordIds: [cand(['99'])], + suggestedRecord: cand(['99']), + }, + }); + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, + ]); + const addressesSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + // The ranking AI call rejects (provider outage/timeout). + const invoke = jest.fn().mockRejectedValue(new Error('AI provider timed out')); + const model = { + bindTools: jest.fn().mockReturnValue({ invoke }), + } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: addressesSchema, + }), + incomingPendingData: { fieldName: 'address' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(finalSave.pendingData.suggestedField).toEqual({ + name: 'address', + displayName: 'Address', + }); + expect(finalSave.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); + // Degraded → no AI suggestion and no stale "No X to load" flag. + expect(finalSave.pendingData.suggestedRecord).toBeUndefined(); + expect(finalSave.pendingData.suggestNoRecord).toBeUndefined(); + }); + + it('rethrows a non-AI failure during a field switch instead of degrading', async () => { + // A data-fetch failure (not an AI failure) must surface as a step error — only tagged AI + // failures degrade to the Manual path. + const execution = makePendingExecution({ + pendingData: { + availableFields: [ + { name: 'order', displayName: 'Order' }, + { name: 'address', displayName: 'Address' }, + ], + suggestedField: { name: 'order', displayName: 'Order' }, + availableRecordIds: [cand(['99'])], + suggestedRecord: cand(['99']), + }, + }); + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + ]); + (agentPort.getRelatedData as jest.Mock).mockRejectedValue( + new AgentPortError('getRelatedData', new Error('db down')), + ); + const addressesSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const mockModel = makeMockModel({}); + const runStore = makeMockRunStore({ + getStepExecutions: jest.fn().mockResolvedValue([execution]), + }); + const context = makeContext({ + model: mockModel.model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: makeCollectionSchema(), + addresses: addressesSchema, + }), + incomingPendingData: { fieldName: 'address' }, + stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('error'); + }); + it('reruns xToOne candidate lookup when previewing a BelongsTo relation', async () => { // Same setup but switching to Order (BelongsTo). Verifies the xToOne path is // used inside refreshCandidatesForField — no AI calls, single candidate from From 44e23be614a32627244876588891afcfe9be0b59 Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Fri, 3 Jul 2026 12:22:34 +0000 Subject: [PATCH 09/14] refactor(workflow-executor): tidy load-related-record comment and test name Co-Authored-By: Claude Opus 4.8 --- .../src/executors/load-related-record-step-executor.ts | 8 ++------ .../executors/load-related-record-step-executor.test.ts | 5 ++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index c011f4e67d..82592ebd02 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -451,12 +451,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { expect(result.stepOutcome.error).toBe('The pre-configured step parameters are invalid'); }); - it('errors (Full AI) when the source step loaded nothing — no longer auto-skips', async () => { + it('errors (Full AI) when the source step loaded no record', async () => { // The source step exists in the live path but its run-store execution has no record. Full AI - // now surfaces the error like the await modes (no silent skip); the front offers "continue - // without". + // surfaces the error like the await modes; the front offers "continue without". const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore, From 08c1949213064b7a1219a124cceb5b218bad7f8d Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Fri, 3 Jul 2026 12:26:48 +0000 Subject: [PATCH 10/14] refactor(workflow-executor): annotate max-classes-per-file eslint-disable Co-Authored-By: Claude Opus 4.8 --- .../src/executors/load-related-record-step-executor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 82592ebd02..22d560c8eb 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -1,4 +1,4 @@ -/* eslint-disable max-classes-per-file */ +/* eslint-disable max-classes-per-file -- private AI-degrade control-flow marker, not a public domain error */ import type { StepExecutionResult } from '../types/execution-context'; import type { LoadRelatedRecordCandidate, From 9fcdda28ee4e993821a3d21bd9eedc56b16f9a1c Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Fri, 3 Jul 2026 12:46:38 +0000 Subject: [PATCH 11/14] docs(workflow-executor): trim comments to why-only Co-Authored-By: Claude Opus 4.8 --- .../load-related-record-step-executor.ts | 41 ++++------ .../load-related-record-step-executor.test.ts | 80 +++++++------------ 2 files changed, 46 insertions(+), 75 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 22d560c8eb..04494ad294 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -82,9 +82,8 @@ function isFollowableRelation(field: FieldSchema): boolean { return field.isRelationship && Boolean(field.relatedCollectionName || field.polymorphicTypeField); } -// Internal marker: wraps any failure raised while calling the AI (timeout, provider outage, -// missing/malformed tool call, out-of-range choice) so the step handlers can degrade to the -// deterministic Manual path instead of erroring the run. Never leaves this module. +// Marks any AI-call failure (timeout, outage, malformed/missing tool call, out-of-range pick) so +// handlers degrade to the deterministic Manual path instead of erroring — never auto-skips. Internal only. class AiAssistUnavailableError extends Error { readonly reason: unknown; @@ -130,8 +129,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const target = await this.resolveTarget(false); const sourceSchema = await this.getCollectionSchema(target.selectedRecordRef.collectionName); @@ -234,9 +230,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor 1 && useAi ? await this.withAiAssist(() => this.selectRelationToFollow(eligible)) @@ -554,9 +549,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor this.selectRelevantFields(relatedSchema, this.context.stepDefinition.prompt), ); @@ -592,8 +585,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor(call: () => Promise): Promise { try { return await call(); @@ -829,9 +822,8 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor(messages, tool); - // NOTE: The Zod schema's .min().max() shapes the tool prompt only — it is NOT validated against - // the AI response. This guard is the sole runtime enforcement. -1 (none relevant) is accepted - // only when noneAllowed (allowNone and the list was not truncated). + // The Zod .min().max() shapes the tool prompt only — NOT validated against the AI response; this + // guard is the sole runtime enforcement. -1 is accepted only when noneAllowed (allowNone && !truncated). if (!Number.isInteger(recordIndex) || recordIndex < minIndex || recordIndex > maxIndex) { throw new InvalidAIResponseError( `AI selected record index ${recordIndex} which is out of range (${minIndex}-${maxIndex}) or not an integer`, @@ -839,8 +831,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { const result = await executor.execute(); - // An out-of-range index is an AI failure → degrade to the Manual path (deterministic list, - // user picks) rather than erroring the run or auto-loading a guess. + // Out-of-range index = AI failure → degrade to Manual (see AiAssistUnavailableError), not an error/auto-load. expect(result.stepOutcome.status).toBe('awaiting-input'); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); @@ -886,7 +885,7 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // An invalid field-selection response is an AI failure → degrade to the Manual path. + // Invalid field-selection response = AI failure → degrade to Manual (see AiAssistUnavailableError). expect(result.stepOutcome.status).toBe('awaiting-input'); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); @@ -978,8 +977,8 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.status).toBe('success'); - // To-many path: /relationships call with the candidate-list limit (a lone record short-circuits - // ranking, so it loads without an AI call). + // To-many path uses the candidate-list limit (50); a lone record short-circuits ranking, so it + // loads with no AI call. expect(agentPort.getRelatedData).toHaveBeenCalledWith( expect.objectContaining({ collection: 'customers', @@ -1001,8 +1000,7 @@ describe('LoadRelatedRecordStepExecutor', () => { ); }); - // Empty list → nothing to load: Full AI degrades to an AI-assisted confirmation ("No X to load" - // pre-checked) instead of skipping, so a human decides. + // Empty list: Full AI hands off with "No X to load" pre-checked rather than auto-skipping. it('falls back to awaiting-input (No X to load) when getRelatedData returns an empty array (Branch B)', async () => { const belongsToManySchema = makeCollectionSchema({ fields: [ @@ -1029,7 +1027,6 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Full AI with no candidate → hands off to the user with "No X to load" pre-checked. expect(result.stepOutcome.status).toBe('awaiting-input'); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; expect(saved.pendingData.availableRecordIds).toEqual([]); @@ -1082,7 +1079,6 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); expect(result.stepOutcome.status).toBe('awaiting-input'); - // Manual mode performs zero AI calls — no relation chooser, no field/record ranking. expect(bindTools).not.toHaveBeenCalled(); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls[0][1]; @@ -1091,7 +1087,6 @@ describe('LoadRelatedRecordStepExecutor', () => { }); it('still pre-fills the single xToOne record in Manual (the only option, not an AI pick)', async () => { - // BelongsTo 'order' → one linked record; pre-filled even in Manual, with no AI call. const agentPort = makeMockAgentPort(); // default: 1 related order #99 via getSingleRelatedData const bindTools = jest.fn(); const model = { bindTools } as unknown as ExecutionContext['model']; @@ -1157,7 +1152,6 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); expect(result.stepOutcome.status).toBe('awaiting-input'); - // Manual: switching to a multi-record relation must neither call the AI nor pre-select. expect(bindTools).not.toHaveBeenCalled(); const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; @@ -1222,8 +1216,8 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); - // AI-assisted no longer forces a pick: the AI may return -1, and the user reviews with - // "No X to load" pre-checked (rather than a potentially misleading forced suggestion). + // AI-assisted lets the AI decline (-1): the user reviews with "No X to load" pre-checked rather + // than a misleading forced suggestion. expect(result.stepOutcome.status).toBe('awaiting-input'); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); @@ -1285,8 +1279,8 @@ describe('LoadRelatedRecordStepExecutor', () => { await new LoadRelatedRecordStepExecutor(context).execute(); - // 2nd AI call = record selection. Without the decline guidance the base prompt forces a pick, - // so an impossible request would return a weak match instead of -1 (Bug 2). + // 2nd AI call = record selection. Without the decline guidance the base prompt forces a pick, so + // an impossible request returns a weak match instead of -1. const recordSelectionMessages = invoke.mock.calls[1][0] as Array<{ content: unknown }>; const content = recordSelectionMessages.map(m => String(m.content)).join('\n'); expect(content).toContain('-1'); @@ -1350,8 +1344,8 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); expect(result.stepOutcome.status).toBe('awaiting-input'); - // Two AI calls run (field + record selection); the record call returns -1 → hand off to the - // user with the candidates listed and "No X to load" pre-checked. + // Field + record-selection AI calls both run; the record call returns -1 → hand off with "No X + // to load" pre-checked. expect(bindTools).toHaveBeenCalledTimes(2); expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; @@ -1422,15 +1416,14 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await new LoadRelatedRecordStepExecutor(context).execute(); - // Full AI can't confidently single one out → hand off to a human with the AI's best guess - // (index 0 → recordId [1]) pre-selected, rather than auto-loading an arbitrary pick. + // Full AI can't confidently single one out → hand off with the AI's best guess (index 0 → + // recordId [1]) pre-selected, not an auto-load. expect(result.stepOutcome.status).toBe('awaiting-input'); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; expect(saved.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); expect(saved.pendingData.suggestedRecord).toEqual(cand([1])); - // Not a "none relevant" outcome — the "No X to load" box must stay unchecked. + // A best guess, not "none relevant" — the "No X to load" box must stay unchecked. expect(saved.pendingData.suggestNoRecord).toBeUndefined(); - // Nothing was auto-loaded. expect(saved.executionResult).toBeUndefined(); }); @@ -1502,7 +1495,6 @@ describe('LoadRelatedRecordStepExecutor', () => { { collectionName: 'addresses', recordId: [2], values: { city: 'Lyon' } }, ]; const agentPort = makeMockAgentPort(relatedData); - // AI provider outage/timeout: the model invoke rejects. const invoke = jest.fn().mockRejectedValue(new Error('AI provider timed out')); const bindTools = jest.fn().mockReturnValue({ invoke }); const model = { bindTools } as unknown as ExecutionContext['model']; @@ -2412,8 +2404,7 @@ describe('LoadRelatedRecordStepExecutor', () => { it('clears a stale suggestNoRecord flag when switching to a relation that yields a suggestion', async () => { // Previous field ended in "No X to load" (suggestNoRecord true, no suggestion). Switching to a - // relation that resolves a record must drop the stale flag and surface the new suggestion — - // pendingData is rebuilt for the new field, not spread over the old one. + // relation that resolves a record must drop the stale flag — pendingData is rebuilt, not spread. const execution = makePendingExecution({ pendingData: { availableFields: [ @@ -2446,13 +2437,12 @@ describe('LoadRelatedRecordStepExecutor', () => { const finalSave = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; expect(finalSave.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); expect(finalSave.pendingData.suggestedRecord).toEqual(cand(['99'])); - // The stale "No X to load" flag from the previous field must not linger. expect(finalSave.pendingData.suggestNoRecord).toBeUndefined(); }); it('degrades to the no-AI candidate list when the AI fails during a field switch', async () => { - // AI mode + user switches to a HasMany relation; the ranking AI call throws → refresh must - // fall back to the deterministic list (no suggestion) instead of erroring the run. + // HasMany field switch; the ranking AI call throws → falls back to the deterministic list, no + // suggestion (degrade, not error). const execution = makePendingExecution({ pendingData: { availableFields: [ @@ -2473,7 +2463,6 @@ describe('LoadRelatedRecordStepExecutor', () => { collectionDisplayName: 'Addresses', fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], }); - // The ranking AI call rejects (provider outage/timeout). const invoke = jest.fn().mockRejectedValue(new Error('AI provider timed out')); const model = { bindTools: jest.fn().mockReturnValue({ invoke }), @@ -2502,14 +2491,13 @@ describe('LoadRelatedRecordStepExecutor', () => { displayName: 'Address', }); expect(finalSave.pendingData.availableRecordIds).toEqual([cand([1]), cand([2])]); - // Degraded → no AI suggestion and no stale "No X to load" flag. expect(finalSave.pendingData.suggestedRecord).toBeUndefined(); expect(finalSave.pendingData.suggestNoRecord).toBeUndefined(); }); it('rethrows a non-AI failure during a field switch instead of degrading', async () => { - // A data-fetch failure (not an AI failure) must surface as a step error — only tagged AI - // failures degrade to the Manual path. + // A data-fetch failure is not a tagged AI failure, so it surfaces as a step error instead of + // degrading. const execution = makePendingExecution({ pendingData: { availableFields: [ @@ -3013,9 +3001,8 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(saved.pendingData.availableRecordIds).toHaveLength(50); }); - // Safety core (noneAllowed = allowNone && !truncated): when the candidate list is truncated the - // AI must NOT be offered the -1 "none relevant" option, so it can't decline based on a partial - // view — a match could be hiding in the unseen tail. + // Safety core (noneAllowed = allowNone && !truncated): a truncated list must NOT offer the AI the + // -1 "none relevant" option, or it could decline while a match hides in the unseen tail. it('withholds the -1 (none) option from the AI when the candidate list is truncated', async () => { const big = 'y'.repeat(80); const values = Object.fromEntries(Array.from({ length: 6 }, (_, i) => [`f${i}`, big])); @@ -3034,14 +3021,13 @@ describe('LoadRelatedRecordStepExecutor', () => { customers: makeCollectionSchema(), addresses: wideSchema(6), }), - // AI-assisted so allowNone is true at the call site — the truncation guard is what suppresses - // the -1 option, not the mode. + // AI-assisted so allowNone is true at the call site — the truncation guard, not the mode, + // suppresses the -1 option. stepDefinition: makeStep({ executionType: StepExecutionMode.AutomatedWithConfirmation }), }); await new LoadRelatedRecordStepExecutor(context).execute(); - // Neither the decline guidance nor the "-1" index appears in the record-selection prompt. expect(selectRecordPrompt(invoke)).not.toContain('-1'); }); @@ -3243,7 +3229,6 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Full AI: linked record absent → hand off to the user with "No X to load" pre-checked. expect(result.stepOutcome.status).toBe('awaiting-input'); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; expect(saved.pendingData.availableRecordIds).toEqual([]); @@ -3276,7 +3261,6 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Full AI: empty candidate list → hand off to the user with "No X to load" pre-checked. expect(result.stepOutcome.status).toBe('awaiting-input'); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; expect(saved.pendingData.availableRecordIds).toEqual([]); @@ -3335,9 +3319,8 @@ describe('LoadRelatedRecordStepExecutor', () => { }); describe('select-relation-to-follow failure', () => { - // With >=2 candidates the AI must echo back one of the offered labels. A label that matches no - // option (here a fabricated relation) is an invalid AI response → the step degrades to the - // deterministic Manual path instead of erroring the run. + // With >=2 candidates the AI must echo back an offered label; a fabricated one is an invalid + // response → degrade to Manual (see AiAssistUnavailableError). it('degrades to manual selection when AI selects a relation label not among the offered options', async () => { const agentPort = makeMockAgentPort(); // Default schema has two relations (Order, Address) → select-relation-to-follow IS invoked. @@ -3360,8 +3343,7 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); - // Degrade picks the first eligible relation (Order, BelongsTo) deterministically and presents - // its single linked record for confirmation — no error, no auto-load off a bad relation pick. + // Degrade picks the first eligible relation (Order) and presents its linked record — no auto-load. expect(result.stepOutcome.status).toBe('awaiting-input'); const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; expect(saved.pendingData.suggestedField).toEqual({ name: 'order', displayName: 'Order' }); @@ -3370,8 +3352,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); describe('AI malformed/missing tool call', () => { - // A malformed or missing tool call is an AI failure. In an AI mode it degrades to the Manual - // path (first eligible relation + its candidate) rather than erroring the run. + // A malformed/missing tool call is an AI failure → degrade to Manual (see AiAssistUnavailableError). it('degrades to manual selection on a malformed tool call', async () => { const invoke = jest.fn().mockResolvedValue({ tool_calls: [], @@ -4307,8 +4288,8 @@ describe('LoadRelatedRecordStepExecutor', () => { }); it('errors (Full AI) when the source step loaded no record', async () => { - // The source step exists in the live path but its run-store execution has no record. Full AI - // surfaces the error like the await modes; the front offers "continue without". + // Source step exists but its run-store execution has no record. Full AI surfaces the error (the + // front offers "continue without"). const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore, @@ -4326,8 +4307,7 @@ describe('LoadRelatedRecordStepExecutor', () => { }); it('surfaces a "no source record" error in await modes when the source step loaded nothing', async () => { - // Same as Full AI now — every mode surfaces the error so the front can offer - // "continue without". + // Await modes surface the same error as Full AI so the front can offer "continue without". const runStore = makeMockRunStore({ getStepExecutions: jest.fn().mockResolvedValue([]) }); const context = makeContext({ runStore, From 04b2be8138299344b7ed67fc07387d71f61791d7 Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Mon, 6 Jul 2026 14:52:56 +0000 Subject: [PATCH 12/14] fix(workflow-executor): rank lone toMany candidate; narrow withAiAssist A HasMany/BelongsToMany relation filtered to one candidate was auto-loaded as confident without consulting the AI, so Full AI could load an irrelevant sole record instead of degrading to 'No X to load'. It now goes through AI ranking (allowNone) so the AI can decline (-1). withAiAssist in selectRelationToFollow now wraps only the AI call, not the run-store read, so a store failure surfaces instead of being mistagged and degraded to Manual. Co-Authored-By: Claude Opus 4.8 --- .../load-related-record-step-executor.ts | 40 ++-- .../load-related-record-step-executor.test.ts | 186 ++++++++++++++++-- 2 files changed, 184 insertions(+), 42 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 04494ad294..71be6c75aa 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -233,9 +233,7 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor 1 && useAi - ? await this.withAiAssist(() => this.selectRelationToFollow(eligible)) - : eligible[0]; + eligible.length > 1 && useAi ? await this.selectRelationToFollow(eligible) : eligible[0]; return this.targetFromCandidate(chosen); } @@ -558,13 +556,9 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor( - messages, - tool, - ); + // Wrap only the AI call + response validation: buildPreviousStepsMessages above reads the run + // store, and a store failure must surface, not be mistagged as an AI failure and degraded. + return this.withAiAssist(async () => { + const { relation } = await this.invokeWithTool<{ relation: string; reasoning: string }>( + messages, + tool, + ); - const index = labels.indexOf(relation); + const index = labels.indexOf(relation); - if (index === -1) { - throw new InvalidAIResponseError( - `AI selected relation "${relation}" which does not match any available option`, - ); - } + if (index === -1) { + throw new InvalidAIResponseError( + `AI selected relation "${relation}" which does not match any available option`, + ); + } - return candidates[index]; + return candidates[index]; + }); } /** AI call 1 for HasMany: selects the most relevant fields to compare candidates. */ diff --git a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts index 96b1a82fd7..0cbe1b0684 100644 --- a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts +++ b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts @@ -746,7 +746,7 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(bindTools.mock.calls[0][0][0].name).toBe('select-record-by-content'); }); - it('takes the single candidate directly without AI record-selection calls', async () => { + it('consults the AI for a lone candidate and loads it when the AI picks index 0', async () => { const hasManySchema = makeCollectionSchema({ fields: [ { @@ -761,15 +761,37 @@ describe('LoadRelatedRecordStepExecutor', () => { const agentPort = makeMockAgentPort([ { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, ]); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); - const invoke = jest.fn(); + // A lone candidate in an AI mode still runs the AI so it can be declined (-1); here the AI + // picks index 0. Call 1: select-fields → ['City']; Call 2: select-record-by-content → 0. + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 0, reasoning: 'Paris is relevant' }, + id: 'c3', + }, + ], + }); const bindTools = jest.fn().mockReturnValue({ invoke }); const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); const context = makeContext({ model, agentPort, - workflowPort: makeMockWorkflowPort({ customers: hasManySchema }), + runStore, + workflowPort: makeMockWorkflowPort({ customers: hasManySchema, addresses: addressSchema }), stepDefinition: makeStep({ executionType: StepExecutionMode.FullyAutomated }), }); const executor = new LoadRelatedRecordStepExecutor(context); @@ -777,9 +799,16 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.status).toBe('success'); - // Single relation (auto-picked, no select-relation-to-follow) AND single related - // candidate (no field/record AI calls) → no AI calls at all. - expect(bindTools).not.toHaveBeenCalled(); + expect(bindTools).toHaveBeenCalledTimes(2); + expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); + expect(runStore.saveStepExecution).toHaveBeenCalledWith( + 'run-1', + expect.objectContaining({ + executionResult: expect.objectContaining({ + record: expect.objectContaining({ collectionName: 'addresses', recordId: [1] }), + }), + }), + ); }); it('degrades to manual selection when the AI returns an out-of-range record index', async () => { @@ -962,14 +991,39 @@ describe('LoadRelatedRecordStepExecutor', () => { }, ], }); - const agentPort = makeMockAgentPort([{ collectionName: 'tags', recordId: [7], values: {} }]); - const mockModel = makeMockModel({ relationName: 'Tags', reasoning: 'Load tags' }); + const agentPort = makeMockAgentPort([ + { collectionName: 'tags', recordId: [7], values: { name: 'urgent' } }, + ]); + const tagsSchema = makeCollectionSchema({ + collectionName: 'tags', + collectionDisplayName: 'Tags', + fields: [{ fieldName: 'name', displayName: 'Name', isRelationship: false }], + }); + // A lone toMany candidate is still ranked by the AI (so Full AI can decline it); here it picks + // index 0. Call 1: select-fields → ['Name']; Call 2: select-record-by-content → 0. + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['Name'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 0, reasoning: 'The only tag is relevant' }, + id: 'c3', + }, + ], + }); + const model = { + bindTools: jest.fn().mockReturnValue({ invoke }), + } as unknown as ExecutionContext['model']; const runStore = makeMockRunStore(); const context = makeContext({ - model: mockModel.model, + model, agentPort, runStore, - workflowPort: makeMockWorkflowPort({ customers: belongsToManySchema }), + workflowPort: makeMockWorkflowPort({ customers: belongsToManySchema, tags: tagsSchema }), stepDefinition: makeStep({ executionType: StepExecutionMode.FullyAutomated }), }); const executor = new LoadRelatedRecordStepExecutor(context); @@ -977,8 +1031,7 @@ describe('LoadRelatedRecordStepExecutor', () => { const result = await executor.execute(); expect(result.stepOutcome.status).toBe('success'); - // To-many path uses the candidate-list limit (50); a lone record short-circuits ranking, so it - // loads with no AI call. + // To-many path uses the candidate-list limit (50) and ranks even a lone candidate via the AI. expect(agentPort.getRelatedData).toHaveBeenCalledWith( expect.objectContaining({ collection: 'customers', @@ -1353,6 +1406,70 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(saved.pendingData.suggestedRecord).toBeUndefined(); expect(saved.pendingData.suggestNoRecord).toBe(true); }); + + it('consults the AI for a lone candidate and hands off (No X to load) when it returns -1', async () => { + const customersWithAddress = makeCollectionSchema({ + fields: [ + { + fieldName: 'address', + displayName: 'Address', + isRelationship: true, + relationType: 'HasMany', + relatedCollectionName: 'addresses', + }, + ], + }); + const addressSchema = makeCollectionSchema({ + collectionName: 'addresses', + collectionDisplayName: 'Addresses', + fields: [{ fieldName: 'city', displayName: 'City', isRelationship: false }], + }); + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + ]); + // A lone candidate must not auto-load in Full AI: it still goes through the AI, which returns + // -1 to decline. Call 1: select-fields → ['City']; Call 2: select-record-by-content → -1. + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['City'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: -1, reasoning: 'The only candidate does not fit the request' }, + id: 'c3', + }, + ], + }); + const bindTools = jest.fn().mockReturnValue({ invoke }); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.FullyAutomated, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + expect(bindTools.mock.calls[1][0][0].name).toBe('select-record-by-content'); + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls.at(-1)?.[1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1])]); + expect(saved.pendingData.suggestedRecord).toBeUndefined(); + expect(saved.pendingData.suggestNoRecord).toBe(true); + expect(saved.executionResult).toBeUndefined(); + }); }); describe('executionType=FullyAutomated: low-confidence match → confirmation', () => { @@ -3797,16 +3914,43 @@ describe('LoadRelatedRecordStepExecutor', () => { collectionDisplayName: 'Addresses', }), }); - const mockModel = makeMockModel({ - relation: relationOption({ - recordId: [42], - relationDisplayName: 'Address', - relatedCollectionName: 'addresses', - }), - reasoning: 'Load address', - }); + // 3 AI calls (2 relations → follow; lone candidate → field + record ranking). The ranking + // reuses the already-fetched 'addresses' schema, so no extra getCollectionSchema call. + const invoke = jest + .fn() + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-relation-to-follow', + args: { + relation: relationOption({ + recordId: [42], + relationDisplayName: 'Address', + relatedCollectionName: 'addresses', + }), + reasoning: 'Load address', + }, + id: 'c1', + }, + ], + }) + .mockResolvedValueOnce({ + tool_calls: [{ name: 'select-fields', args: { fieldNames: ['Email'] }, id: 'c2' }], + }) + .mockResolvedValueOnce({ + tool_calls: [ + { + name: 'select-record-by-content', + args: { recordIndex: 0, reasoning: 'The only candidate is relevant' }, + id: 'c3', + }, + ], + }); + const model = { + bindTools: jest.fn().mockReturnValue({ invoke }), + } as unknown as ExecutionContext['model']; const context = makeContext({ - model: mockModel.model, + model, workflowPort, stepDefinition: makeStep({ executionType: StepExecutionMode.FullyAutomated }), }); From 53670e43bda8d7a6b295033f470b1f5f8e6fb360 Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Mon, 6 Jul 2026 15:10:15 +0000 Subject: [PATCH 13/14] refactor(workflow-executor): move AiAssistUnavailableError to errors.ts Matches house style (exported from errors.ts, kept out of the index barrel like AiInvokeTimeoutError so it stays internal and never HTTP-maps) and drops the file-level max-classes-per-file suppression. Co-Authored-By: Claude Opus 4.8 --- packages/workflow-executor/src/errors.ts | 11 +++++++++++ .../executors/load-related-record-step-executor.ts | 14 +------------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/packages/workflow-executor/src/errors.ts b/packages/workflow-executor/src/errors.ts index 530c97405c..10173ea78a 100644 --- a/packages/workflow-executor/src/errors.ts +++ b/packages/workflow-executor/src/errors.ts @@ -270,6 +270,17 @@ export class AiInvokeTimeoutError extends WorkflowExecutorError { } } +// Wraps any AI-call failure (timeout, outage, malformed/missing tool call, out-of-range pick) so +// handlers degrade to the Manual path. Caught internally — never HTTP-mapped; not exported from index. +export class AiAssistUnavailableError extends WorkflowExecutorError { + readonly reason: unknown; + + constructor(reason: unknown) { + super('AI assistance unavailable'); + this.reason = reason; + } +} + export class NoMcpToolsError extends WorkflowExecutorError { constructor(requestedMcpServerId: string) { super( diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index 71be6c75aa..d8fce2e142 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -1,4 +1,3 @@ -/* eslint-disable max-classes-per-file -- private AI-degrade control-flow marker, not a public domain error */ import type { StepExecutionResult } from '../types/execution-context'; import type { LoadRelatedRecordCandidate, @@ -17,6 +16,7 @@ import { DynamicStructuredTool, HumanMessage, SystemMessage } from '@forestadmin import { z } from 'zod'; import { + AiAssistUnavailableError, InvalidAIResponseError, InvalidPreRecordedArgsError, NoRelationshipFieldsError, @@ -82,18 +82,6 @@ function isFollowableRelation(field: FieldSchema): boolean { return field.isRelationship && Boolean(field.relatedCollectionName || field.polymorphicTypeField); } -// Marks any AI-call failure (timeout, outage, malformed/missing tool call, out-of-range pick) so -// handlers degrade to the deterministic Manual path instead of erroring — never auto-skips. Internal only. -class AiAssistUnavailableError extends Error { - readonly reason: unknown; - - constructor(reason: unknown) { - super('AI assistance unavailable'); - this.name = 'AiAssistUnavailableError'; - this.reason = reason; - } -} - export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor { protected async doExecute(): Promise { // Branch A -- Re-entry after pending execution found in RunStore From 2f50ee32ccc9439a39476f9701414bc449393507 Mon Sep 17 00:00:00 2001 From: Brian Fox Date: Mon, 6 Jul 2026 15:14:23 +0000 Subject: [PATCH 14/14] fix(workflow-executor): keep Manual prefill for a sole toMany candidate The lone-toMany ranking fix also dropped Manual's single-candidate pre-selection (bestIndex -1), regressing prefill and diverging from xToOne Manual. The !opts.rank branch now returns bestIndex 0 only when there is exactly one candidate; with several the user still picks. Co-Authored-By: Claude Opus 4.8 --- .../load-related-record-step-executor.ts | 20 ++++++++++-- .../load-related-record-step-executor.test.ts | 31 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts index d8fce2e142..2f8483bcdd 100644 --- a/packages/workflow-executor/src/executors/load-related-record-step-executor.ts +++ b/packages/workflow-executor/src/executors/load-related-record-step-executor.ts @@ -544,12 +544,26 @@ export default class LoadRelatedRecordStepExecutor extends RecordStepExecutor diff --git a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts index 0cbe1b0684..0e4ba9b9c1 100644 --- a/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts +++ b/packages/workflow-executor/test/executors/load-related-record-step-executor.test.ts @@ -1139,6 +1139,37 @@ describe('LoadRelatedRecordStepExecutor', () => { expect(saved.pendingData.suggestedRecord).toBeUndefined(); }); + it('pre-fills a sole toMany candidate in Manual (the only option, not an AI pick)', async () => { + const agentPort = makeMockAgentPort([ + { collectionName: 'addresses', recordId: [1], values: { city: 'Paris' } }, + ]); + const bindTools = jest.fn(); + const model = { bindTools } as unknown as ExecutionContext['model']; + const runStore = makeMockRunStore(); + const context = makeContext({ + model, + agentPort, + runStore, + workflowPort: makeMockWorkflowPort({ + customers: customersWithAddress, + addresses: addressSchema, + }), + stepDefinition: makeStep({ + executionType: StepExecutionMode.Manual, + preRecordedArgs: { relationName: 'address' }, + }), + }); + + const result = await new LoadRelatedRecordStepExecutor(context).execute(); + + expect(result.stepOutcome.status).toBe('awaiting-input'); + expect(bindTools).not.toHaveBeenCalled(); + + const saved = (runStore.saveStepExecution as jest.Mock).mock.calls[0][1]; + expect(saved.pendingData.availableRecordIds).toEqual([cand([1])]); + expect(saved.pendingData.suggestedRecord).toEqual(cand([1])); + }); + it('still pre-fills the single xToOne record in Manual (the only option, not an AI pick)', async () => { const agentPort = makeMockAgentPort(); // default: 1 related order #99 via getSingleRelatedData const bindTools = jest.fn();