Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 63 additions & 4 deletions packages/contracts/src/replay-backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,30 @@ import type { DaemonError } from '@agent-device/kernel/errors';
import type { DaemonResponse, SessionRuntimeHints } from '@agent-device/kernel/contracts';
import type { DeviceInfo } from '@agent-device/kernel/device';
import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot';
import type { CommandFlags } from './command-flags.ts';
import type { DaemonWireRequest } from './daemon-wire-request.ts';
import type {
ReplayDivergenceResume,
ReplayDivergenceSuggestionBasis,
} from './replay-divergence.ts';
import type { ReplayDispatchOptions, ReplayObservationAuthorityBinder } from './replay.ts';
import type {
ReplayDispatchOptions,
ReplayObservationAuthorityBinder,
ReplayScriptSourceBundle,
} from './replay.ts';
import type { SessionAction } from './session-action.ts';
import type {
SnapshotDiagnosticsState,
SnapshotDiagnosticsSummary,
} from './snapshot-diagnostics.ts';

/**
* The replay backend contract (#3377): the session and command shapes a non-native replay engine
* is handed, the capabilities it implements, and the failure descriptor the daemon reports for a
* failed step. Everything here is type-only, so an engine can depend on this module without
* The replay backend plugin contract (#3377): what a non-native replay engine implements
* (`ReplayBackendPlugin`) and the host services the daemon lends it (`ReplayBackendHost`).
*
* The replay port owns the registry and builds the host; an engine package owns its plugin and
* reaches process state, session policy, and daemon dispatch only through `host` and the command
* it is handed. Everything here is type-only, so an engine can depend on this module without
* evaluating any host code.
*/

Expand Down Expand Up @@ -205,3 +213,54 @@ export type ReplayBackendActionFailure = Readonly<{
snapshot: Readonly<{ createdAt: number; nodes: SnapshotNode[] }>,
): readonly Readonly<{ node: SnapshotNode; basis: ReplayDivergenceSuggestionBasis }>[];
}>;

/** The device-binding facts a backend resolves a target device from. */
export type ReplayBackendDeviceBinding = Readonly<{
/** The flow's static app target, when it declares one. */
appTarget: string | undefined;
platform: CommandFlags['platform'] | undefined;
}>;

/**
* The host services a backend plugin reaches. Everything holding process state or daemon
* session policy is a member; stateless workspace packages stay direct imports of the plugin.
*/
export type ReplayBackendHost = Readonly<{
/** Appends one record to the run's trace file, if any; failures are diagnostics, never thrown. */
appendTraceEvent(tracePath: string | undefined, event: Record<string, unknown>): void;
/** The `AD_*` variables every replay run is seeded with. */
buildBuiltinVars(params: {
req: DaemonWireRequest;
sessionName: string;
resolvedPath: string;
}): Record<string, string>;
/** Reads one file out of the request's script source bundle; throws when it was not sent. */
readScriptSourceFile(bundle: ReplayScriptSourceBundle, resolvedPath: string): string;
scriptSourceRequiredMessage: string;
/**
* Selects the device a run binds when no session or explicit platform already names one. It
* applies the Maestro device-binding rule (the flow's app target steers selection on iOS only),
* the same rule `packages/daemon/src/server/replay-device-selection.ts` applies when it
* pre-binds a device for the request.
*/
resolveTargetDevice(
flags: CommandFlags,
binding: ReplayBackendDeviceBinding,
): Promise<DeviceInfo>;
/** The abort signal of the in-flight daemon request. */
requestSignal(requestId: string | undefined): AbortSignal | undefined;
/** Builds the REPLAY_DIVERGENCE failure response: screen capture, ranking, bounding, resume. */
buildActionFailureResponse(failure: ReplayBackendActionFailure): Promise<DaemonResponse>;
}>;

/**
* Engine-free metadata, importable without evaluating the engine, so help stays light.
*/
export type ReplayBackendDeclaration<TId extends string = string> = Readonly<{
id: TId;
help: Readonly<{ summary: string; body: string }>;
}>;

export type ReplayBackendPlugin<TId extends string = string> = Readonly<{
create(host: ReplayBackendHost): ReplayBackend<TId>;
}>;
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import fs from 'node:fs';
import path from 'node:path';
import { expect, test } from 'vitest';
import { stringify } from 'yaml';
import { runTypedMaestroReplay } from '@agent-device/replay-port/session-replay-maestro-runtime';
import { getReplayBackend } from '@agent-device/replay-port/replay-backend-registry';
import { SessionStore } from '../../../core/session-store.ts';
import { replayCommandForTest } from '../replay-runtime/replay-command-fixture.ts';
import { makeIosSession } from '../../../__tests__/support/session-factories.ts';
Expand Down Expand Up @@ -35,7 +35,9 @@ test('a typed Maestro replay error keeps its recovery hint', async () => {
meta: { requestId: 'req-maestro-hint' },
} as unknown as DaemonRequest;

const response = await runTypedMaestroReplay(
const response = await (
await getReplayBackend('maestro')
).runReplay(
replayCommandForTest({
req,
sessionName: 'default',
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { noMaestroIncludeSources } from '@agent-device/replay-port/replay-script-source-fixtures';
import { beforeEach, expect, test, vi } from 'vitest';
import { mkdtempForTestSync } from '@agent-device/test-kit/tmp-dir';

Expand All @@ -13,23 +12,12 @@ vi.mock('../../../core/snapshot-interactor-capture.ts', () => ({
import fs from 'node:fs';
import path from 'node:path';
import { stringify } from 'yaml';
import {
executeMaestroFlow,
inspectMaestroFlow,
type MaestroFailedAction,
type MaestroRuntimeCommand,
type MaestroRuntimePort,
} from '@agent-device/maestro';
import {
buildTypedMaestroFailureReportProjection,
buildTypedMaestroFailureResponse,
} from '@agent-device/replay-port/session-replay-maestro-failure';
import type { MaestroRuntimeCommand } from '@agent-device/maestro';
import { runReplayForTest } from '../replay-runtime/replay-command-fixture.ts';
import { SessionStore } from '../../../core/session-store.ts';
import { captureSnapshotWithInteractor } from '../../../core/snapshot-interactor-capture.ts';
import { makeIosSession } from '../../../__tests__/support/session-factories.ts';
import { baseReplayRequest as baseReq } from '../replay-runtime/session-replay-runtime.fixtures.ts';
import { replaySessionForTest } from '../replay-divergence/replay-session-fixture.ts';
import type { SnapshotNode } from '@agent-device/kernel/snapshot';
import {
captureSnapshotThroughLegacyDispatchFixture,
Expand All @@ -47,30 +35,35 @@ beforeEach(() => {
mockCaptureSnapshotWithInteractor.mockImplementation(captureSnapshotThroughLegacyDispatchFixture);
});

/**
* Runs a one-step flow whose action fails, so the divergence report is built the way the daemon
* builds it: `nodes` is the screen captured after the failure, `engineNodes` what the flow's own
* target lookup sees before it.
*/
async function buildFailureScenario(
command: MaestroRuntimeCommand,
nodes: SnapshotNode[],
options: { warnings?: readonly string[] } = {},
): Promise<{
response: Extract<Awaited<ReturnType<typeof buildTypedMaestroFailureResponse>>, { ok: false }>;
sessionStore: SessionStore;
sessionName: string;
}> {
options: { engineNodes?: SnapshotNode[] } = {},
) {
const root = mkdtempForTestSync('agent-device-maestro-suggestions-');
const sessionName = 'default';
const sessionStore = new SessionStore(path.join(root, 'sessions'));
sessionStore.publish(sessionName, makeIosSession(sessionName));
const replaySession = replaySessionForTest(sessionStore, sessionName);
const flowPath = path.join(root, 'flow.yaml');
fs.writeFileSync(flowPath, maestroFlowForCommand(command));
mockDispatchCommand.mockResolvedValue({ nodes, truncated: false, backend: 'xctest' });
const failure = await captureMaestroFailure(command, path.join(root, 'flow.yaml'));
const response = await buildTypedMaestroFailureResponse({
error: { code: 'COMMAND_FAILED', message: 'typed Maestro action failed' },
failure: options.warnings ? { ...failure, warnings: options.warnings } : failure,
replayPath: path.join(root, 'flow.yaml'),
req: baseReq({ flags: { replayBackend: 'maestro', platform: 'ios' } }),
sessionStore: replaySession.store,
observationStore: replaySession.observationStore,
const response = await runReplayForTest({
req: baseReq({
positionals: [flowPath],
flags: { replayBackend: 'maestro', platform: 'ios' },
}),
sessionName,
logPath: path.join(root, 'daemon.log'),
sessionStore,
invoke: async (req) =>
req.command === 'snapshot'
? { ok: true, data: { nodes: options.engineNodes ?? nodes } }
: { ok: false, error: { code: 'COMMAND_FAILED', message: 'typed Maestro action failed' } },
});
if (response.ok) throw new Error('expected typed Maestro failure response');
return { response, sessionStore, sessionName };
Expand All @@ -79,85 +72,11 @@ async function buildFailureScenario(
async function buildFailureResponse(
command: MaestroRuntimeCommand,
nodes: SnapshotNode[],
options: { warnings?: readonly string[] } = {},
): Promise<Extract<Awaited<ReturnType<typeof buildTypedMaestroFailureResponse>>, { ok: false }>> {
options: { engineNodes?: SnapshotNode[] } = {},
) {
return (await buildFailureScenario(command, nodes, options)).response;
}

test('typed Maestro failure response carries warnings accumulated before the failing step (#2560)', async () => {
const response = await buildFailureResponse(
{
kind: 'tapOn' as const,
source: { path: '/flows/login.yaml', line: 4 },
target: { space: 'target' as const, selector: { id: 'save' } },
},
[],
{ warnings: ['Optional Maestro assertVisible skipped at line 2: no match'] },
);

if (!response.error.details) throw new Error('expected error details on the failure response');
const details = response.error.details;
expect(details.warnings).toEqual(['Optional Maestro assertVisible skipped at line 2: no match']);
expect((details.divergence as { kind: string }).kind).toBe('action-failure');
});

test('typed Maestro failure response omits an empty warnings channel', async () => {
const response = await buildFailureResponse(
{
kind: 'tapOn' as const,
source: { path: '/flows/login.yaml', line: 4 },
target: { space: 'target' as const, selector: { id: 'save' } },
},
[],
{ warnings: [] },
);

expect('warnings' in (response.error.details ?? {})).toBe(false);
});

test('typed Maestro failure projection keeps action and source provenance', async () => {
const command = {
kind: 'tapOn' as const,
source: { path: '/flows/login.yaml', line: 4 },
target: { space: 'target' as const, selector: { id: 'save' } },
};
const request = baseReq({ flags: { replayBackend: 'maestro' } });
const failure = await captureMaestroFailure(command, command.source.path);
const projection = buildTypedMaestroFailureReportProjection(failure, request);

expect(projection.command).toEqual({ kind: 'tapOn' });
expect(projection.source).toEqual({ path: command.source.path, line: 3 });
expect(projection.progress).toEqual({ command: 'tapOn', value: 'save' });
expect(projection.action).toEqual({
command: 'click',
positionals: ['save'],
flags: request.flags,
});
expect(Object.keys(projection.action)).toEqual(['command', 'positionals', 'flags']);
});

async function captureMaestroFailure(
command: MaestroRuntimeCommand,
sourcePath: string,
): Promise<MaestroFailedAction> {
const flow = inspectMaestroFlow(maestroFlowForCommand(command), sourcePath);
const error = new Error('typed Maestro action failed');
const port: MaestroRuntimePort = {
execute: async () => {
throw error;
},
observe: async () => {
throw error;
},
};
const outcome = await executeMaestroFlow(flow, port, {
platform: 'ios',
readSource: noMaestroIncludeSources,
});
if (outcome.ok || !outcome.failure) throw new Error('expected typed Maestro failure');
return outcome.failure;
}

function maestroFlowForCommand(command: MaestroRuntimeCommand): string {
const authored =
command.kind === 'inputText'
Expand Down Expand Up @@ -378,7 +297,7 @@ test('typed Maestro failure publishes exactly the refs exposed by its divergence
source: { path: '/flows/actions.yaml', line: 4 },
target: { space: 'target' as const, selector: { text: 'Missing' } },
} satisfies Extract<MaestroRuntimeCommand, { kind: 'tapOn' }>;
const scenario = await buildFailureScenario(command, [
const screen: SnapshotNode[] = [
{
ref: 'e1',
index: 0,
Expand All @@ -394,7 +313,20 @@ test('typed Maestro failure publishes exactly the refs exposed by its divergence
rect: { x: 16, y: 40, width: 140, height: 44 },
hittable: true,
},
]);
];
const scenario = await buildFailureScenario(command, screen, {
engineNodes: [
...screen,
{
index: 2,
parentIndex: 0,
label: 'Missing',
type: 'Button',
rect: { x: 16, y: 96, width: 140, height: 44 },
hittable: true,
},
],
});
const divergence = scenario.response.error.details?.divergence as {
screen: { state: string; refs: Array<{ ref: string }> };
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import fs from 'node:fs';
import path from 'node:path';
import { expect, test } from 'vitest';
import { stringify } from 'yaml';
import { runTypedMaestroReplay } from '@agent-device/replay-port/session-replay-maestro-runtime';
import { getReplayBackend } from '@agent-device/replay-port/replay-backend-registry';
import { SessionStore } from '../../../core/session-store.ts';
import { replayCommandForTest } from '../replay-runtime/replay-command-fixture.ts';
import { makeIosSession } from '../../../__tests__/support/session-factories.ts';
Expand Down Expand Up @@ -37,7 +37,9 @@ test('a typed Maestro selector conflict names the session store key, not "defaul
meta: { requestId: 'req-maestro-address' },
} as unknown as DaemonRequest;

const response = await runTypedMaestroReplay(
const response = await (
await getReplayBackend('maestro')
).runReplay(
replayCommandForTest({
req,
sessionName: SCOPED_ADDRESS,
Expand Down
11 changes: 8 additions & 3 deletions packages/maestro/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"type": "module",
"description": "Private Maestro compatibility engine for agent-device.",
"dependencies": {
"@agent-device/ad-script": "workspace:*",
"@agent-device/capture-kit": "workspace:*",
"@agent-device/contracts": "workspace:*",
"@agent-device/host-kit": "workspace:*",
Expand All @@ -18,9 +19,13 @@
"types": "./src/index.ts",
"default": "./src/index.ts"
},
"./daemon-runtime-port": {
"types": "./src/daemon-port/daemon-runtime-port.ts",
"default": "./src/daemon-port/daemon-runtime-port.ts"
"./replay-backend": {
"types": "./src/replay-backend/replay-maestro-backend.ts",
"default": "./src/replay-backend/replay-maestro-backend.ts"
},
"./replay-backend-declaration": {
"types": "./src/replay-backend/replay-maestro-declaration.ts",
"default": "./src/replay-backend/replay-maestro-declaration.ts"
},
"./run-script-http": {
"types": "./src/daemon-port/run-script-http.ts",
Expand Down
13 changes: 8 additions & 5 deletions packages/maestro/src/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,19 @@ describe('@agent-device/maestro facade', () => {
expect(exports).not.toContain('rankMaestroFailureCandidates');
});

test('keeps the daemon-side port off the engine entry and declares pure modules', () => {
test('keeps the plugin and helper off the engine entry and declares pure modules', () => {
const manifest = JSON.parse(
fs.readFileSync(path.join(import.meta.dirname, '../package.json'), 'utf8'),
) as { exports: Record<string, unknown>; sideEffects?: boolean };
// The daemon-side runtime port (#2544) and its runScript HTTP helper are subpaths, not part
// of the engine entry. This pins the surface; scripts/__tests__/eager-closure-budgets.test.ts
// is what keeps `.` from starting to evaluate host-kit, capture-kit, or provision-kit.
// The runScript HTTP helper is a subpath, not part of the engine entry, and the replay
// backend plugin (#3377) is the only other way in: its engine-free declaration lets help list
// the backend without evaluating the engine. This pins the surface;
// scripts/__tests__/eager-closure-budgets.test.ts is what keeps `.` from starting to evaluate
// host-kit, capture-kit, or provision-kit.
expect(Object.keys(manifest.exports)).toEqual([
'.',
'./daemon-runtime-port',
'./replay-backend',
'./replay-backend-declaration',
'./run-script-http',
]);
expect(manifest.sideEffects).toBe(false);
Expand Down
3 changes: 0 additions & 3 deletions packages/maestro/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@ export {
export { MAESTRO_SELECTOR_PROJECTION } from './internal/selector-vocabulary.ts';

export {
formatMaestroCompatibilityReference,
MAESTRO_COMPATIBILITY_ADR_URL,
MAESTRO_COMPATIBILITY_ISSUE_URL,
MAESTRO_COMPAT_LIMITATIONS,
MAESTRO_COMPAT_SUPPORTED_CAPABILITIES,
} from './internal/facade-support.ts';
Expand Down
3 changes: 0 additions & 3 deletions packages/maestro/src/internal/facade-support.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
export {
formatMaestroCompatibilityReference,
MAESTRO_COMPATIBILITY_ADR_URL,
MAESTRO_COMPATIBILITY_ISSUE_URL,
MAESTRO_COMPAT_LIMITATIONS,
MAESTRO_COMPAT_SUPPORTED_CAPABILITIES,
} from './support-matrix.ts';
Loading
Loading