From 36b3133e58f562533f0b5194ec91462ba28a4378 Mon Sep 17 00:00:00 2001 From: selarkin Date: Wed, 23 Sep 2026 18:35:53 -0700 Subject: [PATCH 1/3] [rush-daemon] Invalidate warm operations whose declared outputs were deleted or changed Fixes #6058 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../output-invalidation_2026-09-23.json | 10 ++ libraries/rush-daemon/README.md | 6 +- .../src/OperationOutputFingerprints.ts | 104 ++++++++++++++++++ .../src/ProductionDaemonRequestResolver.ts | 11 +- .../ProductionDaemonRequestResolver.test.ts | 60 ++++++++++ 5 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 common/changes/@rushstack/rush-daemon/output-invalidation_2026-09-23.json create mode 100644 libraries/rush-daemon/src/OperationOutputFingerprints.ts diff --git a/common/changes/@rushstack/rush-daemon/output-invalidation_2026-09-23.json b/common/changes/@rushstack/rush-daemon/output-invalidation_2026-09-23.json new file mode 100644 index 0000000000..217487259d --- /dev/null +++ b/common/changes/@rushstack/rush-daemon/output-invalidation_2026-09-23.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-daemon", + "comment": "Re-run or restore a warm operation when its declared output folders were deleted or changed outside the daemon, instead of reporting it as up to date.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-daemon" +} diff --git a/libraries/rush-daemon/README.md b/libraries/rush-daemon/README.md index 713b69bec1..954497a7a3 100644 --- a/libraries/rush-daemon/README.md +++ b/libraries/rush-daemon/README.md @@ -125,7 +125,11 @@ fallback can run immediately after a completed warm request without stopping the A real native command holding the lock causes preparation or execution to be refused; there is no lock bypass or automatic retry. A later explicit request can retry after contention ends, including contention during the first engine initialization. A dirty native lock left by another command invalidates retained successes so the native -incremental/cache pipeline can reconcile possibly changed ignored outputs. Installation validity is also checked on +incremental/cache pipeline can reconcile possibly changed ignored outputs. Declared `outputFolderNames` are also +fingerprinted (one `stat` per folder: existence, identity and modification time) when an operation succeeds or is +restored from cache; a request whose reconciliation finds a missing or changed output folder (for example after +`rm -rf lib`, `git clean -xdf` or `heft clean`) invalidates only that operation, so it is re-executed or restored from +the build cache. In-place edits of nested output files are not detected. Installation validity is also checked on every snapshot refresh. Disposal stops new leases, awaits an outstanding lease, then aborts the graph lifetime and awaits runner/provider cleanup. The existing operation-completion cleanup is unchanged. diff --git a/libraries/rush-daemon/src/OperationOutputFingerprints.ts b/libraries/rush-daemon/src/OperationOutputFingerprints.ts new file mode 100644 index 0000000000..e9882962a6 --- /dev/null +++ b/libraries/rush-daemon/src/OperationOutputFingerprints.ts @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { + OperationStatus, + type IOperationExecutionResult, + type IOperationGraph, + type Operation +} from '@microsoft/rush-lib'; + +const PLUGIN_NAME: 'DaemonOperationOutputFingerprints' = 'DaemonOperationOutputFingerprints'; + +/** + * Retained results that allow the warm graph to skip an operation. Other statuses always re-run. + */ +const TRACKED_STATUSES: ReadonlySet = new Set([ + OperationStatus.Success, + OperationStatus.FromCache +]); + +interface IOutputFingerprint { + readonly record: IOperationExecutionResult; + readonly fingerprint: string; +} + +/** + * Detects retained successful operations whose declared output folders were changed outside the daemon. + * + * @remarks + * Build outputs are normally git-ignored, so they do not contribute to any operation state hash. Without + * this check, deleting an output folder (`rm -rf lib`, `git clean -xdf`, `heft clean`) leaves a warm graph + * that reports the operation as up to date. The fingerprint is deliberately cheap: one `stat` per declared + * output folder, capturing existence, identity and modification time. This detects deletion, recreation, + * and adding, removing or renaming direct children; it does not detect in-place edits of nested files. + */ +export class OperationOutputFingerprints { + readonly #fingerprints: Map = new Map(); + readonly #graph: IOperationGraph; + + public constructor(graph: IOperationGraph) { + this.#graph = graph; + graph.hooks.afterExecuteIterationAsync.tap( + PLUGIN_NAME, + (status: OperationStatus, records: ReadonlyMap) => { + this.#recordIteration(records); + return status; + } + ); + } + + /** + * Returns retained successful operations whose output folders no longer match the recorded fingerprint. + */ + public getOperationsWithChangedOutputs(): Operation[] { + const changed: Operation[] = []; + for (const [operation, { record, fingerprint }] of this.#fingerprints) { + if (!this.#isRetained(operation, record)) { + this.#fingerprints.delete(operation); + } else if (getOutputFingerprint(operation) !== fingerprint) { + this.#fingerprints.delete(operation); + changed.push(operation); + } + } + return changed; + } + + #recordIteration(records: ReadonlyMap): void { + for (const [operation, record] of records) { + // Only records produced by this iteration become the retained result; disabled operations keep + // their earlier record and fingerprint. + if (this.#isRetained(operation, record)) { + const fingerprint: string | undefined = getOutputFingerprint(operation); + if (fingerprint === undefined) { + this.#fingerprints.delete(operation); + } else { + this.#fingerprints.set(operation, { record, fingerprint }); + } + } + } + } + + #isRetained(operation: Operation, record: IOperationExecutionResult): boolean { + return this.#graph.resultByOperation.get(operation) === record && TRACKED_STATUSES.has(record.status); + } +} + +function getOutputFingerprint(operation: Operation): string | undefined { + const folderNames: ReadonlyArray | undefined = operation.settings?.outputFolderNames; + if (!folderNames?.length) { + return undefined; + } + const { projectFolder } = operation.associatedProject; + return folderNames + .map((folderName: string) => { + const stats: fs.Stats | undefined = fs.statSync(path.resolve(projectFolder, folderName), { + throwIfNoEntry: false + }); + return stats ? `${folderName}:${stats.ino}:${stats.mtimeMs}` : `${folderName}:missing`; + }) + .join('|'); +} diff --git a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts index 7a1793510a..21669e8cdf 100644 --- a/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts +++ b/libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts @@ -31,6 +31,7 @@ import { } from './WorkspaceEngineComponentFactory'; import type { IWorkspaceSession, IWorkspaceSessionComponents } from './WorkspaceSession'; import { EngineTerminalProvider } from './EngineTerminalProvider'; +import { OperationOutputFingerprints } from './OperationOutputFingerprints'; import type { IWorkspaceResolverLifecycle } from './WorkspaceResolverLifecycle'; /** @@ -191,6 +192,9 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { } try { terminal.attach(engine.operationGraph); + const outputFingerprints: OperationOutputFingerprints = new OperationOutputFingerprints( + engine.operationGraph + ); const factory: WorkspaceEngineComponentFactory = new WorkspaceEngineComponentFactory({ createEngineComponentsAsync: async () => ({ ...engine, @@ -212,8 +216,11 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver { shape: engine, refreshInputsOnEveryRequest: true, validateGraphInputsAsync: this.#validateGraphInputsAsync, - mapInvalidationsToOperationsAsync: async (invalidationOptions) => - getChangedOperations(invalidationOptions) + mapInvalidationsToOperationsAsync: async (invalidationOptions) => [ + ...getChangedOperations(invalidationOptions), + // Outputs are git-ignored and absent from state hashes, so check them separately. + ...outputFingerprints.getOperationsWithChangedOutputs() + ] }); const components: IWorkspaceSessionComponents = await factory.createAsync(options); return { diff --git a/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts b/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts index b0d02fd298..e25a35c8ee 100644 --- a/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts +++ b/libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts @@ -1329,6 +1329,66 @@ process.exit(23); } }); + it('re-runs only the operation whose declared outputs were deleted after a warm build', async () => { + const fixture: IFixture = await createFixtureAsync(); + try { + await runAsync(fixture, 'initial', ['build']); + expect(runs(fixture)).toEqual(expect.arrayContaining(['a:one:', 'b:one:', 'c:one:'])); + expect((await runAsync(fixture, 'warm', ['build'])).terminal).toMatchObject({ + kind: 'requestResult', + payload: { exitCode: 0, scheduled: false } + }); + const graph: IOperationGraph | undefined = fixture.session.operationGraph; + fs.rmSync(path.join(fixture.repoRoot, 'projects/a/lib'), { recursive: true }); + expect((await runAsync(fixture, 'unrelated', ['build', '--only', 'c'])).terminal).toMatchObject({ + kind: 'requestResult', + payload: { exitCode: 0, scheduled: false } + }); + expect((await runAsync(fixture, 'deleted', ['build', '--to', 'b'])).terminal).toMatchObject({ + kind: 'requestResult', + payload: { exitCode: 0, scheduled: true } + }); + expect(runs(fixture).slice(3)).toEqual(['a:one:']); + expect(fs.readFileSync(path.join(fixture.repoRoot, 'projects/a/lib/output.txt'), 'utf8')).toBe('one'); + fs.writeFileSync(path.join(fixture.repoRoot, 'projects/c/lib/extra.txt'), 'stray'); + expect((await runAsync(fixture, 'changed', ['build'])).terminal).toMatchObject({ + kind: 'requestResult', + payload: { exitCode: 0, scheduled: true } + }); + expect(runs(fixture).slice(4)).toEqual(['c:one:']); + expect((await runAsync(fixture, 'unchanged', ['build'])).terminal).toMatchObject({ + kind: 'requestResult', + payload: { exitCode: 0, scheduled: false } + }); + expect(runs(fixture)).toHaveLength(5); + expect(fixture.session.operationGraph).toBe(graph); + } finally { + await fixture[Symbol.asyncDispose](); + } + }); + + it('restores deleted outputs of a warm operation from the native build cache', async () => { + const fixture: IFixture = await createFixtureAsync(true); + try { + await runAsync(fixture, 'initial', ['build', '--to', 'b']); + fs.rmSync(path.join(fixture.repoRoot, 'projects/a/lib'), { recursive: true }); + const deleted: ITerminalExchange = await runAsync(fixture, 'deleted', ['build', '--to', 'b']); + expect(deleted.terminal).toMatchObject({ kind: 'requestResult', payload: { exitCode: 0 } }); + const { operationResults } = (deleted.terminal as { payload: IDaemonPhasedRequestResult }).payload; + expect(operationResults.filter((result) => result.status !== 'SKIPPED')).toEqual([ + expect.objectContaining({ operationId: 'a (compile)', status: 'FROM CACHE' }) + ]); + expect(runs(fixture)).toEqual(['a:one:', 'b:one:']); + expect(fs.readFileSync(path.join(fixture.repoRoot, 'projects/a/lib/output.txt'), 'utf8')).toBe('one'); + expect((await runAsync(fixture, 'restored', ['build', '--to', 'b'])).terminal).toMatchObject({ + kind: 'requestResult', + payload: { exitCode: 0, scheduled: false } + }); + } finally { + await fixture[Symbol.asyncDispose](); + } + }); + it('reconciles changes made without a connected client and preserves an empty native selection', async () => { const fixture: IFixture = await createFixtureAsync(); let reconnected: DaemonRequestWireClient | undefined; From 5700bc0a3c60caf4cec08f821897b14435012fe3 Mon Sep 17 00:00:00 2001 From: selarkin Date: Wed, 23 Sep 2026 19:19:37 -0700 Subject: [PATCH 2/3] [rush-daemon] Force re-execution of invalidated outputs under legacy skip detection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../src/OperationOutputFingerprints.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/libraries/rush-daemon/src/OperationOutputFingerprints.ts b/libraries/rush-daemon/src/OperationOutputFingerprints.ts index e9882962a6..041229f4d0 100644 --- a/libraries/rush-daemon/src/OperationOutputFingerprints.ts +++ b/libraries/rush-daemon/src/OperationOutputFingerprints.ts @@ -61,6 +61,7 @@ export class OperationOutputFingerprints { this.#fingerprints.delete(operation); } else if (getOutputFingerprint(operation) !== fingerprint) { this.#fingerprints.delete(operation); + forgetLegacySkipState(operation); changed.push(operation); } } @@ -87,6 +88,21 @@ export class OperationOutputFingerprints { } } +/** + * Without a build cache, Rush's legacy skip detection reports an operation as skipped when its recorded + * input state is unchanged, even though its outputs are gone. Remove that record (as the legacy skip logic + * does itself before executing) so the invalidated operation is executed instead. + */ +function forgetLegacySkipState(operation: Operation): void { + fs.rmSync( + path.join( + operation.associatedProject.projectRushTempFolder, + `package-deps_${operation.logFilenameIdentifier}.json` + ), + { force: true } + ); +} + function getOutputFingerprint(operation: Operation): string | undefined { const folderNames: ReadonlyArray | undefined = operation.settings?.outputFolderNames; if (!folderNames?.length) { From e91472ae33db26b916481155d48a509f2abe98ed Mon Sep 17 00:00:00 2001 From: selarkin Date: Thu, 24 Sep 2026 11:04:58 -0700 Subject: [PATCH 3/3] [rush-daemon] Keep output fingerprints until legacy skip cleanup succeeds Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../rush-daemon/src/OperationOutputFingerprints.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libraries/rush-daemon/src/OperationOutputFingerprints.ts b/libraries/rush-daemon/src/OperationOutputFingerprints.ts index 041229f4d0..7536718113 100644 --- a/libraries/rush-daemon/src/OperationOutputFingerprints.ts +++ b/libraries/rush-daemon/src/OperationOutputFingerprints.ts @@ -53,6 +53,10 @@ export class OperationOutputFingerprints { /** * Returns retained successful operations whose output folders no longer match the recorded fingerprint. + * + * @remarks + * Fingerprints of changed operations are forgotten only after all cleanup succeeded, so a failed + * reconciliation retries the output check on the next request instead of trusting the stale result. */ public getOperationsWithChangedOutputs(): Operation[] { const changed: Operation[] = []; @@ -60,11 +64,15 @@ export class OperationOutputFingerprints { if (!this.#isRetained(operation, record)) { this.#fingerprints.delete(operation); } else if (getOutputFingerprint(operation) !== fingerprint) { - this.#fingerprints.delete(operation); - forgetLegacySkipState(operation); changed.push(operation); } } + for (const operation of changed) { + forgetLegacySkipState(operation); + } + for (const operation of changed) { + this.#fingerprints.delete(operation); + } return changed; }