[rush-daemon] Invalidate warm operations whose declared outputs were deleted or changed

common/changes/@rushstack/rush-daemon/output-invalidation_2026-09-23.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rush-daemon",
+      "comment": "Re-run or restore a warm operation when its declared output folders were deleted or changed outside the daemon, instead of reporting it as up to date.",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@rushstack/rush-daemon"
+}

libraries/rush-daemon/README.md

@@ -135,7 +135,11 @@ fallback can run immediately after a completed single-client warm request withou
 A real native command holding the lock causes preparation or execution to be refused; there is no lock bypass or
 automatic retry. A later explicit request can retry after contention ends, including contention during the first
 engine initialization. A dirty native lock left by another command invalidates retained successes so the native
-incremental/cache pipeline can reconcile possibly changed ignored outputs. Installation validity is also checked on
+incremental/cache pipeline can reconcile possibly changed ignored outputs. Declared `outputFolderNames` are also
+fingerprinted (one `stat` per folder: existence, identity and modification time) when an operation succeeds or is
+restored from cache; a request whose reconciliation finds a missing or changed output folder (for example after
+`rm -rf lib`, `git clean -xdf` or `heft clean`) invalidates only that operation, so it is re-executed or restored from
+the build cache. In-place edits of nested output files are not detected. Installation validity is also checked on
 every snapshot refresh. Disposal stops new leases, awaits an outstanding lease, then aborts the graph lifetime and awaits
 runner/provider cleanup. The existing operation-completion cleanup is unchanged.
 

libraries/rush-daemon/src/OperationOutputFingerprints.ts

@@ -0,0 +1,128 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import {
+  OperationStatus,
+  type IOperationExecutionResult,
+  type IOperationGraph,
+  type Operation
+} from '@microsoft/rush-lib';
+
+const PLUGIN_NAME: 'DaemonOperationOutputFingerprints' = 'DaemonOperationOutputFingerprints';
+
+/**
+ * Retained results that allow the warm graph to skip an operation. Other statuses always re-run.
+ */
+const TRACKED_STATUSES: ReadonlySet<OperationStatus> = new Set([
+  OperationStatus.Success,
+  OperationStatus.FromCache
+]);
+
+interface IOutputFingerprint {
+  readonly record: IOperationExecutionResult;
+  readonly fingerprint: string;
+}
+
+/**
+ * Detects retained successful operations whose declared output folders were changed outside the daemon.
+ *
+ * @remarks
+ * Build outputs are normally git-ignored, so they do not contribute to any operation state hash. Without
+ * this check, deleting an output folder (`rm -rf lib`, `git clean -xdf`, `heft clean`) leaves a warm graph
+ * that reports the operation as up to date. The fingerprint is deliberately cheap: one `stat` per declared
+ * output folder, capturing existence, identity and modification time. This detects deletion, recreation,
+ * and adding, removing or renaming direct children; it does not detect in-place edits of nested files.
+ */
+export class OperationOutputFingerprints {
+  readonly #fingerprints: Map<Operation, IOutputFingerprint> = new Map();
+  readonly #graph: IOperationGraph;
+
+  public constructor(graph: IOperationGraph) {
+    this.#graph = graph;
+    graph.hooks.afterExecuteIterationAsync.tap(
+      PLUGIN_NAME,
+      (status: OperationStatus, records: ReadonlyMap<Operation, IOperationExecutionResult>) => {
+        this.#recordIteration(records);
+        return status;
+      }
+    );
+  }
+
+  /**
+   * Returns retained successful operations whose output folders no longer match the recorded fingerprint.
+   *
+   * @remarks
+   * Fingerprints of changed operations are forgotten only after all cleanup succeeded, so a failed
+   * reconciliation retries the output check on the next request instead of trusting the stale result.
+   */
+  public getOperationsWithChangedOutputs(): Operation[] {
+    const changed: Operation[] = [];
+    for (const [operation, { record, fingerprint }] of this.#fingerprints) {
+      if (!this.#isRetained(operation, record)) {
+        this.#fingerprints.delete(operation);
+      } else if (getOutputFingerprint(operation) !== fingerprint) {
+        changed.push(operation);
+      }
+    }
+    for (const operation of changed) {
+      forgetLegacySkipState(operation);
+    }
+    for (const operation of changed) {
+      this.#fingerprints.delete(operation);
+    }
+    return changed;
+  }
+
+  #recordIteration(records: ReadonlyMap<Operation, IOperationExecutionResult>): void {
+    for (const [operation, record] of records) {
+      // Only records produced by this iteration become the retained result; disabled operations keep
+      // their earlier record and fingerprint.
+      if (this.#isRetained(operation, record)) {
+        const fingerprint: string | undefined = getOutputFingerprint(operation);
+        if (fingerprint === undefined) {
+          this.#fingerprints.delete(operation);
+        } else {
+          this.#fingerprints.set(operation, { record, fingerprint });
+        }
+      }
+    }
+  }
+
+  #isRetained(operation: Operation, record: IOperationExecutionResult): boolean {
+    return this.#graph.resultByOperation.get(operation) === record && TRACKED_STATUSES.has(record.status);
+  }
+}
+
+/**
+ * Without a build cache, Rush's legacy skip detection reports an operation as skipped when its recorded
+ * input state is unchanged, even though its outputs are gone. Remove that record (as the legacy skip logic
+ * does itself before executing) so the invalidated operation is executed instead.
+ */
+function forgetLegacySkipState(operation: Operation): void {
+  fs.rmSync(
+    path.join(
+      operation.associatedProject.projectRushTempFolder,
+      `package-deps_${operation.logFilenameIdentifier}.json`
+    ),
+    { force: true }
+  );
+}
+
+function getOutputFingerprint(operation: Operation): string | undefined {
+  const folderNames: ReadonlyArray<string> | undefined = operation.settings?.outputFolderNames;
+  if (!folderNames?.length) {
+    return undefined;
+  }
+  const { projectFolder } = operation.associatedProject;
+  return folderNames
+    .map((folderName: string) => {
+      const stats: fs.Stats | undefined = fs.statSync(path.resolve(projectFolder, folderName), {
+        throwIfNoEntry: false
+      });
+      return stats ? `${folderName}:${stats.ino}:${stats.mtimeMs}` : `${folderName}:missing`;
+    })
+    .join('|');
+}

libraries/rush-daemon/src/ProductionDaemonRequestResolver.ts

@@ -32,6 +32,7 @@ import {
 } from './WorkspaceEngineComponentFactory';
 import type { IWorkspaceSession, IWorkspaceSessionComponents } from './WorkspaceSession';
 import { EngineTerminalProvider } from './EngineTerminalProvider';
+import { OperationOutputFingerprints } from './OperationOutputFingerprints';
 import { getDaemonShutdownReason } from './DaemonShutdownError';
 import type { IWorkspaceResolverLifecycle } from './WorkspaceResolverLifecycle';
 
@@ -197,6 +198,9 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver {
       }
       try {
         terminal.attach(engine.operationGraph);
+        const outputFingerprints: OperationOutputFingerprints = new OperationOutputFingerprints(
+          engine.operationGraph
+        );
         const factory: WorkspaceEngineComponentFactory = new WorkspaceEngineComponentFactory({
           createEngineComponentsAsync: async () => ({
             ...engine,
@@ -218,8 +222,11 @@ export class ProductionDaemonRequestResolver implements IDaemonRequestResolver {
           shape: engine,
           refreshInputsOnEveryRequest: true,
           validateGraphInputsAsync: this.#validateGraphInputsAsync,
-          mapInvalidationsToOperationsAsync: async (invalidationOptions) =>
-            getChangedOperations(invalidationOptions)
+          mapInvalidationsToOperationsAsync: async (invalidationOptions) => [
+            ...getChangedOperations(invalidationOptions),
+            // Outputs are git-ignored and absent from state hashes, so check them separately.
+            ...outputFingerprints.getOperationsWithChangedOutputs()
+          ]
         });
         const components: IWorkspaceSessionComponents = await factory.createAsync(options);
         return {

libraries/rush-daemon/src/test/ProductionDaemonRequestResolver.test.ts

@@ -1330,6 +1330,66 @@ process.exit(23);
     }
   });
 
+  it('re-runs only the operation whose declared outputs were deleted after a warm build', async () => {
+    const fixture: IFixture = await createFixtureAsync();
+    try {
+      await runAsync(fixture, 'initial', ['build']);
+      expect(runs(fixture)).toEqual(expect.arrayContaining(['a:one:', 'b:one:', 'c:one:']));
+      expect((await runAsync(fixture, 'warm', ['build'])).terminal).toMatchObject({
+        kind: 'requestResult',
+        payload: { exitCode: 0, scheduled: false }
+      });
+      const graph: IOperationGraph | undefined = fixture.session.operationGraph;
+      fs.rmSync(path.join(fixture.repoRoot, 'projects/a/lib'), { recursive: true });
+      expect((await runAsync(fixture, 'unrelated', ['build', '--only', 'c'])).terminal).toMatchObject({
+        kind: 'requestResult',
+        payload: { exitCode: 0, scheduled: false }
+      });
+      expect((await runAsync(fixture, 'deleted', ['build', '--to', 'b'])).terminal).toMatchObject({
+        kind: 'requestResult',
+        payload: { exitCode: 0, scheduled: true }
+      });
+      expect(runs(fixture).slice(3)).toEqual(['a:one:']);
+      expect(fs.readFileSync(path.join(fixture.repoRoot, 'projects/a/lib/output.txt'), 'utf8')).toBe('one');
+      fs.writeFileSync(path.join(fixture.repoRoot, 'projects/c/lib/extra.txt'), 'stray');
+      expect((await runAsync(fixture, 'changed', ['build'])).terminal).toMatchObject({
+        kind: 'requestResult',
+        payload: { exitCode: 0, scheduled: true }
+      });
+      expect(runs(fixture).slice(4)).toEqual(['c:one:']);
+      expect((await runAsync(fixture, 'unchanged', ['build'])).terminal).toMatchObject({
+        kind: 'requestResult',
+        payload: { exitCode: 0, scheduled: false }
+      });
+      expect(runs(fixture)).toHaveLength(5);
+      expect(fixture.session.operationGraph).toBe(graph);
+    } finally {
+      await fixture[Symbol.asyncDispose]();
+    }
+  });
+
+  it('restores deleted outputs of a warm operation from the native build cache', async () => {
+    const fixture: IFixture = await createFixtureAsync(true);
+    try {
+      await runAsync(fixture, 'initial', ['build', '--to', 'b']);
+      fs.rmSync(path.join(fixture.repoRoot, 'projects/a/lib'), { recursive: true });
+      const deleted: ITerminalExchange = await runAsync(fixture, 'deleted', ['build', '--to', 'b']);
+      expect(deleted.terminal).toMatchObject({ kind: 'requestResult', payload: { exitCode: 0 } });
+      const { operationResults } = (deleted.terminal as { payload: IDaemonPhasedRequestResult }).payload;
+      expect(operationResults.filter((result) => result.status !== 'SKIPPED')).toEqual([
+        expect.objectContaining({ operationId: 'a (compile)', status: 'FROM CACHE' })
+      ]);
+      expect(runs(fixture)).toEqual(['a:one:', 'b:one:']);
+      expect(fs.readFileSync(path.join(fixture.repoRoot, 'projects/a/lib/output.txt'), 'utf8')).toBe('one');
+      expect((await runAsync(fixture, 'restored', ['build', '--to', 'b'])).terminal).toMatchObject({
+        kind: 'requestResult',
+        payload: { exitCode: 0, scheduled: false }
+      });
+    } finally {
+      await fixture[Symbol.asyncDispose]();
+    }
+  });
+
   it('reconciles changes made without a connected client and preserves an empty native selection', async () => {
     const fixture: IFixture = await createFixtureAsync();
     let reconnected: DaemonRequestWireClient | undefined;