Stabilize parity downloads and retain deterministic failure evidence

.github/workflows/phpunit-feature.yml

@@ -395,7 +395,7 @@ jobs:
                 -w /app \
                 --entrypoint sh \
                 durable-workflow-server-corpus-validator \
-                -lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify "$CORPUS_BASE_REF^{commit}"; php scripts/ci/check-worker-openapi-evolution.php "$CORPUS_BASE_REF"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref "$CORPUS_BASE_REF" --verify-counterfactual'
+                -lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify "$CORPUS_BASE_REF^{commit}"; php scripts/ci/check-worker-openapi-evolution.php "$CORPUS_BASE_REF"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php tests/Unit/ServerParityLargeCommandBatchesProbeTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref "$CORPUS_BASE_REF" --verify-counterfactual'
 
       - name: Select production image qualification for changed build inputs
         id: image_inputs

.github/workflows/server-parity.yml

@@ -272,6 +272,23 @@ jobs:
             mirror.gcr.io/library/rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081 \
             mirror.gcr.io/library/python@sha256:bf44cdfcb76cd3b41e879bc058fc37ec5872002ccfde7fcb765e218cde0cd79c \
             "$(baseline php-2.11.8)"
+      # Every corpus job replays CLI 2.2.0. Fetch it once for the whole run,
+      # bound to its committed digest and size, instead of once per corpus
+      # job: the release download URL, then the release-asset API URL only if
+      # that transfer fails (#1103). Each consumer verifies the digest again.
+      - name: Fetch the parity CLI once, bound to its committed digest
+        timeout-minutes: 9
+        run: |
+          set -euo pipefail
+          bash scripts/ci/fetch-parity-cli.sh \
+            "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" 626966 parity-cli/dw.phar
+      - name: Share the verified parity CLI with the corpus jobs
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: parity-cli-2.2.0
+          path: parity-cli
+          retention-days: 1
+          if-no-files-found: error
       # The adapters' pinned, --locked dependencies build the same way every
       # run. Restore their public Cargo registry data and build tree, keyed
       # by the toolchain image, the runner and every tracked adapter input.
@@ -582,6 +599,12 @@ jobs:
           name: corrected-marker-adapters
           path: corrected-marker-adapters
           digest-mismatch: error
+      - name: Fetch this run's verified parity CLI
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: parity-cli-2.2.0
+          path: parity-cli
+          digest-mismatch: error
       - name: Create TLS fixture certificates
         run: |
           set -euo pipefail
@@ -826,7 +849,7 @@ jobs:
           image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.8.json)
           export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD)
           mkdir -p parity-consumers
-          curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar
+          cp parity-cli/dw.phar parity-consumers/dw.phar
           printf '%s  %s\n' "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" parity-consumers/dw.phar | sha256sum -c > parity-evidence/cli-artifact.txt
           export DW_PARITY_CLI_PHAR=/source/parity-consumers/dw.phar
           docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \
@@ -1360,6 +1383,12 @@ jobs:
           name: corrected-marker-adapters
           path: corrected-marker-adapters
           digest-mismatch: error
+      - name: Fetch this run's verified parity CLI
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: parity-cli-2.2.0
+          path: parity-cli
+          digest-mismatch: error
       # Exact dependency keys include the actual compiler pin, Cargo configuration
       # and vendored crates. Server outputs are discarded before rebuilding.
       - name: Restore native Cargo dependencies from trusted main
@@ -1582,7 +1611,7 @@ jobs:
           image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.8.json)
           export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD)
           mkdir -p parity-consumers
-          curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar
+          cp parity-cli/dw.phar parity-consumers/dw.phar
           printf '%s  %s\n' "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" parity-consumers/dw.phar | sha256sum -c > parity-evidence/cli-artifact.txt
           export DW_PARITY_CLI_PHAR=/source/parity-consumers/dw.phar
           docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \
@@ -2144,6 +2173,12 @@ jobs:
           name: corrected-marker-adapters
           path: corrected-marker-adapters
           digest-mismatch: error
+      - name: Fetch this run's verified parity CLI
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: parity-cli-2.2.0
+          path: parity-cli
+          digest-mismatch: error
       # Exact dependency keys include the actual compiler pin, Cargo configuration
       # and vendored crates. Server outputs are discarded before rebuilding.
       - name: Restore native Cargo dependencies from trusted main
@@ -2339,6 +2374,7 @@ jobs:
               tests/Unit/ServerParityUpdateValidationProcessTest.mjs \
               tests/Unit/ServerParityKilledAuthorPollTest.mjs \
               tests/Unit/ServerParityImagePullTest.mjs \
+              tests/Unit/ServerParityCliFetchTest.mjs \
               tests/Unit/ServerParityAdapterCacheTest.mjs \
               tests/Unit/ServerParityWorkerSessionsContractTest.mjs \
               tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs \
@@ -2410,7 +2446,7 @@ jobs:
           image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.8.json)
           export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD)
           mkdir -p parity-consumers
-          curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar
+          cp parity-cli/dw.phar parity-consumers/dw.phar
           printf '%s  %s\n' "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" parity-consumers/dw.phar | sha256sum -c > parity-evidence/cli-artifact.txt
           export DW_PARITY_CLI_PHAR=/source/parity-consumers/dw.phar
           docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \

rust/src/runtime/mod.rs

@@ -185,6 +185,9 @@ mod task_repair;
 mod task_repair_http;
 mod terminal_summary;
 mod terminal_updates;
+#[cfg(debug_assertions)]
+#[doc(hidden)]
+pub mod timer_holds;
 mod timer_summary;
 mod transaction;
 mod update_summary;

rust/src/runtime/store.rs

@@ -4156,6 +4156,11 @@ where
             if fire_at > fired_at {
                 continue;
             }
+            // Debug builds: a test may hold this timer pending (`timer_holds`).
+            #[cfg(debug_assertions)]
+            if super::timer_holds::held(&DB::string(&timer, "id")?) {
+                continue;
+            }
             let run_id = DB::string(&timer, "workflow_run_id")?;
             match Self::active_run(&mut tx, &run_id).await {
                 Ok(_) => {}

rust/src/runtime/timer_holds.rs

@@ -0,0 +1,55 @@
+//! A debug-build test seam for the maintenance loop's due timers. A test holds
+//! one timer by its ID; while the hold lives, `fire_due_timers` leaves that
+//! timer pending and untouched, and reports the first tick that found it due.
+//! Regressions use it to keep an already-due timer unfired across worker
+//! turns, in a fixed order and without sleeping. Holds are process-wide and
+//! keyed by timer ID, so concurrent tests never share one. Release builds
+//! compile neither this module nor its call site.
+use std::sync::{Mutex, PoisonError};
+use tokio::sync::oneshot;
+
+struct Hold {
+    timer_id: String,
+    found: Option<oneshot::Sender<()>>,
+}
+
+static HOLDS: Mutex<Vec<Hold>> = Mutex::new(Vec::new());
+
+/// A live hold. `skipped` resolves once a maintenance tick has found the
+/// timer due and left it pending; dropping the hold lets later ticks fire it.
+pub struct TimerHold {
+    timer_id: String,
+    pub skipped: oneshot::Receiver<()>,
+}
+
+/// Hold the timer `timer_id` until the returned hold is dropped.
+pub fn hold_timer(timer_id: &str) -> TimerHold {
+    let (found, skipped) = oneshot::channel();
+    HOLDS
+        .lock()
+        .unwrap_or_else(PoisonError::into_inner)
+        .push(Hold { timer_id: timer_id.to_owned(), found: Some(found) });
+    TimerHold { timer_id: timer_id.to_owned(), skipped }
+}
+
+impl Drop for TimerHold {
+    fn drop(&mut self) {
+        HOLDS
+            .lock()
+            .unwrap_or_else(PoisonError::into_inner)
+            .retain(|hold| hold.timer_id != self.timer_id);
+    }
+}
+
+/// A due timer in `fire_due_timers`: whether a test holds it. The first tick
+/// that finds it reports to the test.
+pub(super) fn held(timer_id: &str) -> bool {
+    let mut holds = HOLDS.lock().unwrap_or_else(PoisonError::into_inner);
+    let Some(hold) = holds.iter_mut().find(|hold| hold.timer_id == timer_id) else {
+        return false;
+    };
+    if let Some(found) = hold.found.take() {
+        let _ = found.send(());
+    }
+    true
+}

rust/tests/execution/occurrence_deadlines.rs

@@ -153,16 +153,32 @@ async fn false_wakes_keep_the_first_open_deadline_exactly() {
 
 /// A already passed when the reopen commits (the chain's own timer had not
 /// fired): the reopened timer keeps fire_at = A, is due at once and fires;
-/// no fresh positive budget is invented.
+/// no fresh positive budget is invented. Moving A into the past makes the
+/// first wait's timer due, so a maintenance tick (every 250 ms) inside the
+/// wake turn would fire it and the reopen would be refused, as the control
+/// below pins (#1105). Debug builds hold that one timer, and the wake turn
+/// starts only after a tick has found it due and left it.
+#[cfg(debug_assertions)]
 #[tokio::test]
 async fn a_reopen_after_the_anchor_is_due_immediately_at_the_anchor() {
+    use durable_workflow_server::runtime::timer_holds::hold_timer;
     let (database, runtime, app) = setup("default").await;
     let (run, _) = open(&app, "default", "deadline-expired", "php:condition-wait:0", 60).await;
-    // Move A into the past on the first wait's durable timer only; its timer
-    // task stays far in the future, so the engine does not fire it first.
+    let first = payloads(&history(&app, "default", &run).await, "TimerScheduled")[0]["timer_id"]
+        .as_str().unwrap().to_owned();
+    let mut hold = hold_timer(&first);
+    // Move A into the past on the first wait's durable timer only. The timer
+    // is due from here on; the hold keeps it pending.
     database.execute("UPDATE workflow_run_timers SET fire_at=(SELECT started_at FROM workflow_runs WHERE workflow_runs.id=workflow_run_timers.workflow_run_id) WHERE sequence=1").await;
+    tokio::time::timeout(std::time::Duration::from_secs(10), &mut hold.skipped).await
+        .expect("a maintenance tick found the moved timer due").unwrap();
     let done = false_wake(&app, "default", &run, "php:condition-wait:0", 59, 60, std::time::Duration::ZERO).await;
     assert_eq!(done.0, StatusCode::OK, "{}", done.1);
+    // The reopen's commit cancelled the first timer unfired, so releasing the
+    // hold cannot fire it.
+    drop(hold);
+    assert_eq!(task_retry::rows(&database,
+        "SELECT COUNT(*) FROM workflow_run_timers WHERE sequence=1 AND status='cancelled' AND fired_at IS NULL").await, 1);
     let events = history(&app, "default", &run).await;
     let opened = payloads(&events, "ConditionWaitOpened");
     let timers = payloads(&events, "TimerScheduled");

scripts/ci/fetch-parity-cli.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Fetches CLI 2.2.0's dw.phar once for a Shared Server fixtures run (#1103).
+#
+# Usage: fetch-parity-cli.sh SHA256 SIZE OUTPUT
+#
+# The release download URL is tried first. Only if that transfer itself fails
+# is the same release asset fetched through its GitHub API URL. Either way the
+# file is accepted only with the exact size and the committed SHA-256: a
+# completed transfer with other bytes fails at once, with no second source.
+#
+# Every transfer is HTTPS-only, redirects included, with a 10 s connect and a
+# 60 s per-attempt bound. curl retries with its doubling backoff inside a
+# per-route window, and `timeout` caps each route (release URL 300 s, API URL
+# 180 s), so the whole fetch ends within 480 s.
+set -euo pipefail
+
+want="${1:-}"
+size="${2:-}"
+out="${3:-}"
+[[ "$want" =~ ^[0-9a-f]{64}$ && "$size" =~ ^[1-9][0-9]*$ && -n "$out" ]] || {
+    echo "fetch-parity-cli.sh: usage: fetch-parity-cli.sh SHA256 SIZE OUTPUT" >&2
+    exit 2
+}
+
+primary=https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar
+# Release 404473194 (tag 2.2.0), asset 614926795 "dw.phar", 626,966 bytes.
+asset=https://api.github.com/repos/durable-workflow/cli/releases/assets/614926795
+
+fetch() { # CEILING WINDOW URL [CURL OPTION...]
+    rm -f "$out"
+    timeout "$1" curl --fail --silent --show-error --proto '=https' --proto-redir '=https' --location \
+        --connect-timeout 10 --max-time 60 --retry 8 --retry-all-errors --retry-max-time "$2" \
+        --output "$out" "${@:4}" "$3"
+}
+
+mkdir -p "$(dirname "$out")"
+if fetch 300 240 "$primary"; then
+    echo "parity CLI transport: release download URL"
+else
+    status=$?
+    echo "parity CLI transport: release download URL failed (exit $status); trying the release-asset API URL" >&2
+    if ! fetch 180 120 "$asset" -H 'Accept: application/octet-stream'; then
+        echo "fetch-parity-cli.sh: both transports failed" >&2
+        exit 1
+    fi
+    echo "parity CLI transport: release-asset API URL"
+fi
+
+actual="$(stat -c %s "$out")"
+[[ "$actual" = "$size" ]] || {
+    echo "fetch-parity-cli.sh: $out is $actual bytes, expected $size" >&2
+    exit 1
+}
+printf '%s  %s\n' "$want" "$out" | sha256sum -c

scripts/ci/run-composer.sh

@@ -41,12 +41,12 @@ trap 'rm -rf -- "$ca" "$errors"' EXIT
 sh "$here/checked-ca.sh" "$ca"
 chmod 0755 "$ca"
 
-# A locked install only downloads pinned dists, so it is safe to repeat. Retry
-# it, at most three attempts in all, only when Composer reports a transient
-# download failure such as GitHub codeload's HTTP 429 rate limit. Any other
-# failure, every other Composer command and every kept (named, not --rm)
-# container fails on its first attempt.
-transient='could not be downloaded \((HTTP/[0-9.]+ (429|5[0-9][0-9])|curl error (6|7|28|35|52|56))'
+# A locked --rm install retries downloads at most three times. Match HTTP
+# 429/5xx errors and Composer's "curl error N while downloading" transport
+# errors. The space after N excludes curl 60 (TLS). Other failures, commands
+# and kept containers fail on the first attempt. Keep the existing 20/40s
+# backoff, checked CA pair and pinned Composer image.
+transient='could not be downloaded \(HTTP/[0-9.]+ (429|5[0-9][0-9])[^0-9]|curl error (6|7|28|35|52|56) while downloading https://'
 attempts=1
 if [[ $1 == install ]] && [[ " ${options[*]} " == *" --rm "* ]]; then attempts=3; fi
 for attempt in $(seq "$attempts"); do