diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index be5ac76b9..715688e8d 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -395,7 +395,7 @@ jobs: -w /app \ --entrypoint sh \ durable-workflow-server-corpus-validator \ - -lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify "$CORPUS_BASE_REF^{commit}"; php scripts/ci/check-worker-openapi-evolution.php "$CORPUS_BASE_REF"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref "$CORPUS_BASE_REF" --verify-counterfactual' + -lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify "$CORPUS_BASE_REF^{commit}"; php scripts/ci/check-worker-openapi-evolution.php "$CORPUS_BASE_REF"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php tests/Unit/ServerParityLargeCommandBatchesProbeTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref "$CORPUS_BASE_REF" --verify-counterfactual' - name: Select production image qualification for changed build inputs id: image_inputs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 64c530f9a..36dd4cad0 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -272,6 +272,23 @@ jobs: mirror.gcr.io/library/rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081 \ mirror.gcr.io/library/python@sha256:bf44cdfcb76cd3b41e879bc058fc37ec5872002ccfde7fcb765e218cde0cd79c \ "$(baseline php-2.11.8)" + # Every corpus job replays CLI 2.2.0. Fetch it once for the whole run, + # bound to its committed digest and size, instead of once per corpus + # job: the release download URL, then the release-asset API URL only if + # that transfer fails (#1103). Each consumer verifies the digest again. + - name: Fetch the parity CLI once, bound to its committed digest + timeout-minutes: 9 + run: | + set -euo pipefail + bash scripts/ci/fetch-parity-cli.sh \ + "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" 626966 parity-cli/dw.phar + - name: Share the verified parity CLI with the corpus jobs + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: parity-cli-2.2.0 + path: parity-cli + retention-days: 1 + if-no-files-found: error # The adapters' pinned, --locked dependencies build the same way every # run. Restore their public Cargo registry data and build tree, keyed # by the toolchain image, the runner and every tracked adapter input. @@ -582,6 +599,12 @@ jobs: name: corrected-marker-adapters path: corrected-marker-adapters digest-mismatch: error + - name: Fetch this run's verified parity CLI + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: parity-cli-2.2.0 + path: parity-cli + digest-mismatch: error - name: Create TLS fixture certificates run: | set -euo pipefail @@ -826,7 +849,7 @@ jobs: image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.8.json) export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) mkdir -p parity-consumers - curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar + cp parity-cli/dw.phar parity-consumers/dw.phar printf '%s %s\n' "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" parity-consumers/dw.phar | sha256sum -c > parity-evidence/cli-artifact.txt export DW_PARITY_CLI_PHAR=/source/parity-consumers/dw.phar docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ @@ -1360,6 +1383,12 @@ jobs: name: corrected-marker-adapters path: corrected-marker-adapters digest-mismatch: error + - name: Fetch this run's verified parity CLI + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: parity-cli-2.2.0 + path: parity-cli + digest-mismatch: error # Exact dependency keys include the actual compiler pin, Cargo configuration # and vendored crates. Server outputs are discarded before rebuilding. - name: Restore native Cargo dependencies from trusted main @@ -1582,7 +1611,7 @@ jobs: image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.8.json) export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) mkdir -p parity-consumers - curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar + cp parity-cli/dw.phar parity-consumers/dw.phar printf '%s %s\n' "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" parity-consumers/dw.phar | sha256sum -c > parity-evidence/cli-artifact.txt export DW_PARITY_CLI_PHAR=/source/parity-consumers/dw.phar docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ @@ -2144,6 +2173,12 @@ jobs: name: corrected-marker-adapters path: corrected-marker-adapters digest-mismatch: error + - name: Fetch this run's verified parity CLI + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: parity-cli-2.2.0 + path: parity-cli + digest-mismatch: error # Exact dependency keys include the actual compiler pin, Cargo configuration # and vendored crates. Server outputs are discarded before rebuilding. - name: Restore native Cargo dependencies from trusted main @@ -2339,6 +2374,7 @@ jobs: tests/Unit/ServerParityUpdateValidationProcessTest.mjs \ tests/Unit/ServerParityKilledAuthorPollTest.mjs \ tests/Unit/ServerParityImagePullTest.mjs \ + tests/Unit/ServerParityCliFetchTest.mjs \ tests/Unit/ServerParityAdapterCacheTest.mjs \ tests/Unit/ServerParityWorkerSessionsContractTest.mjs \ tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs \ @@ -2410,7 +2446,7 @@ jobs: image=$(jq -r '.server.images["linux/amd64"] | sub("^durableworkflow/server@"; "ghcr.io/durable-workflow/server@")' tests/Fixtures/ServerParityBaselines/php-2.11.8.json) export DW_PARITY_RUNNER_REVISION=$(git rev-parse HEAD) mkdir -p parity-consumers - curl --fail --location --retry 3 --output parity-consumers/dw.phar https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar + cp parity-cli/dw.phar parity-consumers/dw.phar printf '%s %s\n' "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" parity-consumers/dw.phar | sha256sum -c > parity-evidence/cli-artifact.txt export DW_PARITY_CLI_PHAR=/source/parity-consumers/dw.phar docker run --rm --user=1000:1000 --entrypoint node --network "$RESOURCE_SCOPE" \ diff --git a/rust/src/runtime/mod.rs b/rust/src/runtime/mod.rs index c7d6e51af..9b41f5089 100644 --- a/rust/src/runtime/mod.rs +++ b/rust/src/runtime/mod.rs @@ -185,6 +185,9 @@ mod task_repair; mod task_repair_http; mod terminal_summary; mod terminal_updates; +#[cfg(debug_assertions)] +#[doc(hidden)] +pub mod timer_holds; mod timer_summary; mod transaction; mod update_summary; diff --git a/rust/src/runtime/store.rs b/rust/src/runtime/store.rs index 372b8034b..484103f2e 100644 --- a/rust/src/runtime/store.rs +++ b/rust/src/runtime/store.rs @@ -4156,6 +4156,11 @@ where if fire_at > fired_at { continue; } + // Debug builds: a test may hold this timer pending (`timer_holds`). + #[cfg(debug_assertions)] + if super::timer_holds::held(&DB::string(&timer, "id")?) { + continue; + } let run_id = DB::string(&timer, "workflow_run_id")?; match Self::active_run(&mut tx, &run_id).await { Ok(_) => {} diff --git a/rust/src/runtime/timer_holds.rs b/rust/src/runtime/timer_holds.rs new file mode 100644 index 000000000..ad94e6282 --- /dev/null +++ b/rust/src/runtime/timer_holds.rs @@ -0,0 +1,55 @@ +//! A debug-build test seam for the maintenance loop's due timers. A test holds +//! one timer by its ID; while the hold lives, `fire_due_timers` leaves that +//! timer pending and untouched, and reports the first tick that found it due. +//! Regressions use it to keep an already-due timer unfired across worker +//! turns, in a fixed order and without sleeping. Holds are process-wide and +//! keyed by timer ID, so concurrent tests never share one. Release builds +//! compile neither this module nor its call site. +use std::sync::{Mutex, PoisonError}; +use tokio::sync::oneshot; + +struct Hold { + timer_id: String, + found: Option>, +} + +static HOLDS: Mutex> = Mutex::new(Vec::new()); + +/// A live hold. `skipped` resolves once a maintenance tick has found the +/// timer due and left it pending; dropping the hold lets later ticks fire it. +pub struct TimerHold { + timer_id: String, + pub skipped: oneshot::Receiver<()>, +} + +/// Hold the timer `timer_id` until the returned hold is dropped. +pub fn hold_timer(timer_id: &str) -> TimerHold { + let (found, skipped) = oneshot::channel(); + HOLDS + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(Hold { timer_id: timer_id.to_owned(), found: Some(found) }); + TimerHold { timer_id: timer_id.to_owned(), skipped } +} + +impl Drop for TimerHold { + fn drop(&mut self) { + HOLDS + .lock() + .unwrap_or_else(PoisonError::into_inner) + .retain(|hold| hold.timer_id != self.timer_id); + } +} + +/// A due timer in `fire_due_timers`: whether a test holds it. The first tick +/// that finds it reports to the test. +pub(super) fn held(timer_id: &str) -> bool { + let mut holds = HOLDS.lock().unwrap_or_else(PoisonError::into_inner); + let Some(hold) = holds.iter_mut().find(|hold| hold.timer_id == timer_id) else { + return false; + }; + if let Some(found) = hold.found.take() { + let _ = found.send(()); + } + true +} diff --git a/rust/tests/execution/occurrence_deadlines.rs b/rust/tests/execution/occurrence_deadlines.rs index b96e1beea..c0e81a3b3 100644 --- a/rust/tests/execution/occurrence_deadlines.rs +++ b/rust/tests/execution/occurrence_deadlines.rs @@ -153,16 +153,32 @@ async fn false_wakes_keep_the_first_open_deadline_exactly() { /// A already passed when the reopen commits (the chain's own timer had not /// fired): the reopened timer keeps fire_at = A, is due at once and fires; -/// no fresh positive budget is invented. +/// no fresh positive budget is invented. Moving A into the past makes the +/// first wait's timer due, so a maintenance tick (every 250 ms) inside the +/// wake turn would fire it and the reopen would be refused, as the control +/// below pins (#1105). Debug builds hold that one timer, and the wake turn +/// starts only after a tick has found it due and left it. +#[cfg(debug_assertions)] #[tokio::test] async fn a_reopen_after_the_anchor_is_due_immediately_at_the_anchor() { + use durable_workflow_server::runtime::timer_holds::hold_timer; let (database, runtime, app) = setup("default").await; let (run, _) = open(&app, "default", "deadline-expired", "php:condition-wait:0", 60).await; - // Move A into the past on the first wait's durable timer only; its timer - // task stays far in the future, so the engine does not fire it first. + let first = payloads(&history(&app, "default", &run).await, "TimerScheduled")[0]["timer_id"] + .as_str().unwrap().to_owned(); + let mut hold = hold_timer(&first); + // Move A into the past on the first wait's durable timer only. The timer + // is due from here on; the hold keeps it pending. database.execute("UPDATE workflow_run_timers SET fire_at=(SELECT started_at FROM workflow_runs WHERE workflow_runs.id=workflow_run_timers.workflow_run_id) WHERE sequence=1").await; + tokio::time::timeout(std::time::Duration::from_secs(10), &mut hold.skipped).await + .expect("a maintenance tick found the moved timer due").unwrap(); let done = false_wake(&app, "default", &run, "php:condition-wait:0", 59, 60, std::time::Duration::ZERO).await; assert_eq!(done.0, StatusCode::OK, "{}", done.1); + // The reopen's commit cancelled the first timer unfired, so releasing the + // hold cannot fire it. + drop(hold); + assert_eq!(task_retry::rows(&database, + "SELECT COUNT(*) FROM workflow_run_timers WHERE sequence=1 AND status='cancelled' AND fired_at IS NULL").await, 1); let events = history(&app, "default", &run).await; let opened = payloads(&events, "ConditionWaitOpened"); let timers = payloads(&events, "TimerScheduled"); diff --git a/scripts/ci/fetch-parity-cli.sh b/scripts/ci/fetch-parity-cli.sh new file mode 100755 index 000000000..9630f0180 --- /dev/null +++ b/scripts/ci/fetch-parity-cli.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Fetches CLI 2.2.0's dw.phar once for a Shared Server fixtures run (#1103). +# +# Usage: fetch-parity-cli.sh SHA256 SIZE OUTPUT +# +# The release download URL is tried first. Only if that transfer itself fails +# is the same release asset fetched through its GitHub API URL. Either way the +# file is accepted only with the exact size and the committed SHA-256: a +# completed transfer with other bytes fails at once, with no second source. +# +# Every transfer is HTTPS-only, redirects included, with a 10 s connect and a +# 60 s per-attempt bound. curl retries with its doubling backoff inside a +# per-route window, and `timeout` caps each route (release URL 300 s, API URL +# 180 s), so the whole fetch ends within 480 s. +set -euo pipefail + +want="${1:-}" +size="${2:-}" +out="${3:-}" +[[ "$want" =~ ^[0-9a-f]{64}$ && "$size" =~ ^[1-9][0-9]*$ && -n "$out" ]] || { + echo "fetch-parity-cli.sh: usage: fetch-parity-cli.sh SHA256 SIZE OUTPUT" >&2 + exit 2 +} + +primary=https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar +# Release 404473194 (tag 2.2.0), asset 614926795 "dw.phar", 626,966 bytes. +asset=https://api.github.com/repos/durable-workflow/cli/releases/assets/614926795 + +fetch() { # CEILING WINDOW URL [CURL OPTION...] + rm -f "$out" + timeout "$1" curl --fail --silent --show-error --proto '=https' --proto-redir '=https' --location \ + --connect-timeout 10 --max-time 60 --retry 8 --retry-all-errors --retry-max-time "$2" \ + --output "$out" "${@:4}" "$3" +} + +mkdir -p "$(dirname "$out")" +if fetch 300 240 "$primary"; then + echo "parity CLI transport: release download URL" +else + status=$? + echo "parity CLI transport: release download URL failed (exit $status); trying the release-asset API URL" >&2 + if ! fetch 180 120 "$asset" -H 'Accept: application/octet-stream'; then + echo "fetch-parity-cli.sh: both transports failed" >&2 + exit 1 + fi + echo "parity CLI transport: release-asset API URL" +fi + +actual="$(stat -c %s "$out")" +[[ "$actual" = "$size" ]] || { + echo "fetch-parity-cli.sh: $out is $actual bytes, expected $size" >&2 + exit 1 +} +printf '%s %s\n' "$want" "$out" | sha256sum -c diff --git a/scripts/ci/run-composer.sh b/scripts/ci/run-composer.sh index 528dbcb5b..695c7f4f9 100644 --- a/scripts/ci/run-composer.sh +++ b/scripts/ci/run-composer.sh @@ -41,12 +41,12 @@ trap 'rm -rf -- "$ca" "$errors"' EXIT sh "$here/checked-ca.sh" "$ca" chmod 0755 "$ca" -# A locked install only downloads pinned dists, so it is safe to repeat. Retry -# it, at most three attempts in all, only when Composer reports a transient -# download failure such as GitHub codeload's HTTP 429 rate limit. Any other -# failure, every other Composer command and every kept (named, not --rm) -# container fails on its first attempt. -transient='could not be downloaded \((HTTP/[0-9.]+ (429|5[0-9][0-9])|curl error (6|7|28|35|52|56))' +# A locked --rm install retries downloads at most three times. Match HTTP +# 429/5xx errors and Composer's "curl error N while downloading" transport +# errors. The space after N excludes curl 60 (TLS). Other failures, commands +# and kept containers fail on the first attempt. Keep the existing 20/40s +# backoff, checked CA pair and pinned Composer image. +transient='could not be downloaded \(HTTP/[0-9.]+ (429|5[0-9][0-9])[^0-9]|curl error (6|7|28|35|52|56) while downloading https://' attempts=1 if [[ $1 == install ]] && [[ " ${options[*]} " == *" --rm "* ]]; then attempts=3; fi for attempt in $(seq "$attempts"); do diff --git a/scripts/conformance/server-parity/large-command-batches-probe.php b/scripts/conformance/server-parity/large-command-batches-probe.php index ba5dfe6d9..1b4d34e40 100644 --- a/scripts/conformance/server-parity/large-command-batches-probe.php +++ b/scripts/conformance/server-parity/large-command-batches-probe.php @@ -6,6 +6,26 @@ use GuzzleHttp\Client as LargeCommandBatchesHttp; use GuzzleHttp\HandlerStack; +/** Keep the original claim predicate and retain a bounded refusal without payloads or headers. */ +function largeCommandBatchesClaim(array $poll, string $case, int $pollRetries): array +{ + $response = $poll['response'] ?? null; + $task = is_array($response) ? ($response['task'] ?? null) : null; + if (is_array($task)) { + return $task; + } + $summary = ['http_status' => $poll['status'] ?? null, 'response_type' => get_debug_type($response), + 'task_present' => is_array($response) && array_key_exists('task', $response), 'task_type' => get_debug_type($task), + 'poll_retries' => $pollRetries]; + foreach (['poll_status', 'reason', 'outcome', 'retryable', 'retry_after_seconds'] as $key) { + $value = is_array($response) ? ($response[$key] ?? null) : null; + $summary[$key] = is_string($value) ? substr($value, 0, 96) : (is_scalar($value) ? $value : null); + } + + throw new RuntimeException('Large command batches: no workflow task for '.$case.' ' + .json_encode($summary, JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_SUBSTITUTE)); +} + // Workflow task completions with more than 100 commands, over raw worker // HTTP on protocol 1.20 in a fresh namespace (published Server // WorkerController::completeWorkflowTask, Workflow WorkflowCommandNormalizer @@ -60,11 +80,10 @@ function finishHttpLargeCommandBatches(Client $admin, array $fixture, string $wo if ($registered['status'] !== 201 || $started['status'] !== 201) { throw new RuntimeException('Large command batches setup: '.json_encode([$registered, $started])); } - $task = $raw('POST', 'worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, - 'timeout_seconds' => 5])['response']['task'] ?? null; - if (! is_array($task)) { - throw new RuntimeException('Large command batches: no workflow task for '.$case); - } + $beforeRetries = count(LockPressurePollRetries::$retries); + $poll = $raw('POST', 'worker/workflow-tasks/poll', ['worker_id' => $worker, 'task_queue' => $queue, + 'timeout_seconds' => 5]); + $task = largeCommandBatchesClaim($poll, $case, count(LockPressurePollRetries::$retries) - $beforeRetries); $commands = []; for ($index = 0; $index < $shape['commands']; $index++) { $commands[] = $shape['kind'] === 'timers' diff --git a/tests/Unit/ServerParityCliFetchTest.mjs b/tests/Unit/ServerParityCliFetchTest.mjs new file mode 100644 index 000000000..801e06686 --- /dev/null +++ b/tests/Unit/ServerParityCliFetchTest.mjs @@ -0,0 +1,132 @@ +// The Shared Server fixtures producer fetches CLI 2.2.0's dw.phar once (#1103): +// the release download URL first, then the release-asset API URL only when +// that transfer fails; any completed transfer must match the committed size +// and SHA-256, with no second source after a mismatch. curl and timeout are +// replaced by recording fakes, so no network is used. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {createHash} from 'node:crypto'; +import {chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join, resolve} from 'node:path'; +import {spawnSync} from 'node:child_process'; + +const script = resolve('scripts/ci/fetch-parity-cli.sh'); +const workflow = readFileSync(resolve('.github/workflows/server-parity.yml'), 'utf8'); +const PRIMARY = 'https://github.com/durable-workflow/cli/releases/download/2.2.0/dw.phar'; +const API = 'https://api.github.com/repos/durable-workflow/cli/releases/assets/614926795'; +const GOOD = 'genuine parity CLI bytes\n'; +const SAME_SIZE_WRONG = 'Genuine parity CLI bytes\n'; +const SHA = createHash('sha256').update(GOOD).digest('hex'); + +// FAKE_PRIMARY / FAKE_API: ok | wrong | short | fail. Each call is one log line. +const fakeCurl = `#!/usr/bin/env bash +printf '%s\\x1f' "$@" >> "$CALL_LOG"; printf '\\n' >> "$CALL_LOG" +out=""; url="" +while (($#)); do case "$1" in --output) out="$2"; shift 2 ;; https://*) url="$1"; shift ;; *) shift ;; esac; done +case "$url" in ${PRIMARY}) mode="$FAKE_PRIMARY" ;; ${API}) mode="$FAKE_API" ;; *) exit 3 ;; esac +case "$mode" in + ok) printf '%s' "$GOOD" > "$out" ;; + wrong) printf '%s' "$SAME_SIZE_WRONG" > "$out" ;; + short) printf 'x' > "$out" ;; + *) echo "curl: (22) The requested URL returned error: 504" >&2; exit 22 ;; +esac +`; +const fakeTimeout = `#!/usr/bin/env bash +echo "$1" >> "$TIMEOUT_LOG"; shift; exec "$@" +`; + +function run(t, primary, api, args = [SHA, String(GOOD.length)]) { + const dir = mkdtempSync(join(tmpdir(), 'fetch-parity-cli-')); + t.after(() => rmSync(dir, {recursive: true, force: true})); + for (const [name, body] of [['curl', fakeCurl], ['timeout', fakeTimeout]]) { + writeFileSync(join(dir, name), body); + chmodSync(join(dir, name), 0o755); + } + writeFileSync(join(dir, 'calls'), ''); + writeFileSync(join(dir, 'timeouts'), ''); + const out = join(dir, 'parity-cli', 'dw.phar'); + const result = spawnSync('bash', [script, ...args, out], { + encoding: 'utf8', timeout: 10000, + env: {...process.env, PATH: `${dir}:${process.env.PATH}`, CALL_LOG: join(dir, 'calls'), TIMEOUT_LOG: join(dir, 'timeouts'), + FAKE_PRIMARY: primary, FAKE_API: api, GOOD, SAME_SIZE_WRONG}, + }); + const calls = readFileSync(join(dir, 'calls'), 'utf8').split('\n').filter(Boolean) + .map(line => line.split('\x1f').filter(Boolean)); + const timeouts = readFileSync(join(dir, 'timeouts'), 'utf8').trim().split('\n').filter(Boolean); + return {...result, calls, timeouts, out}; +} + +function assertBounded(call, window) { + for (const flag of ['--fail', '--location', '--retry-all-errors']) assert.ok(call.includes(flag), flag); + for (const [flag, value] of [['--proto', '=https'], ['--proto-redir', '=https'], ['--connect-timeout', '10'], + ['--max-time', '60'], ['--retry', '8'], ['--retry-max-time', window]]) { + assert.equal(call[call.indexOf(flag) + 1], value, flag); + } +} + +test('the release download URL alone is used when it yields the committed bytes', t => { + const result = run(t, 'ok', 'ok'); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.calls.length, 1); + assert.equal(result.calls[0].at(-1), PRIMARY); + assert.ok(!result.calls[0].includes('-H'), 'no API header on the release URL'); + assertBounded(result.calls[0], '240'); + assert.deepEqual(result.timeouts, ['300']); + assert.match(result.stdout, /parity CLI transport: release download URL\n/); + assert.match(result.stdout, /dw\.phar: OK\n/); + assert.equal(readFileSync(result.out, 'utf8'), GOOD); +}); + +test('a completed transfer with other bytes fails at once and never falls back', t => { + for (const mode of ['wrong', 'short']) { + const result = run(t, mode, 'ok'); + assert.equal(result.status, 1, mode); + assert.equal(result.calls.length, 1, `${mode}: no second source`); + assert.equal(result.calls[0].at(-1), PRIMARY); + } +}); + +test('only a failed release download falls back to the release-asset API URL', t => { + const result = run(t, 'fail', 'ok'); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.calls.length, 2); + assert.equal(result.calls[1].at(-1), API); + assert.equal(result.calls[1][result.calls[1].indexOf('-H') + 1], 'Accept: application/octet-stream'); + assertBounded(result.calls[0], '240'); + assertBounded(result.calls[1], '120'); + assert.deepEqual(result.timeouts, ['300', '180']); + assert.match(result.stderr, /release download URL failed \(exit 22\); trying the release-asset API URL/); + assert.match(result.stdout, /parity CLI transport: release-asset API URL\n/); + assert.match(result.stdout, /dw\.phar: OK\n/); +}); + +test('the API URL is held to the same size and digest, and both failing fails the step', t => { + for (const [api, expected] of [['wrong', 1], ['short', 1], ['fail', 1]]) { + const result = run(t, 'fail', api); + assert.equal(result.status, expected, api); + assert.equal(result.calls.length, 2, `${api}: exactly two transfers`); + } + assert.match(run(t, 'fail', 'fail').stderr, /both transports failed/); +}); + +test('a malformed digest or size is refused before any transfer', t => { + for (const args of [['', '25'], ['abc', '25'], [SHA, '0'], [SHA, 'x']]) { + const result = run(t, 'ok', 'ok', args); + assert.equal(result.status, 2, JSON.stringify(args)); + assert.deepEqual(result.calls, []); + } +}); + +test('the producer step runs the fetch with the committed digest and the bound asset size', () => { + const producer = workflow.slice(workflow.indexOf(' - name: Fetch the parity CLI once, bound to its committed digest\n')); + const step = producer.slice(0, producer.indexOf('\n - ', 1)); + assert.match(step, /\n {8}timeout-minutes: 9\n/); + assert.ok(step.includes(`bash scripts/ci/fetch-parity-cli.sh \\ + "$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)" 626966 parity-cli/dw.phar`)); + assert.doesNotMatch(step, /curl /, 'every transfer goes through the script'); + // Each corpus job still checks the shared file against the committed digest before use. + const consumer = "printf '%s %s\\n' \"$(jq -r '.visibility.cli.phar_sha256' tests/Fixtures/ServerParity/visibility-current-runs.json)\" parity-consumers/dw.phar | sha256sum -c > parity-evidence/cli-artifact.txt"; + assert.equal(workflow.split(consumer).length - 1, 3); + assert.match(workflow, /\n {6}- scripts\/ci\/\*\*\n/); +}); diff --git a/tests/Unit/ServerParityLargeCommandBatchesProbeTest.php b/tests/Unit/ServerParityLargeCommandBatchesProbeTest.php new file mode 100644 index 000000000..c95644117 --- /dev/null +++ b/tests/Unit/ServerParityLargeCommandBatchesProbeTest.php @@ -0,0 +1,67 @@ +getMessage()); + self::assertStringNotContainsString('SECRET', $error->getMessage()); + self::assertLessThan(1024, strlen($error->getMessage())); + + return json_decode(substr($error->getMessage(), strlen($prefix)), true, flags: JSON_THROW_ON_ERROR); + } + self::fail('A response without a task was accepted.'); + } + + public function test_a_valid_claim_is_returned_unchanged(): void + { + $task = ['task_id' => 'task-1', 'lease_owner' => 'worker-1', 'workflow_task_attempt' => 1, + 'history_events' => [['payload' => 'SECRET-PAYLOAD']]]; + self::assertSame($task, \largeCommandBatchesClaim(['status' => 200, 'response' => ['task' => $task]], 'timers', 3)); + self::assertSame([], \largeCommandBatchesClaim(['status' => 200, 'response' => ['task' => []]], 'timers', 0)); + } + + public function test_a_final_lock_refusal_keeps_its_own_retry_count_and_hint(): void + { + $result = $this->refusal(['status' => 503, 'response' => ['task' => null, 'poll_status' => 'deferred', + 'reason' => 'backend_lock_pressure', 'outcome' => 'deferred', 'retryable' => true, 'retry_after_seconds' => 1, + 'message' => 'SECRET-MESSAGE', 'server_capabilities' => ['SECRET-CAPABILITY']]], 20); + self::assertSame(503, $result['http_status']); + self::assertSame('backend_lock_pressure', $result['reason']); + self::assertSame([20, true, 1], [$result['poll_retries'], $result['retryable'], $result['retry_after_seconds']]); + } + + public function test_an_empty_success_and_missing_task_remain_distinguishable(): void + { + $empty = $this->refusal(['status' => 200, 'response' => ['poll_status' => 'empty', 'task' => null]]); + self::assertSame([200, 'empty', true, 'null'], + [$empty['http_status'], $empty['poll_status'], $empty['task_present'], $empty['task_type']]); + $missing = $this->refusal(['status' => 409, 'response' => ['reason' => 'stale_registration']]); + self::assertSame([409, false, 'stale_registration'], + [$missing['http_status'], $missing['task_present'], $missing['reason']]); + } + + public function test_malformed_and_large_metadata_cannot_fill_the_failure_record(): void + { + $result = $this->refusal(['status' => 503, 'response' => ['task' => 'SECRET-TASK', + 'poll_status' => str_repeat('x', 100000), 'reason' => ['SECRET-NESTED'], + 'headers' => ['Authorization' => 'SECRET-TOKEN'], 'history_events' => ['SECRET-HISTORY']]]); + self::assertSame('string', $result['task_type']); + self::assertSame(96, strlen($result['poll_status'])); + self::assertNull($result['reason']); + self::assertSame('string', $this->refusal(['status' => 502, 'response' => 'SECRET-BODY'])['response_type']); + } +} diff --git a/tests/Unit/Support/composer-sinks.json b/tests/Unit/Support/composer-sinks.json index 8457401a9..365ec0184 100644 --- a/tests/Unit/Support/composer-sinks.json +++ b/tests/Unit/Support/composer-sinks.json @@ -19,7 +19,7 @@ }, { "path": ".github/workflows/phpunit-feature.yml", - "line": "-lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify \"$CORPUS_BASE_REF^{commit}\"; php scripts/ci/check-worker-openapi-evolution.php \"$CORPUS_BASE_REF\"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref \"$CORPUS_BASE_REF\" --verify-counterfactual'", + "line": "-lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify \"$CORPUS_BASE_REF^{commit}\"; php scripts/ci/check-worker-openapi-evolution.php \"$CORPUS_BASE_REF\"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php tests/Unit/ServerParityLargeCommandBatchesProbeTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref \"$CORPUS_BASE_REF\" --verify-counterfactual'", "count": 2, "confinement": "Runs in the source-free corpus tool image. The feature step checks its system bundle, hashed directory and Composer selection before fresh source and dependency installation, on cold builds and cache hits.", "guards": [ diff --git a/tests/Unit/Support/composer_trust_test.py b/tests/Unit/Support/composer_trust_test.py index 5a97ba8bb..70e9362c2 100644 --- a/tests/Unit/Support/composer_trust_test.py +++ b/tests/Unit/Support/composer_trust_test.py @@ -411,12 +411,54 @@ def test_a_locked_install_retries_transient_download_failures(self) -> None: result, runs = self.retrying(3, rate_limited, "--rm", "--", "install") self.assertEqual((100, 3), (result.returncode, runs), "three attempts at most, then Composer's own status") + # Composer 2.10 reports a transport failure as `curl error N while + # downloading URL`, not as `could not be downloaded (...)`. These lines are + # verbatim from the Shared Server fixtures runs of #1104 (Server #1103). + CURL_56 = ("Failed to download psr/http-message from dist: curl error 56 while downloading " + "https://api.github.com/repos/php-fig/http-message/zipball/402d35bcb92c70c026d1a6a9883f06b2ead23d71: " + "Connection died, tried 5 times before giving up") + CURL_28 = ("Failed to download psr/container from dist: curl error 28 while downloading " + "https://api.github.com/repos/php-fig/container/zipball/c71ecc56dfe541dbd90c5360474fbc405f8d5963: " + "Operation timed out after 12877 milliseconds with 0 bytes received") + + def test_a_locked_install_retries_curl_transport_failures(self) -> None: + for failure in (self.CURL_56, self.CURL_28): + with self.subTest(failure[:60]): + result, runs = self.retrying(2, failure, "--rm", "--", "install", "--no-dev") + self.assertEqual((0, 3), (result.returncode, runs), result.stderr) + self.assertEqual(2, result.stderr.count("transient download failure")) + + result, runs = self.retrying(3, failure, "--rm", "--", "install") + self.assertEqual((100, 3), (result.returncode, runs), "three attempts at most") + + def test_http_429_and_5xx_retry_with_or_without_a_reason_phrase(self) -> None: + for status in ("HTTP/2 429 ", "HTTP/2 500 ", "HTTP/2 504 ", "HTTP/1.1 503 Service Unavailable", + "HTTP/1.1 504 Gateway Timeout", "HTTP/1.1 429 Too Many Requests"): + with self.subTest(status): + failure = f'The "https://api.github.com/repos/a/b/zipball/c" file could not be downloaded ({status})' + result, runs = self.retrying(1, failure, "--rm", "--", "install") + self.assertEqual((0, 2), (result.returncode, runs), result.stderr) + + def test_tls_and_other_statuses_are_not_retried(self) -> None: + for failure in ( + "Failed to download a/b from dist: curl error 60 while downloading https://api.github.com/x: " + "SSL certificate problem: unable to get local issuer certificate", + "Failed to download a/b from dist: curl error 22 while downloading https://api.github.com/x: " + "The requested URL returned error: 404", + 'The "https://api.github.com/x" file could not be downloaded (HTTP/2 404 )', + 'The "https://api.github.com/x" file could not be downloaded (HTTP/1.1 403 Forbidden)'): + with self.subTest(failure[:70]): + result, runs = self.retrying(1, failure, "--rm", "--", "install") + self.assertEqual((100, 1), (result.returncode, runs), result.stderr) + def test_other_failures_commands_and_kept_containers_are_not_retried(self) -> None: rate_limited = 'The "https://codeload.github.com/a" file could not be downloaded (HTTP/2 429 )' for failure, arguments in ( ("Your lock file does not contain a compatible set of packages.", ("--rm", "--", "install")), (rate_limited, ("--rm", "--", "update")), - (rate_limited, ("--name", "kept-sdk", "--", "install"))): + (rate_limited, ("--name", "kept-sdk", "--", "install")), + (self.CURL_56, ("--rm", "--", "update")), + (self.CURL_56, ("--name", "kept-sdk", "--", "install"))): with self.subTest(arguments): result, runs = self.retrying(1, failure, *arguments) self.assertEqual((100, 1), (result.returncode, runs), result.stderr)