diff --git a/.github/workflows/phpunit-feature.yml b/.github/workflows/phpunit-feature.yml index 3e1ff9829..be5ac76b9 100644 --- a/.github/workflows/phpunit-feature.yml +++ b/.github/workflows/phpunit-feature.yml @@ -175,6 +175,9 @@ jobs: DW_REQUIRE_COMPOSER_TRUST_TEST: '1' run: python3 -m unittest tests/Unit/Support/composer_trust_test.py tests/Unit/Support/release_composer_audit_test.py + - name: Prove profile SDK installs use only a verified job-scoped cache or the unchanged install + run: python3 -m unittest tests/Unit/Support/sdk_composer_cache_test.py + - name: Prove the image installs and checks the fixed Brotli decoder run: python3 -m unittest tests/Unit/Support/brotli_trust_test.py @@ -392,7 +395,7 @@ jobs: -w /app \ --entrypoint sh \ durable-workflow-server-corpus-validator \ - -lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify "$CORPUS_BASE_REF^{commit}"; php scripts/ci/check-worker-openapi-evolution.php "$CORPUS_BASE_REF"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref "$CORPUS_BASE_REF" --verify-counterfactual' + -lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify "$CORPUS_BASE_REF^{commit}"; php scripts/ci/check-worker-openapi-evolution.php "$CORPUS_BASE_REF"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref "$CORPUS_BASE_REF" --verify-counterfactual' - name: Select production image qualification for changed build inputs id: image_inputs diff --git a/.gitignore b/.gitignore index 077a501e9..4f03cde0f 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ yarn-error.log /rust/target/ /corrected-marker-adapters/ /parity-corrected-state/ +/parity-composer-cache-* /parity-external-consumers-state/ /parity-upgrade-source-state/ /scripts/conformance/server-parity/corrected-markers/php/vendor/ diff --git a/scripts/conformance/server-parity/range-repair-profile.sh b/scripts/conformance/server-parity/range-repair-profile.sh index 068ebec60..22876eaa2 100644 --- a/scripts/conformance/server-parity/range-repair-profile.sh +++ b/scripts/conformance/server-parity/range-repair-profile.sh @@ -51,8 +51,24 @@ case "$backend" in database=(-e DB_CONNECTION=sqlite -v "$state:/range-state") ;; *) echo 'Choose mysql, pgsql or sqlite' >&2; exit 2 ;; esac +# Install from this job's verified read-only dist cache (sdk-composer-cache.sh); +# when no verified cache exists, run the unchanged authenticated install. +set +e +bash scripts/conformance/server-parity/sdk-composer-cache.sh install "$state/sdk" "${limits[@]}" \ + > "$evidence/sdk-install.txt" 2>&1 +cached=$? +set -e +case "$cached" in + 0) ;; + 3) + rm -rf "$state/sdk/vendor" bash scripts/ci/run-composer.sh --name "$resource-sdk" "${limits[@]}" -v "$state/sdk:/sdk" -w /sdk -- \ install --no-interaction --no-progress --no-cache > "$evidence/sdk-install.txt" 2>&1 + ;; + *) echo "Refusing the cache-backed SDK install (exit $cached); see $evidence/sdk-install.txt" >&2; exit 1 ;; +esac +# The job log records which install produced this profile's SDK. +if test "$cached" = 0; then echo "SDK install ($tag): verified job cache, offline"; else echo "SDK install ($tag): authenticated install (no verified cache)"; fi printf '%s %s\n' "$(jq -r '.sdk_php_composer_lock_sha256' "$manifest")" "$state/sdk/composer.lock" \ | sha256sum -c > "$evidence/sdk-lock.txt" reader_tests=(tests/Unit/ServerParityRangeRepairContractTest.mjs) diff --git a/scripts/conformance/server-parity/ready-age-claim.php b/scripts/conformance/server-parity/ready-age-claim.php new file mode 100644 index 000000000..9c53b5d74 --- /dev/null +++ b/scripts/conformance/server-parity/ready-age-claim.php @@ -0,0 +1,72 @@ + ['method' => 'POST', 'header' => implode("\r\n", $headers), + 'content' => json_encode($body, JSON_THROW_ON_ERROR), 'timeout' => 5, 'ignore_errors' => true]]); + $response = file_get_contents(rtrim($url, '/').$path, false, $context); + $status = $http_response_header[0] ?? ''; + $GLOBALS['readyAgeLastHttpStatus'] = substr($status, 0, 64); + if ($response === false || preg_match('/^HTTP\/\S+ 2[0-9]{2}\b/', $status) !== 1) { + throw new RuntimeException('Ready-age source request refused: '.$path.' '.$status); + } + + return json_decode($response, true, flags: JSON_THROW_ON_ERROR); +} + +function readyAgeClaim(array $response, array $context = []): array +{ + $task = $response['task'] ?? null; + if (($response['server_capabilities']['claimed_task_ready_age'] ?? null) !== true || ! is_array($task) + || ! is_int($task['ready_age_ms'] ?? null) || $task['ready_age_ms'] < 0 || $task['ready_age_ms'] > 9007199254740991) { + throw new RuntimeException('Native successful claim lacks its declared literal ready-age value. ' + .json_encode(readyAgeRefusalShape($response, $context), JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES)); + } + $projection = array_intersect_key($task, array_flip(['task_id', 'run_id', 'workflow_task_attempt', 'activity_attempt_id', 'attempt_number', 'ready_age_ms'])); + if (! is_string($projection['task_id'] ?? null) || ! is_string($projection['run_id'] ?? null)) { + throw new RuntimeException('Ready-age claim lost its original task/run identity.'); + } + + // JSONB may reorder object keys. Field names and literal values are exact. + ksort($projection); + + return $projection; +} + +// Bounded facts for a refused claim (Server #1096): statuses, key names, +// types and the two ready-age values only. No payload, token or other value. +function readyAgeRefusalShape(array $response, array $context): array +{ + $scalar = static fn (mixed $value): array => ['present' => $value !== null, 'type' => get_debug_type($value), + 'value' => is_bool($value) || is_int($value) ? $value : null]; + $keys = static fn (array $value): array => array_slice(array_map('strval', array_keys($value)), 0, 64); + $capabilities = is_array($response['server_capabilities'] ?? null) ? $response['server_capabilities'] : []; + $task = $response['task'] ?? null; + $shape = [ + 'http_status' => is_string($GLOBALS['readyAgeLastHttpStatus'] ?? null) ? $GLOBALS['readyAgeLastHttpStatus'] : null, + 'poll_status' => is_string($response['poll_status'] ?? null) ? substr($response['poll_status'], 0, 64) : null, + 'reason' => is_string($response['reason'] ?? null) ? substr($response['reason'], 0, 64) : null, + 'response_keys' => $keys($response), + 'task_type' => get_debug_type($task), + 'task_keys' => is_array($task) ? $keys($task) : null, + 'claimed_task_ready_age' => $scalar($capabilities['claimed_task_ready_age'] ?? null), + 'ready_age_ms' => $scalar(is_array($task) ? ($task['ready_age_ms'] ?? null) : null), + ]; + if (isset($context['saved_task_id'])) { + $shape['same_task_id'] = is_array($task) && ($task['task_id'] ?? null) === $context['saved_task_id']; + } + foreach (['phase', 'kind', 'seconds_since_prepare'] as $field) { + if (array_key_exists($field, $context)) { + $shape[$field] = $context[$field]; + } + } + + return $shape; +} diff --git a/scripts/conformance/server-parity/ready-age-restart.php b/scripts/conformance/server-parity/ready-age-restart.php index 1fbfdd049..197f2e31f 100644 --- a/scripts/conformance/server-parity/ready-age-restart.php +++ b/scripts/conformance/server-parity/ready-age-restart.php @@ -2,41 +2,8 @@ declare(strict_types=1); -// The owning source job kills the native process between these two phases. -// Raw worker HTTP isolates this optional producer field from SDK observations. -function readyAgeRequest(string $url, string $path, array $body): array -{ - $headers = ['Content-Type: application/json', 'X-Namespace: default', - 'X-Durable-Workflow-Protocol-Version: 1.20', - 'Authorization: Bearer '.(getenv('DW_PARITY_TOKEN') ?: 'parity-test-token')]; - $context = stream_context_create(['http' => ['method' => 'POST', 'header' => implode("\r\n", $headers), - 'content' => json_encode($body, JSON_THROW_ON_ERROR), 'timeout' => 5, 'ignore_errors' => true]]); - $response = file_get_contents(rtrim($url, '/').$path, false, $context); - $status = $http_response_header[0] ?? ''; - if ($response === false || preg_match('/^HTTP\/\S+ 2[0-9]{2}\b/', $status) !== 1) { - throw new RuntimeException('Ready-age source request refused: '.$path.' '.$status); - } - - return json_decode($response, true, flags: JSON_THROW_ON_ERROR); -} - -function readyAgeClaim(array $response): array -{ - $task = $response['task'] ?? null; - if (($response['server_capabilities']['claimed_task_ready_age'] ?? null) !== true || ! is_array($task) - || ! is_int($task['ready_age_ms'] ?? null) || $task['ready_age_ms'] < 0 || $task['ready_age_ms'] > 9007199254740991) { - throw new RuntimeException('Native successful claim lacks its declared literal ready-age value.'); - } - $projection = array_intersect_key($task, array_flip(['task_id', 'run_id', 'workflow_task_attempt', 'activity_attempt_id', 'attempt_number', 'ready_age_ms'])); - if (! is_string($projection['task_id'] ?? null) || ! is_string($projection['run_id'] ?? null)) { - throw new RuntimeException('Ready-age claim lost its original task/run identity.'); - } - - // JSONB may reorder object keys. Field names and literal values are exact. - ksort($projection); - - return $projection; -} +// Raw worker HTTP and the claim predicate live in ready-age-claim.php. +require_once __DIR__.'/ready-age-claim.php'; if ($argc !== 4 || ! in_array($argv[1], ['prepare', 'finish'], true)) { throw new InvalidArgumentException('Usage: ready-age-restart.php prepare|finish URL RECEIPT'); @@ -65,7 +32,7 @@ function readyAgeClaim(array $response): array 'arguments' => $client->payloadCodec()->envelope(['source restart'])]]]); $response = readyAgeRequest($url, '/api/worker/activity-tasks/poll', $request); } - $claims[$kind] = ['request' => $request, 'claim' => readyAgeClaim($response)]; + $claims[$kind] = ['request' => $request, 'claim' => readyAgeClaim($response, ['phase' => 'prepare', 'kind' => $kind])]; } file_put_contents($path, json_encode(['prepared_at' => microtime(true), 'claims' => $claims], JSON_THROW_ON_ERROR)."\n"); echo json_encode(['outcome' => 'prepared', 'claims' => array_keys($claims)], JSON_THROW_ON_ERROR)."\n"; @@ -77,7 +44,9 @@ function readyAgeClaim(array $response): array } foreach ($receipt['claims'] as $kind => $saved) { $response = readyAgeRequest($url, '/api/worker/'.$kind.'-tasks/poll', $saved['request']); - $actual = readyAgeClaim($response); + $actual = readyAgeClaim($response, ['phase' => 'finish', 'kind' => $kind, + 'seconds_since_prepare' => round(microtime(true) - $receipt['prepared_at'], 3), + 'saved_task_id' => $saved['claim']['task_id'] ?? null]); if ($actual !== $saved['claim']) { $changed = array_values(array_filter(array_unique([...array_keys($actual), ...array_keys($saved['claim'])]), static fn (string $key): bool => ! array_key_exists($key, $actual) || ! array_key_exists($key, $saved['claim']) || $actual[$key] !== $saved['claim'][$key])); diff --git a/scripts/conformance/server-parity/sdk-composer-cache.sh b/scripts/conformance/server-parity/sdk-composer-cache.sh new file mode 100644 index 000000000..fefcd94d9 --- /dev/null +++ b/scripts/conformance/server-parity/sdk-composer-cache.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# A job-scoped, read-only Composer dist cache for the locked published-SDK +# fixture installs of the parity profiles (Server #1093). +# +# Each profile used to download every pinned dist from GitHub again in its +# own single-attempt named container, so one transient zipball 504 failed the +# profile. Here the first profile in a job warms one cache with an ordinary +# locked install through run-composer.sh (its existing transient retry), and +# records the exact lock, every cached file's SHA-256 and the installed vendor +# tree. Every profile then installs from a verified, read-only copy of those +# bytes with no network, and must reproduce the recorded vendor tree. +# +# Nothing is restored from another job or a global cache: the root lives in +# this job's workspace. The lock, the profiles, the trust context and the +# post-install lock check are unchanged. +# +# Usage: +# sdk-composer-cache.sh warm LOCK_DIR +# sdk-composer-cache.sh install LOCK_DIR [docker run options...] +# Exit status of install: 0 installed from the cache; 3 the cache is not +# usable (the caller runs its unchanged authenticated install instead); +# 4 a cache-backed install did not reproduce the recorded vendor tree (refuse). +set -euo pipefail + +here=$(dirname "$0") +command=${1:?warm or install} +lock_dir=${2:?lock directory} +lock_sha=$(sha256sum < "$lock_dir/composer.lock" | cut -d' ' -f1) +json_sha=$(sha256sum < "$lock_dir/composer.json" | cut -d' ' -f1) +# Keyed by the full lock and manifest, so a different lock never reuses it. +root="$PWD/parity-composer-cache-$lock_sha-$json_sha" +home_config='{"config":{"cache-read-only":true,"cache-dir":"/composer-cache"}}' +# The job that built a root. A root from any other job, restored or left in a +# reused workspace, does not verify and is never used. +job="${GITHUB_RUN_ID:-local}.${GITHUB_RUN_ATTEMPT:-0}.${GITHUB_JOB:-local}.${RUNNER_NAME:-$(hostname)}" + +# A tree's identity: every entry's type and mode, then each regular file's +# SHA-256, in byte order. Ownership and times are deliberately excluded. +listing() { + (cd "$1" && find . -mindepth 1 -printf '%y %m %P\n' | LC_ALL=C sort \ + && find . -type f -print0 | LC_ALL=C sort -z | xargs -0 -r sha256sum) +} +# Only regular files and directories: no symlink, device, socket or pipe. +plain() { test -z "$(find "$1" -mindepth 1 ! -type f ! -type d -print -quit)"; } + +verified() { + test -d "$root" && ! test -L "$root" || return 1 + test "$(cat "$root/manifest/lock" 2>/dev/null)" = "$lock_sha $json_sha" || return 1 + test "$(cat "$root/manifest/job" 2>/dev/null)" = "$job" || return 1 + plain "$root/cache" && plain "$root/home" || return 1 + test "$(cat "$root/home/config.json" 2>/dev/null)" = "$home_config" || return 1 + test -s "$root/manifest/files" && test -s "$root/manifest/vendor" || return 1 + listing "$root/cache" | cmp -s - "$root/manifest/files" +} + +case "$command" in + warm) + exec 9> "$root.lock" + flock 9 + if verified; then exit 0; fi + # An unverifiable existing root is never repaired or reused. + if test -e "$root"; then + echo "sdk-composer-cache: $root exists but does not verify; not using it" >&2 + exit 3 + fi + work=$(mktemp -d "$root.tmp.XXXXXX") + trap 'rm -rf -- "$work"' EXIT + mkdir "$work/project" "$work/cache" "$work/home" "$work/manifest" + cp "$lock_dir/composer.json" "$lock_dir/composer.lock" "$work/project/" + chmod 0777 "$work/project" "$work/cache" + chmod 0666 "$work/project/composer.json" "$work/project/composer.lock" + bash scripts/ci/run-composer.sh --rm -v "$work/project:/sdk" \ + -v "$work/cache:/composer-cache" -e COMPOSER_CACHE_DIR=/composer-cache -w /sdk -- \ + install --no-interaction --no-progress + plain "$work/cache" && plain "$work/project/vendor" || exit 3 + printf '%s\n' "$home_config" > "$work/home/config.json" + printf '%s %s\n' "$lock_sha" "$json_sha" > "$work/manifest/lock" + printf '%s\n' "$job" > "$work/manifest/job" + listing "$work/project/vendor" > "$work/manifest/vendor" + rm -rf "$work/project" + # Record the cache's identity after it becomes read-only. + chmod -R a-w "$work/cache" "$work/home" + listing "$work/cache" > "$work/manifest/files" + chmod -R a-w "$work/manifest" + mv "$work" "$root" + trap - EXIT + verified + ;; + install) + shift 2 + if ! bash "$here/sdk-composer-cache.sh" warm "$lock_dir" || ! verified; then + echo "sdk-composer-cache: no verified cache; use the authenticated install" >&2 + exit 3 + fi + # No network: every dist must come from the verified bytes. The + # container removes itself, so no cleanup step owns a new name. + if ! bash scripts/ci/run-composer.sh --rm "$@" --network none -v "$lock_dir:/sdk" \ + -v "$root/cache:/composer-cache:ro" -v "$root/home:/composer-home:ro" \ + -e COMPOSER_HOME=/composer-home -w /sdk -- install --no-interaction --no-progress; then + echo "sdk-composer-cache: cache-backed install failed; use the authenticated install" >&2 + exit 3 + fi + if ! plain "$lock_dir/vendor" || ! listing "$lock_dir/vendor" | cmp -s - "$root/manifest/vendor"; then + echo "sdk-composer-cache: installed vendor tree differs from the recorded authenticated install" >&2 + exit 4 + fi + ;; + *) + echo 'Usage: sdk-composer-cache.sh warm LOCK_DIR | install LOCK_DIR [docker run options...]' >&2 + exit 2 + ;; +esac diff --git a/scripts/conformance/server-parity/sdk-direction-profile.sh b/scripts/conformance/server-parity/sdk-direction-profile.sh index a0a96e7ac..fbcfb27e4 100644 --- a/scripts/conformance/server-parity/sdk-direction-profile.sh +++ b/scripts/conformance/server-parity/sdk-direction-profile.sh @@ -120,8 +120,24 @@ case "$backend" in database=(-e DB_CONNECTION=sqlite -v "$state:/range-state") ;; *) echo 'Choose mysql, pgsql or sqlite' >&2; exit 2 ;; esac +# Install from this job's verified read-only dist cache (sdk-composer-cache.sh); +# when no verified cache exists, run the unchanged authenticated install. +set +e +bash scripts/conformance/server-parity/sdk-composer-cache.sh install "$state/sdk" "${limits[@]}" \ + > "$evidence/sdk-install.txt" 2>&1 +cached=$? +set -e +case "$cached" in + 0) ;; + 3) + rm -rf "$state/sdk/vendor" bash scripts/ci/run-composer.sh --name "$resource-sdk" "${limits[@]}" -v "$state/sdk:/sdk" -w /sdk -- \ install --no-interaction --no-progress --no-cache > "$evidence/sdk-install.txt" 2>&1 + ;; + *) echo "Refusing the cache-backed SDK install (exit $cached); see $evidence/sdk-install.txt" >&2; exit 1 ;; +esac +# The job log records which install produced this profile's SDK. +if test "$cached" = 0; then echo "SDK install ($tag): verified job cache, offline"; else echo "SDK install ($tag): authenticated install (no verified cache)"; fi printf '%s %s\n' "$(jq -r '.sdk_php_composer_lock_sha256' "$manifest")" "$state/sdk/composer.lock" \ | sha256sum -c > "$evidence/sdk-lock.txt" docker run --name "$resource-tests" "${limits[@]}" --network none --entrypoint node "${shared[@]}" "$image" \ diff --git a/tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php b/tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php new file mode 100644 index 000000000..a0dd86943 --- /dev/null +++ b/tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php @@ -0,0 +1,91 @@ + 'leased', 'server_capabilities' => ['claimed_task_ready_age' => true], + 'task' => ['task_id' => 'task-1', 'run_id' => 'run-1', 'workflow_task_attempt' => 1, 'ready_age_ms' => 42, + 'payload' => 'SECRET-PAYLOAD', 'lease_owner' => 'SECRET-OWNER']]; + } + + private function refusal(array $response, array $context = []): array + { + $GLOBALS['readyAgeLastHttpStatus'] = 'HTTP/1.1 200 OK'; + try { + readyAgeClaim($response, $context); + } catch (RuntimeException $error) { + $message = $error->getMessage(); + self::assertStringStartsWith(self::PREFIX, $message); + self::assertStringNotContainsString('SECRET', $message); + + return json_decode(substr($message, strlen(self::PREFIX)), true, flags: JSON_THROW_ON_ERROR); + } + self::fail('The refused claim was accepted.'); + } + + public function test_a_valid_claim_is_unchanged_and_ignores_diagnostic_context(): void + { + $expected = ['ready_age_ms' => 42, 'run_id' => 'run-1', 'task_id' => 'task-1', 'workflow_task_attempt' => 1]; + self::assertSame($expected, readyAgeClaim($this->claim())); + self::assertSame($expected, readyAgeClaim($this->claim(), ['phase' => 'finish', 'saved_task_id' => 'task-1'])); + } + + public function test_an_empty_answer_records_its_status_and_shape(): void + { + $shape = $this->refusal(['poll_status' => 'empty', 'task' => null, + 'server_capabilities' => ['claimed_task_ready_age' => true]], + ['phase' => 'finish', 'kind' => 'workflow', 'seconds_since_prepare' => 31.5, 'saved_task_id' => 'task-1']); + self::assertSame('HTTP/1.1 200 OK', $shape['http_status']); + self::assertSame('empty', $shape['poll_status']); + self::assertSame('null', $shape['task_type']); + self::assertNull($shape['task_keys']); + self::assertSame(['poll_status', 'task', 'server_capabilities'], $shape['response_keys']); + self::assertFalse($shape['same_task_id']); + self::assertSame(['finish', 'workflow', 31.5], [$shape['phase'], $shape['kind'], $shape['seconds_since_prepare']]); + } + + public function test_a_missing_or_false_capability_is_recorded(): void + { + $response = $this->claim(); + unset($response['server_capabilities']['claimed_task_ready_age']); + $shape = $this->refusal($response); + self::assertSame(['present' => false, 'type' => 'null', 'value' => null], $shape['claimed_task_ready_age']); + $response['server_capabilities']['claimed_task_ready_age'] = false; + self::assertSame(['present' => true, 'type' => 'bool', 'value' => false], $this->refusal($response)['claimed_task_ready_age']); + } + + public function test_a_missing_or_malformed_ready_age_is_recorded_without_other_task_values(): void + { + foreach ([ + 'missing' => [null, ['present' => false, 'type' => 'null', 'value' => null]], + 'string' => ['42', ['present' => true, 'type' => 'string', 'value' => null]], + 'negative' => [-1, ['present' => true, 'type' => 'int', 'value' => -1]], + ] as $case => [$value, $expected]) { + $response = $this->claim(); + if ($value === null) { + unset($response['task']['ready_age_ms']); + } else { + $response['task']['ready_age_ms'] = $value; + } + $shape = $this->refusal($response, ['saved_task_id' => 'task-1']); + self::assertSame($expected, $shape['ready_age_ms'], $case); + self::assertTrue($shape['same_task_id'], $case); + self::assertContains('payload', $shape['task_keys'], $case); + } + } +} diff --git a/tests/Unit/Support/composer-sinks.json b/tests/Unit/Support/composer-sinks.json index 923679a48..8457401a9 100644 --- a/tests/Unit/Support/composer-sinks.json +++ b/tests/Unit/Support/composer-sinks.json @@ -19,7 +19,7 @@ }, { "path": ".github/workflows/phpunit-feature.yml", - "line": "-lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify \"$CORPUS_BASE_REF^{commit}\"; php scripts/ci/check-worker-openapi-evolution.php \"$CORPUS_BASE_REF\"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref \"$CORPUS_BASE_REF\" --verify-counterfactual'", + "line": "-lc 'set -eu; tar --no-same-owner -xf - -C /app; cp -a /app/workflow-package /workflow; php scripts/ci/prepare-release-workflow-composer-metadata.php; composer update durable-workflow/workflow --with-dependencies --minimal-changes --no-install --no-scripts --no-autoloader --no-interaction --no-progress --prefer-dist; composer install --no-interaction --no-progress --prefer-dist; git --version; vendor/bin/phpunit --version; git rev-parse --verify \"$CORPUS_BASE_REF^{commit}\"; php scripts/ci/check-worker-openapi-evolution.php \"$CORPUS_BASE_REF\"; vendor/bin/phpunit tests/Feature tests/Unit/NativeReadyOriginSchemaTest.php tests/Unit/NativeScopedDispatchSchemaTest.php tests/Unit/NativePostgresScopedDispatchSchemaTest.php tests/Unit/WorkerProtocolOpenApiContractTest.php tests/Unit/NexusContractTest.php tests/Unit/NativePublishedComposeContractTest.php tests/Unit/PublishedServerAutoSelectionTest.php tests/Unit/CodecRegressionCorpusTest.php tests/Unit/PhpunitFeatureWorkflowContractTest.php tests/Unit/OpenApiDocumentEvolutionTest.php tests/Unit/NamespaceConformanceRunnerContractTest.php tests/Unit/NamespaceRuntimeContractTest.php tests/Unit/SchedulesConformanceRunnerContractTest.php tests/Unit/SchedulesRuntimeContractTest.php tests/Unit/ServerParitySdkDirectionResponseSummaryTest.php tests/Unit/ServerParitySdkQueryTransportTest.php tests/Unit/ServerParitySdkUpdateTransportTest.php tests/Unit/ServerParityLockPressureRetryTest.php tests/Unit/ServerParityManyUpdatesRetryTest.php tests/Unit/ServerParityReadyAgeRestartDiagnosticsTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImageContinuationTest.php tests/Unit/ReleaseImageImmutabilityGuardTest.php --no-progress --colors=never; vendor/bin/phpunit tests/Unit/ReleaseImagePublishWorkflowContractTest.php --filter test_php_security_audits_fail_before_release_publication --no-progress --colors=never; python3 scripts/ci/validate-regression-corpus.py --base-ref \"$CORPUS_BASE_REF\" --verify-counterfactual'", "count": 2, "confinement": "Runs in the source-free corpus tool image. The feature step checks its system bundle, hashed directory and Composer selection before fresh source and dependency installation, on cold builds and cache hits.", "guards": [ @@ -1532,6 +1532,28 @@ "distance": 8 } ] + }, + { + "path": "scripts/conformance/server-parity/sdk-composer-cache.sh", + "line": "bash scripts/ci/run-composer.sh --rm -v \"$work/project:/sdk\" \\", + "count": 1, + "confinement": "scripts/ci/run-composer.sh runs Composer in the pinned image after Alpine's fixed Brotli decoder (brotli-safe-composer.sh), never in the image under test. It sets COMPOSER_CAFILE to the checked store and COMPOSER_CAPATH to an empty directory after the caller's options, which may not name either or an entrypoint. This warm run is the job-scoped cache's one authenticated locked install; its dist cache is recorded by SHA-256 and made read-only.", + "guards": [ + { + "type": "composer-wrapper" + } + ] + }, + { + "path": "scripts/conformance/server-parity/sdk-composer-cache.sh", + "line": "if ! bash scripts/ci/run-composer.sh --rm \"$@\" --network none -v \"$lock_dir:/sdk\" \\", + "count": 1, + "confinement": "scripts/ci/run-composer.sh runs Composer in the pinned image after Alpine's fixed Brotli decoder (brotli-safe-composer.sh), never in the image under test. It sets COMPOSER_CAFILE to the checked store and COMPOSER_CAPATH to an empty directory after the caller's options, which may not name either or an entrypoint. This cache-backed install runs with no network from the verified read-only cache and must reproduce the recorded vendor tree.", + "guards": [ + { + "type": "composer-wrapper" + } + ] } ] } diff --git a/tests/Unit/Support/sdk_composer_cache_test.py b/tests/Unit/Support/sdk_composer_cache_test.py new file mode 100644 index 000000000..c3359b43a --- /dev/null +++ b/tests/Unit/Support/sdk_composer_cache_test.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Causal controls for the job-scoped SDK Composer cache (Server #1093). + +The real helper runs against a fake run-composer.sh that models the two +container installs from their mounts, so these controls need no Docker or +network. The actual cold-versus-cache install identity is shown by the +profile jobs themselves, which compare every installed vendor file. +""" +import os +from pathlib import Path +import shutil +import stat +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[3] +HELPER = ROOT / "scripts/conformance/server-parity/sdk-composer-cache.sh" +PROFILES = [ROOT / "scripts/conformance/server-parity/sdk-direction-profile.sh", + ROOT / "scripts/conformance/server-parity/range-repair-profile.sh"] +FALLBACK = ('bash scripts/ci/run-composer.sh --name "$resource-sdk" "${limits[@]}" -v "$state/sdk:/sdk" -w /sdk -- \\\n' + ' install --no-interaction --no-progress --no-cache > "$evidence/sdk-install.txt" 2>&1\n') + +# Models a locked install. A warm run (writable cache) "downloads" each dist +# into the cache; a cache-backed run (read-only cache, no network) extracts +# only what the cache holds. Every call is logged. +FAKE = r'''#!/usr/bin/env bash +set -euo pipefail +echo "$*" >> "$FAKE_LOG" +sdk= cache= ro= offline= +while (($#)) && [[ $1 != -- ]]; do + case "$1" in + -v) shift; src=${1%%:*}; rest=${1#*:}; dst=${rest%%:*} + case "$dst" in /sdk) sdk=$src ;; /composer-cache) cache=$src; [[ $rest == *:ro ]] && ro=1 ;; esac ;; + --network) shift; [[ $1 == none ]] && offline=1 ;; + esac + shift +done +if [[ -z $ro ]]; then + [[ -z ${FAKE_WARM_FAIL:-} ]] || { echo "could not be downloaded (HTTP/2 504 )" >&2; exit 100; } + mkdir -p "$cache/files/acme/lib" "$sdk/vendor/acme/lib" "$sdk/vendor/composer" + printf 'dist-lib\n' > "$cache/files/acme/lib/1.zip" + printf 'code-lib\n' > "$sdk/vendor/acme/lib/Lib.php" + printf '{"packages":["acme/lib"]}\n' > "$sdk/vendor/composer/installed.json" + [[ -z ${FAKE_WARM_LINK:-} ]] || ln -s 1.zip "$cache/files/acme/lib/alias.zip" + exit 0 +fi +[[ -n $offline ]] || { echo "cache-backed install must be offline" >&2; exit 9; } +[[ -z ${FAKE_INSTALL_FAIL:-} ]] || exit 1 +test -f "$cache/files/acme/lib/1.zip" || exit 1 +mkdir -p "$sdk/vendor/acme/lib" "$sdk/vendor/composer" +sed 's/dist-/code-/' "$cache/files/acme/lib/1.zip" > "$sdk/vendor/acme/lib/Lib.php" +printf '{"packages":["acme/lib"]}\n' > "$sdk/vendor/composer/installed.json" +[[ -z ${FAKE_DRIFT:-} ]] || printf 'extra\n' > "$sdk/vendor/acme/lib/Extra.php" +[[ -z ${FAKE_EXEC:-} ]] || chmod 0755 "$sdk/vendor/acme/lib/Lib.php" +[[ -z ${FAKE_LINK:-} ]] || ln -s Lib.php "$sdk/vendor/acme/lib/Alias.php" +''' + + +class SdkComposerCacheTest(unittest.TestCase): + def setUp(self): + self.work = Path(tempfile.mkdtemp(prefix="sdk-composer-cache-")) + (self.work / "scripts/ci").mkdir(parents=True) + (self.work / "scripts/conformance/server-parity").mkdir(parents=True) + shutil.copy(HELPER, self.work / "scripts/conformance/server-parity/sdk-composer-cache.sh") + (self.work / "scripts/ci/run-composer.sh").write_text(FAKE) + self.log = self.work / "calls.log" + self.log.touch() + self.lock = self.lock_dir("a") + + def tearDown(self): + for path in self.work.rglob("*"): + if not path.is_symlink(): + path.chmod(path.stat().st_mode | stat.S_IWUSR) + shutil.rmtree(self.work) + + def lock_dir(self, name, lock="lock-a\n"): + directory = self.work / f"state-{name}/sdk" + directory.mkdir(parents=True) + (directory / "composer.json").write_text('{"require":{"acme/lib":"1"}}\n') + (directory / "composer.lock").write_text(lock) + return directory + + def run_helper(self, *args, **env): + return subprocess.run(["bash", "scripts/conformance/server-parity/sdk-composer-cache.sh", *map(str, args)], + cwd=self.work, env={"PATH": os.environ["PATH"], "FAKE_LOG": str(self.log), **env}, + capture_output=True, text=True, timeout=60) + + def calls(self): + return self.log.read_text().splitlines() + + def roots(self): + return sorted(p for p in self.work.glob("parity-composer-cache-*") if not p.name.endswith(".lock")) + + def test_one_warm_serves_every_offline_read_only_install(self): + for name in ("a", "b"): + directory = self.lock if name == "a" else self.lock_dir("b") + result = self.run_helper("install", directory, "--user=1000:1000") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((directory / "vendor/acme/lib/Lib.php").read_text(), "code-lib\n") + warm = [c for c in self.calls() if ":/composer-cache:ro" not in c] + offline = [c for c in self.calls() if ":/composer-cache:ro" in c] + self.assertEqual(len(warm), 1, "the same lock is downloaded once per job") + self.assertEqual(len(offline), 2) + self.assertTrue(all("--network none" in c and "--rm" in c for c in offline)) + root, = self.roots() + for path in root.rglob("*"): + self.assertFalse(path.stat().st_mode & 0o222, f"{path} must be read-only") + + def test_a_tampered_cache_is_never_used(self): + self.assertEqual(self.run_helper("warm", self.lock).returncode, 0) + root, = self.roots() + dist = root / "cache/files/acme/lib/1.zip" + dist.chmod(0o644) + dist.write_text("dist-evil\n") + before = len(self.calls()) + result = self.run_helper("install", self.lock) + self.assertEqual(result.returncode, 3, result.stderr) + self.assertEqual(len(self.calls()), before, "no install runs from an unverified cache") + self.assertFalse((self.lock / "vendor").exists()) + + def test_a_cache_for_another_lock_is_not_substituted(self): + self.assertEqual(self.run_helper("warm", self.lock).returncode, 0) + other = self.lock_dir("other", "lock-other\n") + source, = self.roots() + # Put lock A's verified cache where lock B's would live. + self.run_helper("warm", other, FAKE_WARM_FAIL="1") + target = source.with_name(source.name.replace(*self._key_pair(other))) + if target.exists(): + shutil.rmtree(target) + shutil.copytree(source, target, symlinks=True) + result = self.run_helper("install", other) + self.assertEqual(result.returncode, 3, result.stderr) + self.assertFalse((other / "vendor").exists()) + + def _key_pair(self, other): + import hashlib + digest = lambda p: hashlib.sha256(p.read_bytes()).hexdigest() + return (digest(self.lock / "composer.lock"), digest(other / "composer.lock")) + + def test_a_symlinked_cache_root_is_refused(self): + real = self.work / "elsewhere" + self.assertEqual(self.run_helper("warm", self.lock).returncode, 0) + root, = self.roots() + root.rename(real) + root.symlink_to(real) + self.assertEqual(self.run_helper("install", self.lock).returncode, 3) + + def test_a_failed_warm_leaves_no_cache_and_falls_back(self): + result = self.run_helper("install", self.lock, FAKE_WARM_FAIL="1") + self.assertEqual(result.returncode, 3, result.stderr) + self.assertEqual(self.roots(), []) + self.assertEqual(list(self.work.glob("parity-composer-cache-*.tmp.*")), []) + + def test_a_failed_cache_backed_install_falls_back(self): + self.assertEqual(self.run_helper("install", self.lock, FAKE_INSTALL_FAIL="1").returncode, 3) + + def test_a_drifting_install_is_refused_not_fallen_back(self): + result = self.run_helper("install", self.lock, FAKE_DRIFT="1") + self.assertEqual(result.returncode, 4, result.stderr) + + def test_a_cache_built_by_another_job_is_refused(self): + other_job = {"GITHUB_RUN_ID": "1", "GITHUB_RUN_ATTEMPT": "1", "GITHUB_JOB": "a", "RUNNER_NAME": "r1"} + self.assertEqual(self.run_helper("warm", self.lock, **other_job).returncode, 0) + this_job = {**other_job, "GITHUB_RUN_ID": "2"} + before = len(self.calls()) + result = self.run_helper("install", self.lock, **this_job) + self.assertEqual(result.returncode, 3, result.stderr) + self.assertEqual(len(self.calls()), before, "a restored root is neither used nor rebuilt in place") + self.assertEqual(self.run_helper("install", self.lock, **other_job).returncode, 0) + + def test_a_symlink_or_mode_change_in_the_cache_is_refused(self): + self.assertEqual(self.run_helper("warm", self.lock).returncode, 0) + root, = self.roots() + files = root / "cache/files/acme/lib" + files.chmod(0o755) + (files / "2.zip").symlink_to("1.zip") + self.assertEqual(self.run_helper("install", self.lock).returncode, 3) + (files / "2.zip").unlink() + (files / "1.zip").chmod(0o555) # mode only; bytes unchanged + files.chmod(0o555) + self.assertEqual(self.run_helper("install", self.lock).returncode, 3) + + def test_a_symlink_present_at_warm_is_never_recorded_as_trusted(self): + result = self.run_helper("install", self.lock, FAKE_WARM_LINK="1") + self.assertEqual(result.returncode, 3, result.stderr) + self.assertEqual(self.roots(), []) + + def test_an_installed_mode_drift_or_symlink_is_refused(self): + self.assertEqual(self.run_helper("install", self.lock, FAKE_EXEC="1").returncode, 4) + other = self.lock_dir("link") + self.assertEqual(self.run_helper("install", other, FAKE_LINK="1").returncode, 4) + + def test_profiles_keep_the_unchanged_authenticated_install_as_fallback(self): + for profile in PROFILES: + text = profile.read_text() + self.assertIn('sdk-composer-cache.sh install "$state/sdk" "${limits[@]}"', text, profile.name) + self.assertIn(" 3)\n rm -rf \"$state/sdk/vendor\"\n" + FALLBACK + " ;;\n", text, profile.name) + self.assertIn(' *) echo "Refusing the cache-backed SDK install', text, profile.name) + # The existing pinned-lock check still follows every install. + self.assertGreater(text.index("sha256sum -c > \"$evidence/sdk-lock.txt\""), text.index(FALLBACK)) + + +if __name__ == "__main__": + unittest.main()