diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 9d8ae12db1..c915b6b4bf 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -3,11 +3,14 @@ self-hosted-runner: - aarch64-darwin - aarch64-linux - arm-native-runner + - blacksmith-6vcpu-macos-26 + - blacksmith-12vcpu-macos-26 - blacksmith-2vcpu-ubuntu-2404 - blacksmith-2vcpu-ubuntu-2404-arm - blacksmith-4vcpu-ubuntu-2404 - blacksmith-4vcpu-ubuntu-2404-arm - blacksmith-8vcpu-ubuntu-2404 - blacksmith-32vcpu-ubuntu-2404 + - blacksmith-32vcpu-ubuntu-2404-arm - large-linux-arm - large-linux-x86 diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index ba987904d2..f827dff5f8 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -5,6 +5,10 @@ inputs: description: AWS region for the Nix binary cache S3 bucket required: false default: us-east-1 + installer: + description: upstream (default) or determinate + required: false + default: upstream nix-signing-key: description: Key used to sign uploads to binary cache, required if push-to-cache is true required: false @@ -49,6 +53,7 @@ runs: env: AWS_REGION: ${{ inputs.aws-region }} GITHUB_TOKEN: ${{ github.token }} + INSTALLER: ${{ inputs.installer }} NIX_SIGN_SECRET_KEY: ${{ inputs.nix-signing-key }} PUSH_TO_CACHE: ${{ steps.resolve-push-to-cache.outputs.value }} diff --git a/.github/actions/nix-install-ephemeral/setup-nix.sh b/.github/actions/nix-install-ephemeral/setup-nix.sh index ffba142791..46ca4e89c7 100755 --- a/.github/actions/nix-install-ephemeral/setup-nix.sh +++ b/.github/actions/nix-install-ephemeral/setup-nix.sh @@ -31,6 +31,7 @@ cat >"$tmpdir/nix.conf" <<-EOF extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= max-jobs = 5 + stalled-download-timeout = 5 EOF if [[ -e /dev/kvm ]]; then @@ -46,8 +47,16 @@ fi maybesudo NIXCONFDIR="$nixconfdir" ./setup-github-access.sh >>"$tmpdir/nix.conf" -curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- $daemon --yes --nix-extra-conf-file "$tmpdir/nix.conf" -cat "$nixconfdir/nix.conf" +if [[ ${INSTALLER:-upstream} == determinate ]]; then + echo 'extra-experimental-features = parallel-eval' >>"$tmpdir/nix.conf" + url=https://install.determinate.systems/nix/tag/v3.22.4 + args=(install --no-confirm --extra-conf "$tmpdir/nix.conf") +else + url=https://releases.nixos.org/nix/nix-2.34.6/install + args=("$daemon" --yes --nix-extra-conf-file "$tmpdir/nix.conf") +fi +curl --proto '=https' --tlsv1.2 -sSfL "$url" | sh -s -- "${args[@]}" +cat "$nixconfdir"/nix*.conf # Add nix to PATH for subsequent steps echo "$path" >>"${GITHUB_PATH:?}" diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index e286015cca..143a25e6a1 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -19,7 +19,8 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - nix-eval: + nix-eval-x86_64-linux: + name: Nix Eval (x86_64-linux) runs-on: blacksmith-32vcpu-ubuntu-2404 outputs: packages_matrix: ${{ steps.gen-matrix.outputs.packages_matrix }} @@ -27,36 +28,97 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Install nix + uses: ./.github/actions/nix-install-ephemeral + with: + installer: determinate + push-to-cache: true + nix-signing-key: ${{ secrets.NIX_SIGN_SECRET_KEY }} + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + + - name: Evaluate flake (parallel) + run: | + set -Eeu -o pipefail + ./scripts/parallel-nix-eval-jobs.sh | tee jobs.jsonl + - name: Generate Nix Matrix + id: gen-matrix + run: | + set -Eeu -o pipefail + nix run .#github-matrix -- x86_64-linux >"$GITHUB_OUTPUT" + echo "checks_matrix=$(jq -c .checks matrix.json)" >>"$GITHUB_OUTPUT" + + nix-eval-aarch64-linux: + name: Nix Eval (aarch64-linux) + runs-on: blacksmith-32vcpu-ubuntu-2404-arm + outputs: + packages_matrix: ${{ steps.gen-matrix.outputs.packages_matrix }} + checks_matrix: ${{ steps.gen-matrix.outputs.checks_matrix }} + steps: + - name: Checkout Repo + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Install nix uses: ./.github/actions/nix-install-ephemeral with: + installer: determinate push-to-cache: true nix-signing-key: ${{ secrets.NIX_SIGN_SECRET_KEY }} role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + - name: Evaluate flake (parallel) + run: | + set -Eeu -o pipefail + ./scripts/parallel-nix-eval-jobs.sh | tee jobs.jsonl + - name: Generate Nix Matrix id: gen-matrix run: | set -Eeu -o pipefail - nix run --accept-flake-config .\#github-matrix -- checks legacyPackages + nix run .#github-matrix -- aarch64-linux >"$GITHUB_OUTPUT" + echo "checks_matrix=$(jq -c .checks matrix.json)" >>"$GITHUB_OUTPUT" + + nix-eval-aarch64-darwin: + name: Nix Eval (aarch64-darwin) + runs-on: blacksmith-12vcpu-macos-26 + outputs: + packages_matrix: ${{ steps.gen-matrix.outputs.packages_matrix }} + checks_matrix: ${{ steps.gen-matrix.outputs.checks_matrix }} + steps: + - name: Checkout Repo + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - name: Install nix + uses: ./.github/actions/nix-install-ephemeral + with: + installer: determinate + push-to-cache: true + nix-signing-key: ${{ secrets.NIX_SIGN_SECRET_KEY }} + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + + - name: Evaluate flake (parallel) + run: | + set -Eeu -o pipefail + ./scripts/parallel-nix-eval-jobs.sh | tee jobs.jsonl - sudo systemctl stop nix-daemon.socket || true - sudo systemctl stop nix-daemon.service || true - sudo pkill -9 nix-daemon || true - sleep 2 + - name: Generate Nix Matrix + id: gen-matrix + run: | + set -Eeu -o pipefail + nix run .#github-matrix -- aarch64-darwin >"$GITHUB_OUTPUT" + echo "checks_matrix=$(jq -c .checks matrix.json)" >>"$GITHUB_OUTPUT" nix-build-packages-aarch64-linux: name: >- ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (aarch64-linux) - needs: nix-eval + needs: nix-eval-aarch64-linux runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_linux != null }} strategy: fail-fast: false max-parallel: 25 - matrix: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_linux }} + matrix: ${{ fromJSON(needs.nix-eval-aarch64-linux.outputs.packages_matrix) }} steps: - name: Checkout Repo if: ${{ matrix.attr != '' }} @@ -79,14 +141,13 @@ jobs: ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (aarch64-linux) needs: - - nix-eval + - nix-eval-aarch64-linux - nix-build-packages-aarch64-linux runs-on: ${{ matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_linux != null }} strategy: fail-fast: false max-parallel: 25 - matrix: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_linux }} + matrix: ${{ fromJSON(needs.nix-eval-aarch64-linux.outputs.checks_matrix) }} steps: - name: Checkout Repo if: ${{ matrix.attr != '' }} @@ -108,13 +169,12 @@ jobs: name: >- ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (aarch64-darwin) - needs: nix-eval + needs: nix-eval-aarch64-darwin runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_darwin != null }} strategy: fail-fast: false max-parallel: 25 - matrix: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).aarch64_darwin }} + matrix: ${{ fromJSON(needs.nix-eval-aarch64-darwin.outputs.packages_matrix) }} steps: - name: Checkout Repo if: ${{ matrix.attr != '' }} @@ -140,14 +200,13 @@ jobs: ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (aarch64-darwin) needs: - - nix-eval + - nix-eval-aarch64-darwin - nix-build-packages-aarch64-darwin runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_darwin != null }} strategy: fail-fast: false max-parallel: 25 - matrix: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).aarch64_darwin }} + matrix: ${{ fromJSON(needs.nix-eval-aarch64-darwin.outputs.checks_matrix) }} steps: - name: Checkout Repo if: ${{ matrix.attr != '' }} @@ -169,13 +228,12 @@ jobs: name: >- ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (x86_64-linux) - needs: nix-eval + needs: nix-eval-x86_64-linux runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).x86_64_linux != null }} strategy: fail-fast: false max-parallel: 25 - matrix: ${{ fromJSON(needs.nix-eval.outputs.packages_matrix).x86_64_linux }} + matrix: ${{ fromJSON(needs.nix-eval-x86_64-linux.outputs.packages_matrix) }} steps: - name: Checkout Repo if: ${{ matrix.attr != '' }} @@ -198,14 +256,13 @@ jobs: ${{ matrix.name }}${{ matrix.postgresql_version && format(' - Postgres {0}', matrix.postgresql_version) || '' }} (x86_64-linux) needs: - - nix-eval + - nix-eval-x86_64-linux - nix-build-packages-x86_64-linux runs-on: ${{ matrix.attr != '' && matrix.runs_on.group && matrix.runs_on || matrix.runs_on.labels }} - if: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).x86_64_linux != null }} strategy: fail-fast: false max-parallel: 25 - matrix: ${{ fromJSON(needs.nix-eval.outputs.checks_matrix).x86_64_linux }} + matrix: ${{ fromJSON(needs.nix-eval-x86_64-linux.outputs.checks_matrix) }} steps: - name: Checkout Repo if: ${{ matrix.attr != '' }} @@ -237,24 +294,60 @@ jobs: jq -rn --argjson needs '${{ toJSON(needs) }}' '$needs|to_entries[]|select(.value.result!="success")|"::error::"+.key+": "+.value.result' exit 1 - run-testinfra: - needs: - - nix-eval - - nix-build-packages-aarch64-linux - - nix-build-packages-aarch64-darwin - - nix-build-packages-x86_64-linux + run-testinfra-aarch64-linux: + needs: nix-build-packages-aarch64-linux uses: ./.github/workflows/testinfra-ami-build.yml + with: + arch: arm64 secrets: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - run-tests: + run-testinfra-x86_64-linux: + needs: nix-build-packages-x86_64-linux + uses: ./.github/workflows/testinfra-ami-build.yml + with: + arch: amd64 + secrets: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} + NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} + + report-disk-usage: needs: - - nix-eval - - nix-build-packages-aarch64-linux - - nix-build-packages-aarch64-darwin - - nix-build-packages-x86_64-linux + - run-testinfra-aarch64-linux + - run-testinfra-x86_64-linux + runs-on: blacksmith-2vcpu-ubuntu-2404 + steps: + - name: Download AMI disk usage + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: ami-disk-usage-* + path: ami-disk-usage + merge-multiple: true + + - name: Report AMI disk usage + run: | + cat <>"$GITHUB_STEP_SUMMARY" + ### AMI Root Disk Usage + + | Version | Arch | Human | Bytes | + | ------- | ---- | ----- | ----- | + EOF + cat ami-disk-usage/ami-disk-usage-*.json | + sort -V | + jq -rs '.[]|{version,arch,human,bytes}|"| \(join("|")) |"' >>"$GITHUB_STEP_SUMMARY" + + run-tests-aarch64-linux: + needs: nix-build-packages-aarch64-linux + uses: ./.github/workflows/test.yml + with: + arch: arm64 + + run-tests-x86_64-linux: + needs: nix-build-packages-x86_64-linux uses: ./.github/workflows/test.yml + with: + arch: amd64 docker-image-test: needs: nix-build-packages-aarch64-linux @@ -270,8 +363,11 @@ jobs: needs: - docker-image-test - nix-build-checks - - run-testinfra - - run-tests + - report-disk-usage + - run-testinfra-aarch64-linux + - run-testinfra-x86_64-linux + - run-tests-aarch64-linux + - run-tests-x86_64-linux steps: - name: Check dependent job results if: contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'skipped') diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c484a295eb..5a2f3a351d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,19 @@ name: Test Database on: workflow_dispatch: + inputs: + arch: + type: choice + options: + - amd64 + - arm64 + required: true workflow_call: + inputs: + arch: + description: amd64 or arm64 + type: string + required: true permissions: contents: read @@ -12,6 +24,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + runner: ${{ steps.set-runner.outputs.runner }} steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD @@ -21,17 +34,23 @@ jobs: run: | VERSIONS=$(nix run nixpkgs#yq-go -- -o=json -I=0 '.postgres_major' ansible/vars.yml) echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + - name: Set runner + id: set-runner + env: + ARCH: ${{ inputs.arch }} + run: | + case "$ARCH" in + amd64) runner=large-linux-x86 ;; + arm64) runner=large-linux-arm ;; + *) echo "::error::unknown arch '$ARCH'"; exit 1 ;; + esac + echo "runner=$runner" >> "$GITHUB_OUTPUT" build: needs: prepare strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} - include: - - runner: large-linux-x86 - arch: amd64 - - runner: large-linux-arm - arch: arm64 - runs-on: ${{ matrix.runner }} + runs-on: ${{ needs.prepare.outputs.runner }} timeout-minutes: 180 env: POSTGRES_PORT: 5478 diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 7288358399..17553f04e0 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -2,7 +2,19 @@ name: Testinfra Integration Tests Nix on: workflow_dispatch: + inputs: + arch: + type: choice + options: + - amd64 + - arm64 + required: true workflow_call: + inputs: + arch: + description: amd64 or arm64 + type: string + required: true secrets: DEV_AWS_ROLE: description: 'AWS role for dev environment' @@ -23,6 +35,8 @@ jobs: runs-on: blacksmith-2vcpu-ubuntu-2404 outputs: postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + runner: ${{ steps.set-target.outputs.runner }} + instance_type: ${{ steps.set-target.outputs.instance_type }} steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD @@ -36,20 +50,26 @@ jobs: VERSIONS=$(nix run nixpkgs#yq-go -- -o=json -I=0 '.postgres_major' ansible/vars.yml) echo "postgres_versions=$VERSIONS" >> "$GITHUB_OUTPUT" + - name: Set target + id: set-target + env: + ARCH: ${{ inputs.arch }} + run: | + case "$ARCH" in + amd64) runner=blacksmith-2vcpu-ubuntu-2404 instance_type=c6i.4xlarge ;; + arm64) runner=blacksmith-2vcpu-ubuntu-2404-arm instance_type=c6g.4xlarge ;; + *) echo "::error::unknown arch '$ARCH'"; exit 1 ;; + esac + echo "runner=$runner" >> "$GITHUB_OUTPUT" + echo "instance_type=$instance_type" >> "$GITHUB_OUTPUT" + test-ami-nix: needs: gen-matrix strategy: fail-fast: false matrix: postgres_version: ${{ fromJson(needs.gen-matrix.outputs.postgres_versions) }} - target: - - arch: amd64 - instance_type: c6i.4xlarge - runner: blacksmith-2vcpu-ubuntu-2404 - - arch: arm64 - instance_type: c6g.4xlarge - runner: blacksmith-2vcpu-ubuntu-2404-arm - runs-on: ${{ matrix.target.runner }} + runs-on: ${{ needs.gen-matrix.outputs.runner }} timeout-minutes: 150 steps: @@ -91,18 +111,18 @@ jobs: id: build-ami uses: ./.github/actions/build-ami with: - ami_name_prefix: "supabase-postgres-${{ github.run_id }}-${{ matrix.target.arch }}" + ami_name_prefix: "supabase-postgres-${{ github.run_id }}-${{ inputs.arch }}" ami_regions: '["${{ env.AWS_REGION }}"]' - arch: ${{ matrix.target.arch }} + arch: ${{ inputs.arch }} git_sha: ${{ github.sha }} packages_git_sha: ${{ steps.resolve-git-sha.outputs.sha }} - instance_type: ${{ matrix.target.instance_type }} + instance_type: ${{ needs.gen-matrix.outputs.instance_type }} postgres_version: ${{ matrix.postgres_version }} region: ${{ env.AWS_REGION }} - name: Save AMI disk usage env: - ARCH: ${{ matrix.target.arch }} + ARCH: ${{ inputs.arch }} DISK_USAGE_JSON: ${{ steps.build-ami.outputs.disk_usage_json }} POSTGRES_VERSION: ${{ matrix.postgres_version }} run: >- @@ -116,7 +136,7 @@ jobs: - name: Upload AMI disk usage uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: ami-disk-usage-${{ matrix.postgres_version }}-${{ matrix.target.arch }} + name: ami-disk-usage-${{ matrix.postgres_version }}-${{ inputs.arch }} path: ami-disk-usage-*.json overwrite: true retention-days: 1 @@ -188,26 +208,3 @@ jobs: else echo "No stage 2 AMI to clean up" fi - - report-disk-usage: - needs: test-ami-nix - runs-on: blacksmith-2vcpu-ubuntu-2404 - steps: - - name: Download AMI disk usage - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - pattern: ami-disk-usage-* - path: ami-disk-usage - merge-multiple: true - - - name: Report AMI disk usage - run: | - cat <>"$GITHUB_STEP_SUMMARY" - ### AMI Root Disk Usage - - | Version | Arch | Human | Bytes | - | ------- | ---- | ----- | ----- | - EOF - cat ami-disk-usage/ami-disk-usage-*.json | - sort -V | - jq -rs '.[]|{version,arch,human,bytes}|"| \(join("|")) |"' >>"$GITHUB_STEP_SUMMARY" diff --git a/flake.lock b/flake.lock index b48d1c2940..72c7890702 100644 --- a/flake.lock +++ b/flake.lock @@ -131,22 +131,6 @@ "type": "github" } }, - "nix": { - "flake": false, - "locked": { - "lastModified": 1760472641, - "narHash": "sha256-BuKtM7Vr5EcxBXxUENBQPlOBwmNd5mkTRkSmlJi/iQ4=", - "owner": "NixOS", - "repo": "nix", - "rev": "4041bfdb401ad6d1c31a292fab90392254be506a", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "nix", - "type": "github" - } - }, "nix-darwin": { "inputs": { "nixpkgs": [ @@ -190,31 +174,6 @@ "type": "github" } }, - "nix-eval-jobs": { - "inputs": { - "flake-parts": [ - "flake-parts" - ], - "nix": "nix", - "nixpkgs": "nixpkgs", - "treefmt-nix": [ - "treefmt-nix" - ] - }, - "locked": { - "lastModified": 1760478325, - "narHash": "sha256-hA+NOH8KDcsuvH7vJqSwk74PyZP3MtvI/l+CggZcnTc=", - "owner": "nix-community", - "repo": "nix-eval-jobs", - "rev": "daa42f9e9c84aeff1e325dd50fda321f53dfd02c", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nix-eval-jobs", - "type": "github" - } - }, "nix2container": { "inputs": { "nixpkgs": [ @@ -237,15 +196,15 @@ }, "nixpkgs": { "locked": { - "lastModified": 315532800, - "narHash": "sha256-vhAtaRMIQiEghARviANBmSnhGz9Qf2IQJ+nQgsDXnVs=", - "rev": "c12c63cd6c5eb34c7b4c3076c6a99e00fcab86ec", + "lastModified": 1767379071, + "narHash": "sha256-3xDI4xtzovwqE/eAxCwmXxUqBg6Yoam2L1u0IwRNhW4=", + "rev": "fb7944c166a3b630f177938e478f0378e64ce108", "type": "tarball", - "url": "https://releases.nixos.org/nixpkgs/nixpkgs-25.11pre877036.c12c63cd6c5e/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre921484.fb7944c166a3/nixexprs.tar.xz" }, "original": { "type": "tarball", - "url": "https://nixos.org/channels/nixpkgs-unstable/nixexprs.tar.xz" + "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz" } }, "nixpkgs-lib": { @@ -279,19 +238,6 @@ "type": "github" } }, - "nixpkgs_2": { - "locked": { - "lastModified": 1767379071, - "narHash": "sha256-3xDI4xtzovwqE/eAxCwmXxUqBg6Yoam2L1u0IwRNhW4=", - "rev": "fb7944c166a3b630f177938e478f0378e64ce108", - "type": "tarball", - "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre921484.fb7944c166a3/nixexprs.tar.xz" - }, - "original": { - "type": "tarball", - "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz" - } - }, "root": { "inputs": { "devshell": "devshell", @@ -301,9 +247,8 @@ "multigres": "multigres", "nix-darwin": "nix-darwin", "nix-editor": "nix-editor", - "nix-eval-jobs": "nix-eval-jobs", "nix2container": "nix2container", - "nixpkgs": "nixpkgs_2", + "nixpkgs": "nixpkgs", "nixpkgs-oldstable": "nixpkgs-oldstable", "rust-overlay": "rust-overlay", "treefmt-nix": "treefmt-nix" diff --git a/flake.nix b/flake.nix index 828a384e95..6d490799d3 100644 --- a/flake.nix +++ b/flake.nix @@ -18,9 +18,6 @@ nix-editor.inputs.nixpkgs.follows = "nixpkgs"; nix-editor.inputs.utils.follows = "flake-utils"; nix-editor.url = "github:snowfallorg/nix-editor"; - nix-eval-jobs.inputs.flake-parts.follows = "flake-parts"; - nix-eval-jobs.inputs.treefmt-nix.follows = "treefmt-nix"; - nix-eval-jobs.url = "github:nix-community/nix-eval-jobs"; nix2container.inputs.nixpkgs.follows = "nixpkgs"; nix2container.url = "github:nlewo/nix2container"; # Pin to a specific nixpkgs version that has compatible v8 and curl versions diff --git a/nix/checks.nix b/nix/checks.nix index 3fdd5ba2c0..4205208865 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -928,6 +928,7 @@ cli-smoke-test docker-image-inputs docker-image-test + github-matrix goss image-size-analyzer pg_regress diff --git a/nix/devShells.nix b/nix/devShells.nix index a20e92c0bb..5cfba4a83e 100644 --- a/nix/devShells.nix +++ b/nix/devShells.nix @@ -42,6 +42,7 @@ packages = with pkgs; [ coreutils just + jq nix-update #pg_prove shellcheck diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 9cc337e9ff..0bc59a4b10 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -61,9 +61,7 @@ }; docs = pkgs.callPackage ./docs.nix { }; pgbouncer = pkgs.callPackage ../pgbouncer.nix { }; - github-matrix = pkgs.callPackage ./github-matrix { - nix-eval-jobs = inputs'.nix-eval-jobs.packages.default; - }; + github-matrix = pkgs.callPackage ./github-matrix { }; gatekeeper = pkgs.callPackage ./gatekeeper.nix { inherit inputs pkgs; }; supabase-groonga = pkgs.callPackage ../ext/pgroonga/groonga.nix { }; http-mock-server = pkgs.callPackage ./http-mock-server.nix { }; diff --git a/nix/packages/github-matrix/default.nix b/nix/packages/github-matrix/default.nix index 3e1be2df3d..8bf42fb4a3 100644 --- a/nix/packages/github-matrix/default.nix +++ b/nix/packages/github-matrix/default.nix @@ -1,51 +1,45 @@ { + jq, lib, - nix-eval-jobs, - python3Packages, + runtimeShell, + stdenvNoCC, }: -let - pname = "github-matrix"; - - github-action-utils = python3Packages.buildPythonPackage rec { - pname = "github-action-utils"; - version = "1.1.0"; - pyproject = true; - - src = python3Packages.fetchPypi { - inherit pname version; - sha256 = "0q9xrb4jcvbn6954lvpn85gva1yc885ykdqb2q2410cxp280v94a"; - }; - - build-system = with python3Packages; [ setuptools ]; - - meta = with lib; { - description = "Collection of Python functions for GitHub Action Workflow Commands"; - homepage = "https://github.com/saadmk11/github-action-utils"; - license = licenses.mit; - }; - }; -in -python3Packages.buildPythonApplication { - inherit pname; - version = "0.1.0"; - pyproject = false; +stdenvNoCC.mkDerivation { + pname = "github-matrix"; + version = "0.2.0"; src = ./.; - propagatedBuildInputs = [ - github-action-utils - python3Packages.result - ]; + nativeCheckInputs = [ jq ]; - makeWrapperArgs = [ "--suffix PATH : ${lib.makeBinPath [ nix-eval-jobs ]}" ]; + dontBuild = true; + doCheck = true; - nativeCheckInputs = with python3Packages; [ - pytestCheckHook - pytest-mypy - ]; + checkPhase = '' + runHook preCheck + bash tests/run.sh + runHook postCheck + ''; installPhase = '' - install -Dm755 github_matrix.py "$out/bin/${pname}" + runHook preInstall + + install -Dm644 github-matrix.jq -t $out/share/github-matrix + mkdir -p $out/bin + cat >$out/bin/github-matrix <<'EOF' + #!${runtimeShell} + # usage: github-matrix < jobs.jsonl + set -euo pipefail + exec ${lib.getExe jq} -r -s --arg system "''${1:?system}" -f ${placeholder "out"}/share/github-matrix/github-matrix.jq + EOF + chmod 755 $out/bin/github-matrix + + runHook postInstall ''; + + meta = { + description = "Turn nix-eval-jobs-shaped JSONL into GitHub Actions build matrices"; + mainProgram = "github-matrix"; + }; } diff --git a/nix/packages/github-matrix/github-matrix.jq b/nix/packages/github-matrix/github-matrix.jq new file mode 100644 index 0000000000..7a940b8cd3 --- /dev/null +++ b/nix/packages/github-matrix/github-matrix.jq @@ -0,0 +1,59 @@ +# nix-eval-jobs-shaped JSONL (slurped) -> GitHub Actions build matrices for one system. +# +# jq -r -s --arg system x86_64-linux -f github-matrix.jq jobs.jsonl +# +# Evaluation errors are printed as ::error annotations and exit 1. + +def labels(l): { labels: [l] }; + +def runner: + (.requiredSystemFeatures // []) as $f + | (.system | split("-")) as [$arch, $os] + | ($f | any(. == "apple-virt" or . == "kvm")) as $virt + | ($f | any(. == "big-parallel")) as $large + | if $virt and $os == "darwin" then { group: "self-hosted-runners-nix", labels: ["aarch64-darwin"] } + elif $virt and $arch == "aarch64" then labels("arm-native-runner") + elif $virt then labels("blacksmith-16vcpu-ubuntu-2404") + elif $os == "darwin" then labels(if $large then "blacksmith-12vcpu-macos-26" else "blacksmith-6vcpu-macos-26" end) + elif $arch == "aarch64" then labels(if $large then "blacksmith-32vcpu-ubuntu-2404-arm" else "blacksmith-8vcpu-ubuntu-2404-arm" end) + else labels(if $large then "blacksmith-32vcpu-ubuntu-2404" else "blacksmith-8vcpu-ubuntu-2404" end) + end; + +def job: + (.attr | split(".")) as $p + | { attr, name, system, runs_on: runner } + + (if $p[-2] == "exts" then { postgresql_version: ($p[-3] | split("_")[-1]) } else {} end); + +# one annotation per distinct error message +def annotations: + group_by(.error) + | map( + "::error title=Nix Evaluation Error::" + + (if length > 1 then "Affected attributes (\(length)): \(map(.attr) | join(", "))" + else "Attribute: \(.[0].attr)" end) + + "%0A%0A" + (.[0].error | gsub("\n"; "%0A")) + ) + | .[]; + +def entries(kind): + map(select(.attr | startswith(kind + "."))) + | if length > 0 then . else + [ { attr: "", name: "no \(if kind == "checks" then "checks" else "packages" end) to build", + system: $system, runs_on: labels("ubuntu-latest") } ] + end + | { include: . }; + +# first occurrence wins, input order preserved (unique_by would sort) +def dedupe: reduce .[] as $j ([]; if any(.[]; .drvPath == $j.drvPath) then . else . + [$j] end); + +def matrix: + dedupe + | map(select(.cacheStatus == "notBuilt")) + # no runners that can run the NixOS VM tests on darwin + | map(select((.system == "aarch64-darwin" and ((.requiredSystemFeatures // []) | any(. == "nixos-test"))) | not)) + | map(job) + | { packages: entries("legacyPackages"), checks: entries("checks") }; + +map(select(.error != null)) as $errors +| if ($errors | length) > 0 then ($errors | annotations), ("" | halt_error(1)) + else map(select(.error == null)) | matrix end diff --git a/nix/packages/github-matrix/github_matrix.py b/nix/packages/github-matrix/github_matrix.py deleted file mode 100755 index 3053cfcc79..0000000000 --- a/nix/packages/github-matrix/github_matrix.py +++ /dev/null @@ -1,436 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -from collections import Counter, defaultdict -import graphlib -import json -import os -import subprocess -import sys -from typing import ( - Any, - Dict, - List, - Literal, - NamedTuple, - NotRequired, - Optional, - Set, - Tuple, - TypedDict, - get_args, -) - -from github_action_utils import debug, notice, error, set_output, warning -from result import Err, Ok, Result - -System = Literal["x86_64-linux", "aarch64-linux", "aarch64-darwin"] - - -class NixEvalJobsOutput(TypedDict): - """Raw output from nix-eval-jobs command.""" - - attr: str - attrPath: List[str] - cacheStatus: Literal["notBuilt", "cached", "local"] - drvPath: str - name: str - system: System - neededBuilds: NotRequired[List[Any]] - neededSubstitutes: NotRequired[List[Any]] - outputs: NotRequired[Dict[str, str]] - error: NotRequired[str] - requiredSystemFeatures: NotRequired[List[str]] - - -class RunsOnConfig(TypedDict): - """GitHub Actions runs-on configuration.""" - - group: NotRequired[str] - labels: List[str] - - -class GitHubActionPackage(TypedDict): - """Final package output for GitHub Actions matrix.""" - - attr: str - name: str - system: System - runs_on: RunsOnConfig - postgresql_version: NotRequired[str] - - -class NixEvalError(TypedDict): - """Error information from nix evaluation.""" - - attr: str - error: str - - -def build_nix_eval_command( - max_workers: int, max_memory_size: int, flake_outputs: List[str] -) -> List[str]: - """Build the nix-eval-jobs command with appropriate flags.""" - nix_eval_cmd = [ - "nix-eval-jobs", - "--flake", - ".", - "--check-cache-status", - "--force-recurse", - "--quiet", - "--option", - "eval-cache", - "false", - "--option", - "accept-flake-config", - "true", - "--max-memory-size", - str(max_memory_size), - "--workers", - str(max_workers), - "--select", - f"outputs: {{ inherit (outputs) {' '.join(flake_outputs)}; }}", - ] - return nix_eval_cmd - - -def parse_nix_eval_line( - line: str, drv_paths: Set[str] -) -> Result[Optional[NixEvalJobsOutput], NixEvalError]: - """Parse a single line of nix-eval-jobs output. - - Returns: - Ok(package_data) if successful (None for empty/duplicate lines) - Err(NixEvalError) if a nix evaluation error occurred - """ - if not line.strip(): - return Ok(None) - - try: - data: NixEvalJobsOutput = json.loads(line) - if "error" in data: - error_msg = data["error"] - - # Extract the core error message (last "error:" line and following context) - error_lines = error_msg.split("\n") - core_error_idx = -1 - for i in range(len(error_lines) - 1, -1, -1): - if error_lines[i].strip().startswith("error:"): - core_error_idx = i - break - - if core_error_idx >= 0: - # Take the last error line and up to 3 lines of context after it - error_msg = "\n".join( - error_lines[ - core_error_idx : min(core_error_idx + 4, len(error_lines)) - ] - ).strip() - - return Err({"attr": data["attr"], "error": error_msg}) - if data["drvPath"] in drv_paths: - return Ok(None) - if "nixos-test" in data.get("requiredSystemFeatures", []) and data[ - "system" - ] in ("x86_64-linux", "aarch64-darwin"): - return Ok(None) - drv_paths.add(data["drvPath"]) - return Ok(data) - except json.JSONDecodeError as e: - warning(f"Skipping invalid JSON line: {line}", title="JSON Parse Warning") - return Ok(None) - - -def run_nix_eval_jobs( - cmd: List[str], -) -> Tuple[str, List[str]]: - """Run nix-eval-jobs and return parsed package data, warnings, and errors. - - Returns: - Tuple of (stdout, warnings_list) - """ - print(f"Running: {' '.join(cmd)}", file=sys.stderr) - - # Disable colors in nix output - env = os.environ.copy() - env["NO_COLOR"] = "1" - - process = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env - ) - stdout_data, stderr_data = process.communicate() - - # Parse stderr for warnings (lines starting with "warning:") - warnings_list: List[str] = [] - for line in stderr_data.splitlines(): - line = line.strip() - if line.startswith("warning:") or line.startswith("evaluation warning:"): - # Remove "warning:" prefix for cleaner messages - warnings_list.append(line[8:].strip()) - - if process.returncode != 0: - error( - "nix-eval-jobs process failed with non-zero exit code", - title="Process Failure", - ) - - return stdout_data, warnings_list - - -def process_nix_eval_jobs_stdout( - stdout: str, -) -> Tuple[List[NixEvalJobsOutput], List[NixEvalError]]: - # Parse stdout for packages - packages: List[NixEvalJobsOutput] = [] - drv_paths: Set[str] = set() - errors: List[NixEvalError] = [] - for line in stdout.splitlines(): - result = parse_nix_eval_line(line, drv_paths) - if result.is_err(): - errors.append(result._value) - elif result._value is not None: - packages.append(result._value) - - return packages, errors - - -def is_extension_pkg(pkg: NixEvalJobsOutput) -> bool: - """Check if the package is a postgresql extension package.""" - attrs = pkg["attr"].split(".") - return attrs[-2] == "exts" - - -# thank you buildbot-nix https://github.com/nix-community/buildbot-nix/blob/985d069a2a45cf4a571a4346107671adc2bd2a16/buildbot_nix/buildbot_nix/build_trigger.py#L297 -def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutput]: - sorted_jobs = [] - - # Prepare job dependencies - job_set = {job["drvPath"] for job in jobs} - job_closures = { - k["drvPath"]: set(k.get("neededSubstitutes", [])) - .union(set(k.get("neededBuilds", []))) - .intersection(job_set) - .difference({k["drvPath"]}) - for k in jobs - } - - sorter = graphlib.TopologicalSorter(job_closures) - - job_by_drv = {job["drvPath"]: job for job in jobs} - for item in sorter.static_order(): - if item in job_by_drv: - sorted_jobs.append(job_by_drv[item]) - - return sorted_jobs - - -def is_large_pkg(pkg: NixEvalJobsOutput) -> bool: - """Determine if a package is considered large based on its attribute path.""" - return "big-parallel" in pkg.get("requiredSystemFeatures", []) - - -def is_virt_pkg(pkg: NixEvalJobsOutput) -> bool: - """Determine if a package requires hardware virtualization capabilities""" - return bool({"apple-virt", "kvm"} & set(pkg.get("requiredSystemFeatures", []))) - - -def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None: - """Determine the appropriate GitHub Actions runner for a package. - - Priority order: - 1. VM packages on Darwin → self-hosted runners - 2. VM packages on Linux → 16vcpu blacksmith runners - 3. Large packages on D/L → 12/32vcpu blacksmith runners - 5. Default Darwin/Linux → 6/8vcpu blacksmith runners - """ - - system = pkg["system"] - arch, os = system.split("-") - - class Specs(NamedTuple): - vcpu: int = 0 - osv: str = None - - specs = Specs() - - match (is_virt_pkg(pkg), is_large_pkg(pkg), os, arch): - # kvm - case (True, _, "darwin", "aarch64"): - return {"group": "self-hosted-runners-nix", "labels": ["aarch64-darwin"]} - case (True, _, "linux", "aarch64"): - return {"labels": ["arm-native-runner"]} - case (True, _, "linux", "x86_64"): - specs = Specs(16, "ubuntu-2404") - - # large - case (_, True, "darwin", "aarch64"): - specs = Specs(12, "macos-26") - case (_, True, "linux", "aarch64"): - specs = Specs(32, "ubuntu-2404-arm") - case (_, True, "linux", "x86_64"): - specs = Specs(32, "ubuntu-2404") - - # default - case (_, _, "darwin", "aarch64"): - specs = Specs(6, "macos-26") - case (_, _, "linux", "aarch64"): - specs = Specs(8, "ubuntu-2404-arm") - case (_, _, "linux", "x86_64"): - specs = Specs(8, "ubuntu-2404") - - return {"labels": [f"blacksmith-{specs.vcpu}vcpu-{specs.osv}"]} - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Generate GitHub Actions matrix for Nix builds" - ) - parser.add_argument( - "--max-memory-size", - default=3072, - type=int, - help="Maximum memory per eval worker in MiB. Defaults to 3072 (3 GiB).", - ) - parser.add_argument( - "-j", - "--nb-eval-jobs-workers", - default=os.cpu_count() or 1, - type=int, - help="Number of parallel eval jobs. Defaults to the number of logical CPUs in the system.", - ) - parser.add_argument( - "--stdin", - action="store_true", - help="Read nix-eval-jobs output from stdin instead of executing process", - ) - parser.add_argument( - "--stdout", - action="store_true", - help="Send matrix as json to stdout", - ) - parser.add_argument( - "flake_outputs", nargs="+", help="Nix flake outputs to evaluate" - ) - - args = parser.parse_args() - - if args.stdin: - nix_eval_output, warnings_list = sys.stdin.read(), [] - else: - cmd = build_nix_eval_command( - args.nb_eval_jobs_workers, - args.max_memory_size, - args.flake_outputs, - ) - nix_eval_output, warnings_list = run_nix_eval_jobs(cmd) - - packages, errors_list = process_nix_eval_jobs_stdout(nix_eval_output) - gh_action_packages = sort_pkgs_by_closures(packages) - - def clean_package_for_output(pkg: NixEvalJobsOutput) -> GitHubActionPackage: - """Convert nix-eval-jobs output to GitHub Actions matrix package""" - runner = get_runner_for_package(pkg) - if runner is None: - raise ValueError(f"No runner configuration for system: {pkg['system']}") - returned_pkg: GitHubActionPackage = { - "attr": pkg["attr"], - "name": pkg["name"], - "system": pkg["system"], - "runs_on": runner, - } - if is_extension_pkg(pkg): - # Extract PostgreSQL version from attribute path - attrs = pkg["attr"].split(".") - returned_pkg["postgresql_version"] = attrs[-3].split("_")[-1] - return returned_pkg - - # Group packages by system and type (checks vs packages) - packages_by_system: Dict[System, List[GitHubActionPackage]] = defaultdict(list) - checks_by_system: Dict[System, List[GitHubActionPackage]] = defaultdict(list) - for pkg in gh_action_packages: - if pkg.get("cacheStatus") == "notBuilt": - cleaned_pkg = clean_package_for_output(pkg) - if pkg["attr"].startswith("checks."): - checks_by_system[pkg["system"]].append(cleaned_pkg) - elif pkg["attr"].startswith("legacyPackages."): - packages_by_system[pkg["system"]].append(cleaned_pkg) - - packages_output: Dict[str, Dict[str, List[GitHubActionPackage]]] = {} - for pkg_system, pkg_list in packages_by_system.items(): - packages_output[pkg_system.replace("-", "_")] = {"include": pkg_list} - - checks_output: Dict[str, Dict[str, List[GitHubActionPackage]]] = {} - for check_system, check_list in checks_by_system.items(): - checks_output[check_system.replace("-", "_")] = {"include": check_list} - - for system in get_args(System): - s = system.replace("-", "_") - if s not in checks_output: - checks_output[s] = { - "include": [ - { - "attr": "", - "name": "no checks to build", - "system": system, - "runs_on": {"labels": ["ubuntu-latest"]}, - } - ] - } - if s not in packages_output: - packages_output[s] = { - "include": [ - { - "attr": "", - "name": "no packages to build", - "system": system, - "runs_on": {"labels": ["ubuntu-latest"]}, - } - ] - } - - gh_output = { - "packages": packages_output, - "checks": checks_output, - } - - if warnings_list: - warning_counts = Counter(warnings_list) - for warn_msg, count in warning_counts.items(): - if count > 1: - warning( - f"{warn_msg} (occurred {count} times)", - title="Nix Evaluation Warning", - ) - else: - warning(warn_msg, title="Nix Evaluation Warning") - - if errors_list: - # Group errors by error message - errors_by_message: Dict[str, List[str]] = defaultdict(list) - for err in errors_list: - errors_by_message[err["error"]].append(err["attr"]) - - for error_msg, attrs in errors_by_message.items(): - # Format message with attributes on first line, then error details - if len(attrs) > 1: - formatted_msg = f"Affected attributes ({len(attrs)}): {', '.join(attrs)}\n\n{error_msg}" - else: - formatted_msg = f"Attribute: {attrs[0]}\n\n{error_msg}" - formatted_msg = formatted_msg.replace("\n", "%0A") - error(formatted_msg, title="Nix Evaluation Error") - - if errors_list: - sys.exit(1) - elif args.stdout: - print(json.dumps(gh_output)) - else: - formatted_msg = f"Generated GitHub Actions matrix: {json.dumps(gh_output, indent=2)}".replace( - "\n", "%0A" - ) - notice(formatted_msg, title="GitHub Actions Matrix") - set_output("packages_matrix", json.dumps(gh_output["packages"])) - set_output("checks_matrix", json.dumps(gh_output["checks"])) - - -if __name__ == "__main__": - main() diff --git a/nix/packages/github-matrix/tests/cases/builders.aarch64-darwin.expected.json b/nix/packages/github-matrix/tests/cases/builders.aarch64-darwin.expected.json new file mode 100644 index 0000000000..85bef54060 --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/builders.aarch64-darwin.expected.json @@ -0,0 +1,43 @@ +{ + "packages": { + "include": [ + { + "attr": "legacyPackages.aarch64-darwin.psql_17.exts.postgis", + "name": "postgis-3.3.7", + "system": "aarch64-darwin", + "runs_on": { + "labels": [ + "blacksmith-12vcpu-macos-26" + ] + }, + "postgresql_version": "17" + }, + { + "attr": "legacyPackages.aarch64-darwin.psql_orioledb-17.exts.orioledb", + "name": "orioledb-beta12", + "system": "aarch64-darwin", + "runs_on": { + "labels": [ + "blacksmith-6vcpu-macos-26" + ] + }, + "postgresql_version": "orioledb-17" + } + ] + }, + "checks": { + "include": [ + { + "attr": "checks.aarch64-darwin.virt-only", + "name": "virt-only", + "system": "aarch64-darwin", + "runs_on": { + "group": "self-hosted-runners-nix", + "labels": [ + "aarch64-darwin" + ] + } + } + ] + } +} diff --git a/nix/packages/github-matrix/tests/cases/builders.aarch64-darwin.jsonl b/nix/packages/github-matrix/tests/cases/builders.aarch64-darwin.jsonl new file mode 100644 index 0000000000..3a1aadb0ab --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/builders.aarch64-darwin.jsonl @@ -0,0 +1,4 @@ +{"attr":"checks.aarch64-darwin.amcheck-upgrade","attrPath":["checks","aarch64-darwin","amcheck-upgrade"],"name":"vm-test-run-amcheck-upgrade","system":"aarch64-darwin","drvPath":"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-vm-test-run-amcheck-upgrade.drv","outputs":{"out":"/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-vm-test-run-amcheck-upgrade"},"requiredSystemFeatures":["apple-virt","nixos-test"],"cacheStatus":"notBuilt"} +{"attr":"checks.aarch64-darwin.virt-only","attrPath":["checks","aarch64-darwin","virt-only"],"name":"virt-only","system":"aarch64-darwin","drvPath":"/nix/store/cccccccccccccccccccccccccccccccc-virt-only.drv","outputs":{"out":"/nix/store/dddddddddddddddddddddddddddddddd-virt-only"},"requiredSystemFeatures":["apple-virt"],"cacheStatus":"notBuilt"} +{"attr":"legacyPackages.aarch64-darwin.psql_17.exts.postgis","attrPath":["legacyPackages","aarch64-darwin","psql_17","exts","postgis"],"name":"postgis-3.3.7","system":"aarch64-darwin","drvPath":"/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-postgis-3.3.7.drv","outputs":{"out":"/nix/store/ffffffffffffffffffffffffffffffff-postgis-3.3.7"},"requiredSystemFeatures":["big-parallel"],"cacheStatus":"notBuilt"} +{"attr":"legacyPackages.aarch64-darwin.psql_orioledb-17.exts.orioledb","attrPath":["legacyPackages","aarch64-darwin","psql_orioledb-17","exts","orioledb"],"name":"orioledb-beta12","system":"aarch64-darwin","drvPath":"/nix/store/gggggggggggggggggggggggggggggggg-orioledb-beta12.drv","outputs":{"out":"/nix/store/hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh-orioledb-beta12"},"requiredSystemFeatures":[],"cacheStatus":"notBuilt"} diff --git a/nix/packages/github-matrix/tests/cases/builders.aarch64-linux.expected.json b/nix/packages/github-matrix/tests/cases/builders.aarch64-linux.expected.json new file mode 100644 index 0000000000..5833fc5292 --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/builders.aarch64-linux.expected.json @@ -0,0 +1,41 @@ +{ + "packages": { + "include": [ + { + "attr": "legacyPackages.aarch64-linux.psql_15.exts.pg_graphql", + "name": "pg_graphql-1.5.11", + "system": "aarch64-linux", + "runs_on": { + "labels": [ + "blacksmith-32vcpu-ubuntu-2404-arm" + ] + }, + "postgresql_version": "15" + }, + { + "attr": "legacyPackages.aarch64-linux.postgresql_15", + "name": "postgresql-15.14", + "system": "aarch64-linux", + "runs_on": { + "labels": [ + "blacksmith-8vcpu-ubuntu-2404-arm" + ] + } + } + ] + }, + "checks": { + "include": [ + { + "attr": "checks.aarch64-linux.pgvector", + "name": "vm-test-run-vector", + "system": "aarch64-linux", + "runs_on": { + "labels": [ + "arm-native-runner" + ] + } + } + ] + } +} diff --git a/nix/packages/github-matrix/tests/cases/builders.aarch64-linux.jsonl b/nix/packages/github-matrix/tests/cases/builders.aarch64-linux.jsonl new file mode 100644 index 0000000000..531de46664 --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/builders.aarch64-linux.jsonl @@ -0,0 +1,3 @@ +{"attr":"checks.aarch64-linux.pgvector","attrPath":["checks","aarch64-linux","pgvector"],"name":"vm-test-run-vector","system":"aarch64-linux","drvPath":"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-vm-test-run-vector.drv","outputs":{"out":"/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-vm-test-run-vector"},"requiredSystemFeatures":["kvm","nixos-test"],"cacheStatus":"notBuilt"} +{"attr":"legacyPackages.aarch64-linux.psql_15.exts.pg_graphql","attrPath":["legacyPackages","aarch64-linux","psql_15","exts","pg_graphql"],"name":"pg_graphql-1.5.11","system":"aarch64-linux","drvPath":"/nix/store/cccccccccccccccccccccccccccccccc-pg_graphql-1.5.11.drv","outputs":{"out":"/nix/store/dddddddddddddddddddddddddddddddd-pg_graphql-1.5.11"},"requiredSystemFeatures":["big-parallel"],"cacheStatus":"notBuilt"} +{"attr":"legacyPackages.aarch64-linux.postgresql_15","attrPath":["legacyPackages","aarch64-linux","postgresql_15"],"name":"postgresql-15.14","system":"aarch64-linux","drvPath":"/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-postgresql-15.14.drv","outputs":{"out":"/nix/store/ffffffffffffffffffffffffffffffff-postgresql-15.14"},"requiredSystemFeatures":[],"cacheStatus":"notBuilt"} diff --git a/nix/packages/github-matrix/tests/cases/builders.x86_64-linux.expected.json b/nix/packages/github-matrix/tests/cases/builders.x86_64-linux.expected.json new file mode 100644 index 0000000000..e660b26d74 --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/builders.x86_64-linux.expected.json @@ -0,0 +1,41 @@ +{ + "packages": { + "include": [ + { + "attr": "legacyPackages.x86_64-linux.psql_17.exts.plv8", + "name": "plv8-3.1.10", + "system": "x86_64-linux", + "runs_on": { + "labels": [ + "blacksmith-32vcpu-ubuntu-2404" + ] + }, + "postgresql_version": "17" + }, + { + "attr": "legacyPackages.x86_64-linux.pgbouncer", + "name": "pgbouncer-1.24.1", + "system": "x86_64-linux", + "runs_on": { + "labels": [ + "blacksmith-8vcpu-ubuntu-2404" + ] + } + } + ] + }, + "checks": { + "include": [ + { + "attr": "checks.x86_64-linux.pgvector", + "name": "vm-test-run-vector", + "system": "x86_64-linux", + "runs_on": { + "labels": [ + "blacksmith-16vcpu-ubuntu-2404" + ] + } + } + ] + } +} diff --git a/nix/packages/github-matrix/tests/cases/builders.x86_64-linux.jsonl b/nix/packages/github-matrix/tests/cases/builders.x86_64-linux.jsonl new file mode 100644 index 0000000000..cfbc419fff --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/builders.x86_64-linux.jsonl @@ -0,0 +1,3 @@ +{"attr":"checks.x86_64-linux.pgvector","attrPath":["checks","x86_64-linux","pgvector"],"name":"vm-test-run-vector","system":"x86_64-linux","drvPath":"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-vm-test-run-vector.drv","outputs":{"out":"/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-vm-test-run-vector"},"requiredSystemFeatures":["kvm","nixos-test"],"cacheStatus":"notBuilt"} +{"attr":"legacyPackages.x86_64-linux.psql_17.exts.plv8","attrPath":["legacyPackages","x86_64-linux","psql_17","exts","plv8"],"name":"plv8-3.1.10","system":"x86_64-linux","drvPath":"/nix/store/cccccccccccccccccccccccccccccccc-plv8-3.1.10.drv","outputs":{"out":"/nix/store/dddddddddddddddddddddddddddddddd-plv8-3.1.10"},"requiredSystemFeatures":["big-parallel"],"cacheStatus":"notBuilt"} +{"attr":"legacyPackages.x86_64-linux.pgbouncer","attrPath":["legacyPackages","x86_64-linux","pgbouncer"],"name":"pgbouncer-1.24.1","system":"x86_64-linux","drvPath":"/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-pgbouncer-1.24.1.drv","outputs":{"out":"/nix/store/ffffffffffffffffffffffffffffffff-pgbouncer-1.24.1"},"requiredSystemFeatures":[],"cacheStatus":"notBuilt"} diff --git a/nix/packages/github-matrix/tests/cases/cache-status.x86_64-linux.expected.json b/nix/packages/github-matrix/tests/cases/cache-status.x86_64-linux.expected.json new file mode 100644 index 0000000000..d9c796759f --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/cache-status.x86_64-linux.expected.json @@ -0,0 +1,31 @@ +{ + "packages": { + "include": [ + { + "attr": "legacyPackages.x86_64-linux.psql_17.exts.pgvector", + "name": "pgvector-0.8.0", + "system": "x86_64-linux", + "runs_on": { + "labels": [ + "blacksmith-8vcpu-ubuntu-2404" + ] + }, + "postgresql_version": "17" + } + ] + }, + "checks": { + "include": [ + { + "attr": "", + "name": "no checks to build", + "system": "x86_64-linux", + "runs_on": { + "labels": [ + "ubuntu-latest" + ] + } + } + ] + } +} diff --git a/nix/packages/github-matrix/tests/cases/cache-status.x86_64-linux.jsonl b/nix/packages/github-matrix/tests/cases/cache-status.x86_64-linux.jsonl new file mode 100644 index 0000000000..471aac39ed --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/cache-status.x86_64-linux.jsonl @@ -0,0 +1,4 @@ +{"attr":"legacyPackages.x86_64-linux.psql_17.exts.pgvector","attrPath":["legacyPackages","x86_64-linux","psql_17","exts","pgvector"],"name":"pgvector-0.8.0","system":"x86_64-linux","drvPath":"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-pgvector-0.8.0.drv","outputs":{"out":"/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-pgvector-0.8.0"},"requiredSystemFeatures":[],"cacheStatus":"notBuilt"} +{"attr":"legacyPackages.x86_64-linux.psql_17_alias.exts.pgvector","attrPath":["legacyPackages","x86_64-linux","psql_17_alias","exts","pgvector"],"name":"pgvector-0.8.0","system":"x86_64-linux","drvPath":"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-pgvector-0.8.0.drv","outputs":{"out":"/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-pgvector-0.8.0"},"requiredSystemFeatures":[],"cacheStatus":"notBuilt"} +{"attr":"legacyPackages.x86_64-linux.postgresql_15","attrPath":["legacyPackages","x86_64-linux","postgresql_15"],"name":"postgresql-15.14","system":"x86_64-linux","drvPath":"/nix/store/cccccccccccccccccccccccccccccccc-postgresql-15.14.drv","outputs":{"out":"/nix/store/dddddddddddddddddddddddddddddddd-postgresql-15.14"},"requiredSystemFeatures":["big-parallel"],"cacheStatus":"cached"} +{"attr":"checks.x86_64-linux.psql_15","attrPath":["checks","x86_64-linux","psql_15"],"name":"run-check-harness-psql-15","system":"x86_64-linux","drvPath":"/nix/store/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-run-check-harness-psql-15.drv","outputs":{"out":"/nix/store/ffffffffffffffffffffffffffffffff-run-check-harness-psql-15"},"requiredSystemFeatures":[],"cacheStatus":"cached"} diff --git a/nix/packages/github-matrix/tests/cases/errors.x86_64-linux.expected.txt b/nix/packages/github-matrix/tests/cases/errors.x86_64-linux.expected.txt new file mode 100644 index 0000000000..cafb9a03e9 --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/errors.x86_64-linux.expected.txt @@ -0,0 +1,2 @@ +::error title=Nix Evaluation Error::Attribute: checks.x86_64-linux.c%0A%0Aerror: attribute 'foo' missing%0A at /nix/store/x.nix:1:1 +::error title=Nix Evaluation Error::Affected attributes (2): legacyPackages.x86_64-linux.psql_17.exts.a, legacyPackages.x86_64-linux.psql_17.exts.b%0A%0Aevaluation failed diff --git a/nix/packages/github-matrix/tests/cases/errors.x86_64-linux.jsonl b/nix/packages/github-matrix/tests/cases/errors.x86_64-linux.jsonl new file mode 100644 index 0000000000..9eddad3229 --- /dev/null +++ b/nix/packages/github-matrix/tests/cases/errors.x86_64-linux.jsonl @@ -0,0 +1,4 @@ +{"attr":"checks.x86_64-linux.ok","attrPath":["checks","x86_64-linux","ok"],"name":"ok","system":"x86_64-linux","drvPath":"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-ok.drv","outputs":{"out":"/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-ok"},"requiredSystemFeatures":[],"cacheStatus":"notBuilt"} +{"attr":"legacyPackages.x86_64-linux.psql_17.exts.a","error":"evaluation failed"} +{"attr":"legacyPackages.x86_64-linux.psql_17.exts.b","error":"evaluation failed"} +{"attr":"checks.x86_64-linux.c","error":"error: attribute 'foo' missing\n at /nix/store/x.nix:1:1"} diff --git a/nix/packages/github-matrix/tests/run.sh b/nix/packages/github-matrix/tests/run.sh new file mode 100755 index 0000000000..3c20355727 --- /dev/null +++ b/nix/packages/github-matrix/tests/run.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Each case is ..jsonl plus either ..expected.json +# (stdout must match, exit 0) or ..expected.txt (stdout must +# match, exit 1). +set -euo pipefail + +cd "$(dirname "$0")" +failed=0 + +for input in cases/*.jsonl; do + base=${input%.jsonl} + system=${base##*.} + name=$(basename "$base") + + if [[ -e $base.expected.json ]]; then + expected=$(jq -S . "$base.expected.json") + if actual=$(jq -r -S -s --arg system "$system" -f ../github-matrix.jq "$input") && [[ $actual == "$expected" ]]; then + echo "ok $name" + else + echo "FAIL $name" + diff <(echo "$expected") <(echo "$actual") || true + failed=1 + fi + else + expected=$(cat "$base.expected.txt") + if actual=$(jq -r -s --arg system "$system" -f ../github-matrix.jq "$input" 2>/dev/null); then + echo "FAIL $name: expected exit 1" + failed=1 + elif [[ $actual == "$expected" ]]; then + echo "ok $name" + else + echo "FAIL $name" + diff <(echo "$expected") <(echo "$actual") || true + failed=1 + fi + fi +done + +exit $failed diff --git a/nix/packages/github-matrix/tests/test_github_matrix.py b/nix/packages/github-matrix/tests/test_github_matrix.py deleted file mode 100644 index a4192826fd..0000000000 --- a/nix/packages/github-matrix/tests/test_github_matrix.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 - -import json - -import pytest - -from github_matrix import ( - NixEvalJobsOutput, - get_runner_for_package, - is_extension_pkg, - is_virt_pkg, - is_large_pkg, - parse_nix_eval_line, - sort_pkgs_by_closures, -) - - -class TestIsExtensionPkg: - def test_extension_package(self): - pkg: NixEvalJobsOutput = { - "attr": "legacyPackages.x86_64-linux.psql_15.exts.pg_cron", - "attrPath": [ - "legacyPackages", - "x86_64-linux", - "psql_15", - "exts", - "pg_cron", - ], - "cacheStatus": "notBuilt", - "drvPath": "/nix/store/test.drv", - "name": "pg_cron", - "system": "x86_64-linux", - } - assert is_extension_pkg(pkg) is True - - def test_non_extension_package(self): - pkg: NixEvalJobsOutput = { - "attr": "legacyPackages.x86_64-linux.psql_15", - "attrPath": ["legacyPackages", "x86_64-linux", "psql_15"], - "cacheStatus": "notBuilt", - "drvPath": "/nix/store/test.drv", - "name": "postgresql-16.0", - "system": "x86_64-linux", - } - assert is_extension_pkg(pkg) is False - - -class TestIsLargePkg: - @pytest.mark.parametrize( - "attr,expected", - [ - ("legacyPackages.x86_64-linux.psql_15.exts.wrappers", True), - ("legacyPackages.x86_64-linux.psql_15.exts.pg_jsonschema", True), - ("legacyPackages.x86_64-linux.psql_15.exts.pg_graphql", True), - ("legacyPackages.x86_64-linux.psql_15.exts.postgis", True), - ("legacyPackages.x86_64-linux.psql_15.exts.pg_cron", False), - ("legacyPackages.x86_64-linux.psql_15", False), - ], - ) - def test_large_package_detection(self, attr: str, expected: bool): - pkg: NixEvalJobsOutput = { - "attr": attr, - "attrPath": attr.split("."), - "cacheStatus": "notBuilt", - "drvPath": f"/nix/store/{attr}.drv", - "name": attr.split(".")[-1], - "system": "x86_64-linux", - "requiredSystemFeatures": ["big-parallel"] if expected else [], - } - assert is_large_pkg(pkg) is expected - - -class TestIsVirtPkg: - @pytest.mark.parametrize( - "feat,expected", - [ - ("", False), - ("apple-virt", True), - ("big-parallel", False), - ("kvm", True), - ], - ) - def test_is_virt_pkg(self, feat, expected): - pkg: NixEvalJobsOutput = { - "requiredSystemFeatures": [feat], - } - assert is_virt_pkg(pkg) is expected - - -class TestNixOSCheck: - @pytest.mark.parametrize( - "system,expected", - [ - ("aarch64-darwin", False), - ("aarch64-linux", True), - ("x86_64-linux", False), - ], - ) - def test_system(self, system, expected): - check = { - "attr": f"attr-{system}", - "drvPath": f"test_{system}", - "system": system, - "requiredSystemFeatures": ["nixos-test"], - } - result = parse_nix_eval_line(json.dumps(check), set()) - if expected: - assert result._value is not None - else: - assert result._value is None - - -class TestGetRunnerForPackage: - @pytest.mark.parametrize( - "system,feature,expected", - [ - ( - "aarch64-darwin", - "apple-virt", - {"group": "self-hosted-runners-nix", "labels": ["aarch64-darwin"]}, - ), - ( - "aarch64-darwin", - "big-parallel", - {"labels": ["blacksmith-12vcpu-macos-26"]}, - ), - ( - "aarch64-darwin", - None, - {"labels": ["blacksmith-6vcpu-macos-26"]}, - ), - ( - "aarch64-linux", - "kvm", - {"labels": ["arm-native-runner"]}, - ), - ( - "aarch64-linux", - "big-parallel", - {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]}, - ), - ( - "aarch64-linux", - None, - {"labels": ["blacksmith-8vcpu-ubuntu-2404-arm"]}, - ), - ( - "x86_64-linux", - "kvm", - {"labels": ["blacksmith-16vcpu-ubuntu-2404"]}, - ), - ( - "x86_64-linux", - "big-parallel", - {"labels": ["blacksmith-32vcpu-ubuntu-2404"]}, - ), - ( - "x86_64-linux", - None, - {"labels": ["blacksmith-8vcpu-ubuntu-2404"]}, - ), - ], - ) - def test_get_runner_for_package(self, system, feature, expected): - pkg: NixEvalJobsOutput = { - "system": system, - } - if feature: - pkg["requiredSystemFeatures"] = [feature] - - result = get_runner_for_package(pkg) - assert result == expected - - -class TestSortPkgsByClosures: - def test_empty_list(self): - result = sort_pkgs_by_closures([]) - assert result == [] - - def test_single_package(self): - pkg: NixEvalJobsOutput = { - "attr": "packages.x86_64-linux.psql_15", - "attrPath": ["packages", "x86_64-linux", "psql_15"], - "cacheStatus": "notBuilt", - "drvPath": "/nix/store/test.drv", - "name": "postgresql-16.0", - "system": "x86_64-linux", - } - result = sort_pkgs_by_closures([pkg]) - assert result == [pkg] - - def test_dependency_order(self): - pkg1: NixEvalJobsOutput = { - "attr": "packages.x86_64-linux.lib", - "attrPath": ["packages", "x86_64-linux", "lib"], - "cacheStatus": "notBuilt", - "drvPath": "/nix/store/lib.drv", - "name": "lib", - "system": "x86_64-linux", - "neededBuilds": [], - "neededSubstitutes": [], - } - pkg2: NixEvalJobsOutput = { - "attr": "packages.x86_64-linux.app", - "attrPath": ["packages", "x86_64-linux", "app"], - "cacheStatus": "notBuilt", - "drvPath": "/nix/store/app.drv", - "name": "app", - "system": "x86_64-linux", - "neededBuilds": ["/nix/store/lib.drv"], - "neededSubstitutes": [], - } - - # Regardless of input order, lib should come before app - result = sort_pkgs_by_closures([pkg2, pkg1]) - assert result == [pkg1, pkg2] - - result = sort_pkgs_by_closures([pkg1, pkg2]) - assert result == [pkg1, pkg2] diff --git a/scripts/parallel-nix-eval-jobs.sh b/scripts/parallel-nix-eval-jobs.sh new file mode 100755 index 0000000000..ee1c530fd7 --- /dev/null +++ b/scripts/parallel-nix-eval-jobs.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Prototype: parallel eval -> nix-eval-jobs-shaped JSONL -> github-matrix. +# Native system only (DetNix currently breaks on foreign-system eval). +# Run from the repo root. + +CACHE_URL=https://nix-postgres-artifacts.s3.amazonaws.com +SYSTEM=$(nix eval --raw --no-pure-eval --expr builtins.currentSystem) +workdir=$(mktemp -d) +trap 'rm -rf "$workdir"' EXIT + +cat >"$workdir/walk.nix" <<-'EOF' + { prefix, v }: + let + isDrv = x: (x.type or null) == "derivation"; + esc = s: if builtins.match "[a-zA-Z_][a-zA-Z0-9_'-]*" s != null then s else "\"${s}\""; + join = p: builtins.concatStringsSep "." (map esc p); + # lib.lazyDerivation (NixOS tests) hides requiredSystemFeatures from the + # attrset, so read it from the .drv like nix-eval-jobs did + words = s: builtins.filter (w: builtins.isString w && w != "") (builtins.split " " s); + drvFeatures = x: + let found = builtins.filter builtins.isList + (builtins.split "\\(\"requiredSystemFeatures\",\"([^\"]*)\"\\)" (builtins.readFile x.drvPath)); + in if found == [ ] then [ ] else words (builtins.head (builtins.head found)); + go = p: x: + if isDrv x then + let + val = { + attr = join p; + attrPath = p; + name = x.name; + system = x.system; + drvPath = x.drvPath; + outputs = builtins.listToAttrs + (map (o: { name = o; value = x.${o}.outPath; }) (x.outputs or [ "out" ])); + requiredSystemFeatures = x.requiredSystemFeatures or (drvFeatures x); + }; + r = builtins.tryEval (builtins.deepSeq val val); + in if r.success then [ r.value ] else [ { attr = join p; error = "evaluation failed"; } ] + else if builtins.isAttrs x then + # each child subtree is deeply forced as one unit; builtins.parallel + # evaluates the units concurrently, then we splice the results + let + branches = map + (n: let jobs = go (p ++ [ n ]) x.${n}; in builtins.deepSeq jobs jobs) + (builtins.attrNames x); + spliced = builtins.concatLists branches; + in if builtins ? parallel then builtins.parallel branches spliced else spliced + else [ ]; + in go prefix v +EOF + +# builtins.parallel needs an experimental feature that only Determinate Nix has; +# eval-cores defaults to 1, so without it the "parallel" eval is single-threaded +parallel_flags=() +if nix --version 2>/dev/null | grep -qi determinate; then + parallel_flags=(--extra-experimental-features parallel-eval --option eval-cores 0) +fi + +for output in checks legacyPackages; do + echo "== evaluating $output.$SYSTEM" >&2 + time nix eval --json --no-pure-eval --option eval-cache false \ + "${parallel_flags[@]}" \ + ".#$output.$SYSTEM" \ + --apply "v: import $workdir/walk.nix { prefix = [ \"$output\" \"$SYSTEM\" ]; inherit v; }" \ + >"$workdir/$output.json" +done + +# distinct output-path hashes -> parallel narinfo HEAD sweep +jq -r '.[] | select(.error == null) | .outputs[]' \ + "$workdir"/checks.json "$workdir"/legacyPackages.json | + sed -E 's|^/nix/store/([a-z0-9]{32})-.*|\1|' | sort -u >"$workdir/hashes.txt" + +echo "== checking $(wc -l <"$workdir/hashes.txt") narinfos against $CACHE_URL" >&2 +awk -v base="$CACHE_URL" \ + '{ printf "url = \"%s/%s.narinfo\"\noutput = \"/dev/null\"\n", base, $0 }' \ + <"$workdir/hashes.txt" >"$workdir/curl.cfg" +time curl --parallel --parallel-max 100 --head --silent \ + --write-out '%{http_code} %{url}\n' --config "$workdir/curl.cfg" | + awk '$1 == "200" { sub(/.*\//, "", $2); sub(/\.narinfo$/, "", $2); print $2 }' \ + >"$workdir/cached.txt" || true + +# a job is "cached" only if every one of its outputs is in the cache +jq -c --rawfile cached "$workdir/cached.txt" ' + (($cached | split("\n")) | map(select(length > 0)) | map({(.): true}) | add // {}) as $c + | .[] + | if .error != null then . else + . + { cacheStatus: + (if ([ .outputs[] | capture("^/nix/store/(?[a-z0-9]{32})-").h ] + | all($c[.] // false)) + then "cached" else "notBuilt" end) } + end +' "$workdir"/checks.json "$workdir"/legacyPackages.json | tee "$workdir/jobs.jsonl" + +echo "== $(wc -l <"$workdir/jobs.jsonl") jobs, $(jq -sr 'map(select(.cacheStatus == "notBuilt")) | length' "$workdir/jobs.jsonl") notBuilt" >&2