[test do not merge] snapshot golden tests

.github/workflows/RegenSnapshotGoldens.yml

@@ -0,0 +1,241 @@
+# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
+
+# Publish snapshot goldens to
+# ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens.
+#
+# Runs automatically when a merge to main changes GOLDENS_VERSION (the
+# version string lives in
+# src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs). The check-published
+# job reads that version and checks GHCR for its `{version}-complete`
+# marker. If the marker is absent, the matrix walks every (hv, cpu,
+# config) combination, dumps the canonical snapshot, and uploads it as a
+# workflow artifact. A single publish job then downloads every artifact,
+# pushes each as a tag named `{version}-{hv}-{cpu}-{profile}`, and
+# pushes the marker last. Publishing the whole set from one job means a
+# partial run leaves no marker and is republished on the next run.
+#
+# A version whose marker exists is left untouched, so a merge that does
+# not bump the version, or a re-run of the same version, is a no-op.
+# Manual dispatch with `force: true` overwrites an existing version and
+# exists for recovery only.
+#
+# See docs/snapshot-versioning.md
+
+name: Regenerate Snapshot Goldens
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Goldens version string. Must match GOLDENS_VERSION in source (e.g. "v1.0").
+        required: true
+        type: string
+      force:
+        description: Overwrite tags even if the version is already published (recovery only).
+        type: boolean
+        default: false
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: full
+  GHCR_IMAGE: ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens
+
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: regen-snapshot-goldens-${{ github.ref }}
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  check-published:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+    outputs:
+      version: ${{ steps.decide.outputs.version }}
+      needs_publish: ${{ steps.decide.outputs.needs_publish }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+
+      - name: Install oras
+        uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3  # v2.0.0
+        with:
+          version: 1.3.2
+
+      - name: Decide version and whether to publish
+        id: decide
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_VERSION: ${{ inputs.version }}
+          FORCE: ${{ inputs.force }}
+          GHCR_USER: ${{ github.actor }}
+          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
+          if ! [[ "${SRC}" =~ ^v[0-9]+\.[0-9]+$ ]]; then
+            echo "::error::GOLDENS_VERSION in source must match ^v[0-9]+\.[0-9]+$ (e.g. v1.0), found '${SRC}'"
+            exit 1
+          fi
+
+          # On manual dispatch the input must name the version that the
+          # dispatched ref actually carries. This catches a stale input.
+          if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${INPUT_VERSION}" != "${SRC}" ]; then
+            echo "::error::version input '${INPUT_VERSION}' does not match GOLDENS_VERSION in source '${SRC}'"
+            exit 1
+          fi
+
+          echo "version=${SRC}" >> "$GITHUB_OUTPUT"
+
+          if [ "${EVENT_NAME}" = "workflow_dispatch" ] && [ "${FORCE}" = "true" ]; then
+            echo "force requested: will publish ${SRC} even if it already exists"
+            echo "needs_publish=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          # A version is frozen once its completion marker exists on
+          # GHCR. The marker is pushed only after every matrix job has
+          # uploaded its tag, so a partial push (some jobs failed)
+          # leaves no marker and the next run republishes the missing
+          # combinations. Publishing only when the marker is absent makes the
+          # workflow idempotent and never clobbers a complete baseline.
+          echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
+          if oras repo tags "${GHCR_IMAGE}" 2>/dev/null | grep -qxF "${SRC}-complete"; then
+            echo "${SRC} already published (marker ${SRC}-complete present). Nothing to do."
+            echo "needs_publish=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "${SRC} not fully published yet. Will publish."
+            echo "needs_publish=true" >> "$GITHUB_OUTPUT"
+          fi
+
+  build-guests:
+    needs: check-published
+    if: needs.check-published.outputs.needs_publish == 'true'
+    strategy:
+      matrix:
+        config: [debug, release]
+    uses: ./.github/workflows/dep_build_guests.yml
+    with:
+      config: ${{ matrix.config }}
+    secrets: inherit
+
+  generate-snapshots:
+    needs: [check-published, build-guests]
+    if: needs.check-published.outputs.needs_publish == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        hypervisor: [kvm, mshv3, hyperv-ws2025]
+        cpu: [amd, intel]
+        config: [debug, release]
+    runs-on: ${{ fromJson(
+        format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}", "JobId=regen-goldens-{3}-{4}-{5}-{6}"]',
+          matrix.hypervisor == 'hyperv-ws2025' && 'Windows' || 'Linux',
+          matrix.hypervisor == 'hyperv-ws2025' && 'win2025' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor,
+          matrix.cpu,
+          matrix.config,
+          github.run_id,
+          github.run_number,
+          github.run_attempt)) }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
+
+      - uses: hyperlight-dev/ci-setup-workflow@f6bd9cc86d0737976d2128c8b8ced8edc017cbb4  # v1.9.0
+        with:
+          rust-toolchain: "1.94"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Fix cargo home permissions
+        if: runner.os == 'Linux'
+        run: sudo chown -R $(id -u):$(id -g) /opt/cargo || true
+
+      - name: Download Rust guests
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: rust-guests-${{ matrix.config }}
+          path: src/tests/rust_guests/bin/${{ matrix.config }}/
+
+      - name: Confirm source matches resolved version
+        env:
+          RESOLVED_VERSION: ${{ needs.check-published.outputs.version }}
+        run: |
+          set -euo pipefail
+          SRC=$(grep -oE 'GOLDENS_VERSION: &str = "[^"]+"' src/hyperlight_host/tests/snapshot_goldens/goldens_version.rs | head -n1 | sed -E 's/.*"([^"]+)".*/\1/')
+          if [ "${SRC}" != "${RESOLVED_VERSION}" ]; then
+            echo "::error::source GOLDENS_VERSION '${SRC}' does not match resolved '${RESOLVED_VERSION}'"
+            exit 1
+          fi
+
+      - name: Generate snapshots
+        run: just snapshot-goldens-generate ${{ matrix.config }}
+
+      - name: Resolve produced tag
+        id: tag
+        env:
+          GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
+        run: |
+          set -euo pipefail
+          layout=$(echo "target/snapshot-goldens/${GOLDENS_VERSION}"/*/)
+          echo "tag=$(basename "${layout%/}")" >> "$GITHUB_OUTPUT"
+
+      - name: Upload golden layout
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: golden-${{ steps.tag.outputs.tag }}
+          path: target/snapshot-goldens/${{ needs.check-published.outputs.version }}/${{ steps.tag.outputs.tag }}/
+          if-no-files-found: error
+          retention-days: 1
+
+  # Push every matrix job's snapshot from this single job, so the published set is
+  # whole or absent. `generate-snapshots` runs `fail-fast: false` and uploads each
+  # snapshot as an artifact, so this job's `needs` succeeds only when
+  # all matrix jobs did. It downloads every artifact, pushes each tag, then
+  # pushes the `{version}-complete` marker that `check-published` gates on. A
+  # push that dies partway leaves no marker, so the next run republishes.
+  publish:
+    needs: [check-published, generate-snapshots]
+    if: needs.check-published.outputs.needs_publish == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install oras
+        uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3  # v2.0.0
+        with:
+          version: 1.3.2
+
+      - name: Download all golden layouts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          pattern: golden-*
+          path: layouts
+
+      - name: Push goldens and completion marker
+        env:
+          GHCR_USER: ${{ github.actor }}
+          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GOLDENS_VERSION: ${{ needs.check-published.outputs.version }}
+        run: |
+          set -euo pipefail
+          echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
+          for layout in layouts/golden-*/; do
+            tag=$(basename "${layout%/}")
+            tag=${tag#golden-}
+            echo "::group::push ${tag}"
+            oras cp --from-oci-layout "${layout%/}:${tag}" "${GHCR_IMAGE}:${tag}"
+            echo "::endgroup::"
+          done
+          printf '%s' "${GOLDENS_VERSION}" > complete.txt
+          oras push "${GHCR_IMAGE}:${GOLDENS_VERSION}-complete" \
+            --artifact-type application/vnd.hyperlight.goldens.complete.v1 \
+            complete.txt:text/plain

.github/workflows/ValidatePullRequest.yml

@@ -79,17 +79,33 @@ jobs:
     with:
       docs_only: ${{ needs.docs-pr.outputs.docs-only }}
 
+  # Pick the goldens mode. The `regen-goldens` label means regenerate. No label means pull.
+  goldens-mode:
+    runs-on: ubuntu-latest
+    outputs:
+      regen: ${{ steps.check.outputs.regen }}
+    steps:
+      - id: check
+        if: github.event_name == 'pull_request'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh pr view ${{ github.event.pull_request.number }} --repo ${{ github.repository }} \
+            --json labels -q '.labels[].name' | grep -qx regen-goldens \
+            && echo "regen=true" >> "$GITHUB_OUTPUT" || echo "regen=false" >> "$GITHUB_OUTPUT"
+
   # Build and test - needs guest artifacts
   build-test:
     needs:
       - docs-pr
       - build-guests
+      - goldens-mode
     # Required because update-guest-locks is skipped on non-dependabot PRs,
     # and a skipped dependency transitively skips all downstream jobs.
     # See: https://github.com/actions/runner/issues/2205
     if: ${{ !cancelled() && !failure() }}
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         hypervisor: ['hyperv-ws2025', mshv3, kvm]
         cpu: [amd, intel]
@@ -101,6 +117,7 @@ jobs:
       hypervisor: ${{ matrix.hypervisor }}
       cpu: ${{ matrix.cpu }}
       config: ${{ matrix.config }}
+      regen_goldens: ${{ needs.goldens-mode.outputs.regen }}
 
   # Run examples - needs guest artifacts, runs in parallel with build-test
   run-examples:

.github/workflows/dep_build_test.yml

@@ -22,13 +22,19 @@ on:
         description: CPU architecture for the build (passed from caller matrix)
         required: true
         type: string
+      regen_goldens:
+        description: Regenerate snapshot goldens from the branch and skip pulling published ones
+        required: false
+        type: string
+        default: "false"
 
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: full
 
 permissions:
   contents: read
+  packages: read
 
 defaults:
   run:
@@ -138,3 +144,29 @@ jobs:
         env:
           RUST_LOG: debug
         run: just test-rust-tracing ${{ inputs.config }}
+
+      - name: Install oras
+        if: ${{ inputs.regen_goldens != 'true' }}
+        uses: oras-project/setup-oras@38de303aac69abb66f3e6255b7198bff35f323e3  # v2.0.0
+        with:
+          version: 1.3.2
+
+      # Pull the published goldens for this cell and load them with the
+      # branch. A missing tag fails the job and flags a format break.
+      - name: Snapshot goldens (pull and verify)
+        if: ${{ inputs.regen_goldens != 'true' }}
+        env:
+          GHCR_USER: ${{ github.actor }}
+          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "${GHCR_TOKEN}" | oras login ghcr.io -u "${GHCR_USER}" --password-stdin
+          just snapshot-goldens-pull ghcr.io/hyperlight-dev/hyperlight-snapshot-goldens ${{ inputs.config }}
+          just snapshot-goldens-verify ${{ inputs.config }}
+
+      # Label path: generate the goldens from the branch and load them
+      # back. Used when no published tag set exists yet.
+      - name: Snapshot goldens (regenerate and verify)
+        if: ${{ inputs.regen_goldens == 'true' }}
+        run: |
+          just snapshot-goldens-generate ${{ inputs.config }}
+          just snapshot-goldens-verify ${{ inputs.config }}