Filesystem probing: what breaks git-annex, and what can be stood up here

.github/workflows/probe-filesystems.yaml

@@ -0,0 +1,147 @@
+name: Probe candidate filesystems
+
+# Reconnaissance, not a test matrix: for each candidate filesystem, try
+# to stand up a throwaway instance on a stock GitHub-hosted runner and
+# report what it supports. Feeds the survey in FILESYSTEMS.md and
+# answers "is a bin/eval-under-<name> backend even possible here?"
+# before anyone writes one.
+#
+# Every candidate reports a verdict and the job stays green: "could not
+# be brought up on this runner" is the answer to the question, not a
+# failure. Read the roll-up table in the run summary -- the check mark
+# only tells you the probe itself ran.
+#
+# All logic lives in bin/ci/probe-backend.sh -- see .claude/CLAUDE.md.
+
+# Reconnaissance is 28 jobs, so it does not run on every branch push.
+# Dispatch it when you want an answer; master re-runs it when the probe
+# itself changes, to catch bit-rot in the bring-up recipes.
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+    paths:
+      - "bin/ci/probe-backend.sh"
+      - "bin/ci/fs-capabilities.sh"
+      - ".github/workflows/probe-filesystems.yaml"
+
+concurrency:
+  group: probe-filesystems-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  probe:
+    name: ${{ matrix.candidate }} (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04]
+        candidate:
+          - gfs2
+          - ocfs2
+          - f2fs
+          - exfat
+          - ntfs3
+          - udf
+          - nilfs2
+          - bcachefs
+          - zfs
+          - overlay
+          - glusterfs
+          - cephfs
+          - cifs
+          - sshfs
+          - gocryptfs
+          - encfs
+          - ecryptfs
+          - s3-rclone
+          - lustre-client
+          - openafs
+          - vm-only
+        include:
+          # Whamcloud publishes Lustre client packages per Ubuntu LTS,
+          # and 22.04 is the one with the longest support matrix -- so
+          # the client-module question gets asked on both images. The
+          # two cluster filesystems get the same treatment because their
+          # kernel modules live in linux-modules-extra, whose contents
+          # differ between runner images.
+          - os: ubuntu-22.04
+            candidate: lustre-client
+          - os: ubuntu-22.04
+            candidate: gfs2
+          - os: ubuntu-22.04
+            candidate: ocfs2
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Probe ${{ matrix.candidate }}
+        run: |
+          set -o pipefail
+          bin/ci/probe-backend.sh "${{ matrix.candidate }}" \
+            2>&1 | tee "$RUNNER_TEMP/probe.log"
+
+      # Runs even when the probe failed: "could not be brought up, here
+      # is why" is the finding, and it belongs in the roll-up too.
+      - name: Extract the summary line
+        if: always()
+        run: |
+          grep '^SUMMARY|' "$RUNNER_TEMP/probe.log" > "$RUNNER_TEMP/summary.txt" \
+            || echo "SUMMARY|${{ matrix.candidate }}|NO OUTPUT|-|probe produced no summary line|" \
+               > "$RUNNER_TEMP/summary.txt"
+          sed -i "s/^SUMMARY|/SUMMARY|${{ matrix.os }}|/" "$RUNNER_TEMP/summary.txt"
+          cat "$RUNNER_TEMP/summary.txt"
+
+      - name: Upload the summary line
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: summary-${{ matrix.os }}-${{ matrix.candidate }}
+          path: ${{ runner.temp }}/summary.txt
+          retention-days: 30
+
+  # The probe answers "can this filesystem exist here?". This answers the
+  # follow-up: does the *backend* -- the thing a matrix row would call --
+  # actually drive it end to end? ext4 is the control.
+  backend-smoke:
+    name: eval-under loop --fs ${{ matrix.fs }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        fs: [ext4, gfs2, ocfs2]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install backend deps
+        run: bin/ci/install-backend.sh loop "${{ matrix.fs }}"
+
+      - name: Run the capability probe through the backend
+        run: |
+          sudo bin/eval-under loop --fs "${{ matrix.fs }}" --set-home -- \
+            bin/ci/fs-capabilities.sh
+
+  # One place to read the whole run. Without this, answering "which
+  # candidates came up?" means opening 24 job logs.
+  roll-up:
+    name: Roll-up
+    runs-on: ubuntu-24.04
+    needs: [probe, backend-smoke]
+    if: always()
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: summary-*
+          path: summaries
+          merge-multiple: false
+
+      - name: Render the table
+        run: bin/ci/render-probe-table.sh summaries | tee -a "$GITHUB_STEP_SUMMARY"

.github/workflows/reproduce.yaml

@@ -0,0 +1,127 @@
+name: Reproduce (on demand)
+
+# The support entry point: someone reports "X breaks on my $FILESYSTEM",
+# and you want a run of exactly that combination now -- without waiting
+# for the Monday schedule and without adding a matrix cell for a
+# filesystem we are not going to watch every week.
+#
+# Deliberately outside evals/matrix.yaml: no badge, no cron, no push
+# trigger. Every field is a dispatch input, including targets
+# (capabilities) that have no scheduled cells at all.
+#
+# The job body mirrors test.yaml's, minus the matrix fan-out and the
+# status publishing -- a reproduction must never rewrite the status that
+# the README and the report page show for master.
+#
+# All non-trivial shell logic lives in bin/ci/*.sh. See .claude/CLAUDE.md.
+
+on:
+  workflow_dispatch:
+    inputs:
+      backend:
+        description: "Filesystem to run under"
+        required: true
+        type: choice
+        default: nfs
+        options: [nfs, loop, beegfs]
+      backend-version:
+        description: >-
+          loop: filesystem (ext4, vfat, xfs, btrfs, gfs2, ocfs2, f2fs,
+          exfat, nilfs2, bcachefs). beegfs: point release (7.4.6, 8.1.0).
+          nfs: leave as n/a.
+        required: true
+        type: string
+        default: "n/a"
+      target:
+        description: "What to run under it"
+        required: true
+        type: choice
+        default: capabilities
+        options: [capabilities, git-annex, git, stress-ng, pjdfstest]
+      backend-options:
+        description: >-
+          Extra backend flags, e.g. "--sync" or "--no-root-squash"
+          (nfs). Run `bin/eval-under BACKEND --help`
+          for the list.
+        required: false
+        type: string
+        default: ""
+      runner:
+        description: "Runner image (beegfs requires ubuntu-22.04)"
+        required: true
+        type: choice
+        default: ubuntu-24.04
+        options: [ubuntu-24.04, ubuntu-22.04]
+
+# Named for the combination, so two people reproducing two different
+# reports do not cancel each other.
+concurrency:
+  group: >-
+    reproduce-${{ inputs.backend }}-${{ inputs.backend-version }}-${{ inputs.target }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  reproduce:
+    name: ${{ inputs.target }} under ${{ inputs.backend }} ${{ inputs.backend-version }}
+    runs-on: ${{ inputs.runner }}
+    timeout-minutes: 60
+    permissions:
+      contents: read
+      actions: read  # to download artifacts from con/git-annex
+    steps:
+      - uses: actions/checkout@v4
+
+      # Every input reaches the shell through env, never through template
+      # substitution into the run: body. A dispatch input is free-form
+      # text, and ${{ }} is expanded before bash sees the line, so
+      # interpolating it directly would make the input executable. This
+      # also matches how run-under.sh already takes backend-options.
+      - name: Install backend client dependencies
+        env:
+          BACKEND: ${{ inputs.backend }}
+          BACKEND_VERSION: ${{ inputs.backend-version }}
+        run: bin/ci/install-backend.sh "$BACKEND" "$BACKEND_VERSION"
+
+      - name: Install target ${{ inputs.target }}
+        env:
+          TARGET: ${{ inputs.target }}
+        run: bin/ci/install-target.sh "$TARGET"
+
+      - name: Fetch latest git-annex daily build from con/git-annex
+        if: inputs.target == 'git-annex'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: bin/ci/install-git-annex-daily.sh
+
+      - name: Configure git identity
+        run: |
+          git config --global user.email test@github.land
+          git config --global user.name  "GitHub Almighty"
+
+      - name: Run ${{ inputs.target }} under ${{ inputs.backend }}
+        env:
+          BACKEND: ${{ inputs.backend }}
+          BACKEND_VERSION: ${{ inputs.backend-version }}
+          TARGET: ${{ inputs.target }}
+          EVAL_UNDER_BACKEND_OPTS: ${{ inputs.backend-options }}
+        run: >-
+          sudo -E bin/ci/run-under.sh "$BACKEND" "$BACKEND_VERSION" "$TARGET"
+
+      - name: Dump failure logs
+        if: failure()
+        env:
+          BACKEND: ${{ inputs.backend }}
+          BACKEND_VERSION: ${{ inputs.backend-version }}
+          TARGET: ${{ inputs.target }}
+        run: >-
+          bin/ci/dump-failure-logs.sh "$BACKEND" "$BACKEND_VERSION" "$TARGET"
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: reproduce-logs-${{ inputs.backend }}-${{ inputs.target }}-${{ github.run_id }}
+          path: |
+            /var/log/beegfs-*
+            /opt/eval-under-src/git/t/test-results/**
+          if-no-files-found: ignore