From 5ce0b3673ed6593a9c948dc61265529b8d1ba177 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Wed, 12 Aug 2026 19:05:43 +0000 Subject: [PATCH 1/2] Run the LTS test suite on ALCF GitLab CI, bridged to GitHub PRs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .gitlab-ci.yml: two-stage pipeline on the ALCF GitLab mirror (gitlab-ci.alcf.anl.gov/mschanen/oneAPI-jl), which pull-mirrors this repository. The Aurora PBS debug queue caps 1-node jobs at 1 h, so the support-library build and env instantiation/precompilation run on the login-node shell runner into a persistent depot on flare, and only the test run goes through PBS. Manifests are gitignored, so the build stage devs the checkout into the test env (a plain instantiate would silently test the registry release) and ships test/Manifest.toml to the batch job as an artifact. The runtime JIT gets the single-target "native,-avx512fp16" via -C (multi-target strings are image-only syntax), working around the same host-Float16 miscompile ci.yml documents. .github/workflows/alcf.yml: surfaces the GitLab pipeline as a PR check. Push to main and same-repo PRs force a pull-mirror sync and poll the pipeline for the head SHA; fork PRs run only when a maintainer applies the alcf-ci label (refs-only push as gh-pr-N — the workflow never checks out or executes PR code; the Aurora-side execution of labeled fork code is the deliberate, opt-in exception). Validated end-to-end on the mirror: build 96 s warm, full LTS suite green in ~19 min wall including queue wait. The existing self-hosted LTS job stays during burn-in and can be retired separately. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KziQYkXvpMmdQtBtpD8y5q --- .github/workflows/alcf.yml | 109 +++++++++++++++++++++++++++++++++++++ .gitlab-ci.yml | 101 ++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 .github/workflows/alcf.yml create mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/alcf.yml b/.github/workflows/alcf.yml new file mode 100644 index 00000000..5154fce1 --- /dev/null +++ b/.github/workflows/alcf.yml @@ -0,0 +1,109 @@ +name: ALCF CI + +# Bridges GitHub to ALCF GitLab CI (https://gitlab-ci.alcf.anl.gov/mschanen/oneAPI-jl), +# which pull-mirrors this repository and runs the LTS-stack test suite on Aurora (see +# .gitlab-ci.yml). This workflow makes the GitLab pipeline show up as a PR check: +# +# - push to main / same-repo PRs: the mirror already carries the branch; force a mirror +# sync (instead of waiting for the ~30 min schedule), then poll the pipeline for the +# head SHA and adopt its result. +# - fork PRs: never run automatically — GitLab CI jobs execute as an ALCF user on +# Aurora, so running fork code is opt-in. A maintainer applies the 'alcf-ci' label, +# which pushes the PR head to GitLab as branch gh-pr- and triggers a pipeline. +# Re-apply the label to re-run after new pushes. This workflow only moves refs; it +# never checks out or executes PR code. + +on: + push: + branches: [main] + pull_request: + pull_request_target: + types: [labeled] + +concurrency: + group: alcf-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + aurora: + name: LTS stack (ALCF GitLab CI, Aurora) + runs-on: ubuntu-latest + # PBS queue wait + 1 h walltime + build stage; generous but below the 6 h runner cap + timeout-minutes: 330 + if: >- + github.event_name == 'push' || + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'pull_request_target' && + github.event.label.name == 'alcf-ci' && + github.event.pull_request.head.repo.full_name != github.repository) + env: + GITLAB_HOST: gitlab-ci.alcf.anl.gov + GITLAB_PROJECT: mschanen/oneAPI-jl + API: https://gitlab-ci.alcf.anl.gov/api/v4/projects/238 + GITLAB_TOKEN: ${{ secrets.ALCF_GITLAB_TOKEN }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + steps: + - name: Push fork PR head to GitLab (label-gated) + if: github.event_name == 'pull_request_target' + env: + PR: ${{ github.event.pull_request.number }} + run: | + git init -q head && cd head + git fetch -q --depth=1 "https://github.com/$GITHUB_REPOSITORY" "pull/$PR/head" + if [ "$(git rev-parse FETCH_HEAD)" != "$HEAD_SHA" ]; then + echo "::error::PR head moved since labeling; re-apply the label" + exit 1 + fi + git push -q -f "https://oauth2:${GITLAB_TOKEN}@${GITLAB_HOST}/${GITLAB_PROJECT}.git" \ + "FETCH_HEAD:refs/heads/gh-pr-$PR" + + - name: Force mirror sync + if: github.event_name != 'pull_request_target' + run: | + # Best-effort: if this fails, the scheduled mirror sync still triggers the + # pipeline, just later; the polling step below tolerates the delay. + curl -sS -X POST -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$API/mirror/pull" \ + || echo "::warning::mirror sync request failed; relying on scheduled sync" + + - name: Wait for pipeline and adopt its result + run: | + # Transient API failures (network, rate limiting) must not fail the check: + # api() degrades to empty output, and the loops treat that as "try again". + api() { curl -sS --max-time 30 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } + + echo "waiting for a GitLab pipeline for $HEAD_SHA" + ID="" + for i in $(seq 1 80); do # up to 40 min for mirror sync + pipeline creation + ID=$(api "$API/pipelines?sha=$HEAD_SHA&order_by=id&sort=desc&per_page=1" \ + | jq -r '.[0].id // empty' 2>/dev/null || true) + if [ -n "$ID" ]; then break; fi + sleep 30 + done + if [ -z "$ID" ]; then + echo "::error::no GitLab pipeline appeared for $HEAD_SHA within 40 min (mirror not synced, or ref filtered by workflow rules)" + exit 1 + fi + URL="https://${GITLAB_HOST}/${GITLAB_PROJECT}/-/pipelines/$ID" + echo "pipeline: $URL" + echo "[ALCF GitLab pipeline $ID]($URL)" >> "$GITHUB_STEP_SUMMARY" + prev="" + while true; do + STATUS=$(api "$API/pipelines/$ID" | jq -r '.status // empty' 2>/dev/null || true) + if [ -n "$STATUS" ] && [ "$STATUS" != "$prev" ]; then echo "status: $STATUS"; prev="$STATUS"; fi + case "$STATUS" in + success) + exit 0 ;; + failed|canceled|skipped) + echo "failed jobs:" + api "$API/pipelines/$ID/jobs?per_page=100" \ + | jq -r '.[] | select(.status=="failed") | " \(.name): \(.web_url)"' 2>/dev/null \ + | tee -a "$GITHUB_STEP_SUMMARY" || true + echo "::error::GitLab pipeline $STATUS: $URL" + exit 1 ;; + esac + sleep 60 + done diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..68ec980f --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,101 @@ +# ALCF GitLab CI: LTS-stack tests on Aurora. +# +# This file only does something on the ALCF GitLab mirror +# (https://gitlab-ci.alcf.anl.gov/mschanen/oneAPI-jl), which pull-mirrors this repository +# and runs pipelines on mirrored branches. GitHub's .github/workflows/alcf.yml bridges the +# result back to PRs as a check. GitHub.com itself ignores this file. +# +# Layout: the Aurora PBS `debug` queue caps walltime at 1 h (the only 1-node queue), so +# everything that doesn't need a GPU — the support-library build and package +# instantiation/precompilation — runs on the login-node shell runner into a persistent +# depot on flare, and only the test run itself goes through PBS. + +include: + - project: 'anl/ci-resources/defaults' + ref: main + file: ['/runners.yml'] + +workflow: + rules: + - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + +default: + interruptible: true # force-pushes cancel stale runs instead of queueing behind them + +variables: + # Global (not per-script) for two reasons: the ~/.bashrc julia module load warns on + # stdout when JULIA_DEPOT_PATH is unset, corrupting the PBS job ID that Jacamar parses + # from qsub; and the CPU target must be identical in every job or the batch job rebuilds + # the package images the build job precompiled (icelake = login nodes, sapphirerapids = + # compute nodes, -avx512fp16 = host Float16 miscompile workaround, see ci.yml). + JULIA_DEPOT_PATH: /lus/flare/projects/Julia/mschanen/ci/julia_depot + JULIA_CPU_TARGET: "generic;icelake-server;sapphirerapids,-avx512fp16" + # Single-target variant for the runtime JIT (-C): multi-target strings are image-only + # syntax and are rejected at runtime. Must subtract the same avx512fp16 feature as the + # image targets above (see ci.yml for the miscompile this works around). + JULIA_JIT_TARGET: "native,-avx512fp16" + +stages: [build, test] + +.aurora-env: + script: + # Site profile supplies julia 1.12, ONEAPI_LTS=1 and the ZE_* settings. errexit off + # across it — profile.d scripts reference unset variables and return non-zero freely. + - set +eu; source /etc/profile >/dev/null 2>&1; module load julia >/dev/null 2>&1; set -eu + - command -v julia || { echo "ERROR no julia on PATH after site profile" >&2; exit 1; } + - julia --version; echo "depot=$JULIA_DEPOT_PATH" + +# Build the support library from deps/src rather than using the registered +# oneAPI_Support_jll artifact (parity with ci.yml). build_local.jl installs into the +# depot's scratch space, so with the depot on flare the products persist across jobs at a +# stable absolute path; the LocalPreferences.toml files it writes into the checkout just +# point there and travel to the test job as artifacts. +build:support: + stage: build + extends: .aurora-shell-runner + timeout: 3h + resource_group: aurora-ci-depot # pipelines share the depot; serialize against test:lts + script: + - !reference [.aurora-env, script] + - julia --color=yes --project=deps deps/build_local.jl + # Instantiate (and thereby precompile) both environments here so the 1 h batch job + # spends its walltime on tests, not on Pkg. Manifests are gitignored, so the test + # env must be resolved here with oneAPI dev'ed at the checkout (path "..") — a plain + # instantiate would silently test the registry release instead — and the resulting + # test/Manifest.toml travels to test:lts as an artifact. + - julia --color=yes --project=. -e 'using Pkg; Pkg.instantiate()' + - julia --color=yes --project=test -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + # Guard against silently falling back to the JLL and testing the wrong library + # (same check as ci.yml). + - julia --color=yes --project=. -e ' + using TOML, oneAPI_Support_jll; + want = TOML.parsefile("LocalPreferences.toml")["oneAPI_Support_jll"]["liboneapi_support_path"]; + got = oneAPI_Support_jll.liboneapi_support_path; + @info "support library" want got; + want == got || error("oneAPI_Support_jll is not using the locally-built library")' + artifacts: + paths: + - LocalPreferences.toml + - test/LocalPreferences.toml + - test/Manifest.toml + expire_in: 1 week + +test:lts: + stage: test + extends: .aurora-batch-runner + timeout: 5h # must cover PBS queue wait, not just the 1 h walltime + resource_group: aurora-ci-depot + variables: + ANL_AURORA_SCHEDULER_PARAMETERS: "-A Julia -q debug -l select=1,walltime=01:00:00,filesystems=home:flare,place=scatter" + script: + - !reference [.aurora-env, script] + # Each tile of a Max 1550 is its own device; the test runner spreads workers over + # them (ONEAPI_TEST_SPREAD_GPUS, see test/runtests.jl). ONEAPI_SYNC_EACH_SUBMISSION + # works around the Aurora LTS NEO dropped-tail corruption (lib/level-zero/cmdlist.jl). + - export ZE_FLAT_DEVICE_HIERARCHY=FLAT ONEAPI_LTS=1 ONEAPI_TEST_SPREAD_GPUS=1 ONEAPI_SYNC_EACH_SUBMISSION=1 + # -C on the command line, not just the JULIA_CPU_TARGET variable: the variable + # selects pkgimages but does not constrain the runtime JIT, and the AVX512-FP16 + # host-Float16 miscompile (broadcast Float16 reference failure, see ci.yml) needs + # the JIT constrained too. Propagates to the parallel test workers via + # Base.julia_cmd(). + - julia -C "$JULIA_JIT_TARGET" --color=yes --project=test test/runtests.jl --jobs=4 --quickfail From 749e0dd4848d18e9670bdb24a088bc00568fadf6 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Wed, 12 Aug 2026 19:27:00 +0000 Subject: [PATCH 2/2] Test the fork-PR ALCF label path (throwaway) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KziQYkXvpMmdQtBtpD8y5q --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 16be7180..402a3dc5 100644 --- a/README.md +++ b/README.md @@ -408,3 +408,5 @@ The discovered paths will be written to a global file with preferences, typicall `$HOME/.julia/environments/vX.Y/LocalPreferences.toml` (where `vX.Y` refers to the Julia version you are using). You can modify this file, or remove it when you want to revert to default set of binaries. + +