Run the LTS test suite on ALCF GitLab CI, bridged to GitHub PRs

.github/workflows/alcf.yml

@@ -0,0 +1,125 @@
+name: ALCF CI
+
+# Bridges GitHub to ALCF GitLab CI (https://gitlab-ci.alcf.anl.gov/mschanen/oneAPI-jl),
+# which pull-mirrors this repository and runs the LTS-stack test suite on Aurora (see
+# .gitlab-ci.yml). This workflow makes the GitLab pipeline show up as a PR check:
+#
+#  - push to main / same-repo PRs: the mirror already carries the branch; force a mirror
+#    sync (instead of waiting for the ~30 min schedule), then poll the pipeline for the
+#    head SHA and adopt its result.
+#  - fork PRs: never run automatically — GitLab CI jobs execute as an ALCF user on
+#    Aurora, so running fork code is opt-in. A maintainer applies the 'alcf-ci' label,
+#    which pushes the PR head to GitLab as branch gh-pr-<N> and triggers a pipeline.
+#    Re-apply the label to re-run after new pushes. This workflow only moves refs; it
+#    never checks out or executes PR code.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  pull_request_target:
+    types: [labeled]
+
+concurrency:
+  group: alcf-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  aurora:
+    name: LTS stack (ALCF GitLab CI, Aurora)
+    runs-on: ubuntu-latest
+    # PBS queue wait + 1 h walltime + build stage; generous but below the 6 h runner cap
+    timeout-minutes: 330
+    if: >-
+      github.event_name == 'push' ||
+      (github.event_name == 'pull_request' &&
+       github.event.pull_request.head.repo.full_name == github.repository) ||
+      (github.event_name == 'pull_request_target' &&
+       github.event.label.name == 'alcf-ci' &&
+       github.event.pull_request.head.repo.full_name != github.repository)
+    env:
+      GITLAB_HOST: gitlab-ci.alcf.anl.gov
+      GITLAB_PROJECT: mschanen/oneAPI-jl
+      API: https://gitlab-ci.alcf.anl.gov/api/v4/projects/238
+      GITLAB_TOKEN: ${{ secrets.ALCF_GITLAB_TOKEN }}
+      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+    steps:
+      - name: Push fork PR head to GitLab (label-gated)
+        if: github.event_name == 'pull_request_target'
+        env:
+          PR: ${{ github.event.pull_request.number }}
+        run: |
+          git init -q head && cd head
+          git fetch -q --depth=1 "https://github.com/$GITHUB_REPOSITORY" "pull/$PR/head"
+          if [ "$(git rev-parse FETCH_HEAD)" != "$HEAD_SHA" ]; then
+            echo "::error::PR head moved since labeling; re-apply the label"
+            exit 1
+          fi
+          git push -q -f "https://oauth2:${GITLAB_TOKEN}@${GITLAB_HOST}/${GITLAB_PROJECT}.git" \
+            "FETCH_HEAD:refs/heads/gh-pr-$PR"
+
+      - name: Force mirror sync
+        if: github.event_name != 'pull_request_target'
+        run: |
+          # Best-effort: if this fails, the scheduled mirror sync still triggers the
+          # pipeline, just later; the polling step below tolerates the delay.
+          curl -sS -X POST -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$API/mirror/pull" \
+            || echo "::warning::mirror sync request failed; relying on scheduled sync"
+
+      - name: Wait for pipeline and adopt its result
+        run: |
+          # Transient API failures (network, rate limiting) must not fail the check:
+          # api() degrades to empty output, and the loops treat that as "try again".
+          api() { curl -sS --max-time 30 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; }
+
+          echo "waiting for a GitLab pipeline for $HEAD_SHA"
+          ID=""
+          for i in $(seq 1 80); do   # up to 40 min for mirror sync + pipeline creation
+            ID=$(api "$API/pipelines?sha=$HEAD_SHA&order_by=id&sort=desc&per_page=1" \
+              | jq -r '.[0].id // empty' 2>/dev/null || true)
+            if [ -n "$ID" ]; then break; fi
+            sleep 30
+          done
+          if [ -z "$ID" ]; then
+            echo "::error::no GitLab pipeline appeared for $HEAD_SHA within 40 min (mirror not synced, or ref filtered by workflow rules)"
+            exit 1
+          fi
+          URL="https://${GITLAB_HOST}/${GITLAB_PROJECT}/-/pipelines/$ID"
+          echo "pipeline: $URL"
+          echo "PIPELINE_ID=$ID" >> "$GITHUB_ENV"
+          echo "[ALCF GitLab pipeline $ID]($URL)" >> "$GITHUB_STEP_SUMMARY"
+          prev=""
+          while true; do
+            STATUS=$(api "$API/pipelines/$ID" | jq -r '.status // empty' 2>/dev/null || true)
+            if [ -n "$STATUS" ] && [ "$STATUS" != "$prev" ]; then echo "status: $STATUS"; prev="$STATUS"; fi
+            case "$STATUS" in
+              success)
+                exit 0 ;;
+              failed|canceled|skipped)
+                echo "failed jobs:"
+                api "$API/pipelines/$ID/jobs?per_page=100" \
+                  | jq -r '.[] | select(.status=="failed") | "  \(.name): \(.web_url)"' 2>/dev/null \
+                  | tee -a "$GITHUB_STEP_SUMMARY" || true
+                echo "::error::GitLab pipeline $STATUS: $URL"
+                exit 1 ;;
+            esac
+            sleep 60
+          done
+
+      # The GitLab instance requires an ALCF login, so mirror the job logs into this
+      # run where any contributor can read them.
+      - name: Publish GitLab job logs
+        if: always() && env.PIPELINE_ID != ''
+        run: |
+          api() { curl -sS --max-time 60 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; }
+          api "$API/pipelines/$PIPELINE_ID/jobs?per_page=100" \
+            | jq -r '.[] | "\(.id)|\(.name)|\(.status)"' \
+            | while IFS='|' read -r jid jname jstatus; do
+                echo "::group::${jname} — ${jstatus}"
+                api "$API/jobs/$jid/trace"
+                echo ""
+                echo "::endgroup::"
+              done

.github/workflows/docs.yml

@@ -18,7 +18,7 @@ jobs:
       DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       JULIA_DEBUG: Documenter
-    runs-on: [self-hosted, linux, X64]
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v7

.gitlab-ci.yml

@@ -0,0 +1,131 @@
+# ALCF GitLab CI: LTS-stack tests on Aurora.
+#
+# This file only does something on the ALCF GitLab mirror
+# (https://gitlab-ci.alcf.anl.gov/mschanen/oneAPI-jl), which pull-mirrors this repository
+# and runs pipelines on mirrored branches. GitHub's .github/workflows/alcf.yml bridges the
+# result back to PRs as a check. GitHub.com itself ignores this file.
+#
+# Layout: the Aurora PBS `debug` queue caps walltime at 1 h (the only 1-node queue), so
+# everything that doesn't need a GPU — the support-library build and package
+# instantiation/precompilation — runs on the login-node shell runner into a persistent
+# depot on flare, and only the test run itself goes through PBS.
+
+include:
+  - project: 'anl/ci-resources/defaults'
+    ref: main
+    file: ['/runners.yml']
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+
+default:
+  interruptible: true  # force-pushes cancel stale runs instead of queueing behind them
+
+variables:
+  # Global (not per-script) for two reasons: the ~/.bashrc julia module load warns on
+  # stdout when JULIA_DEPOT_PATH is unset, corrupting the PBS job ID that Jacamar parses
+  # from qsub; and the CPU target must be identical in every job or the batch job rebuilds
+  # the package images the build job precompiled (icelake = login nodes, sapphirerapids =
+  # compute nodes, -avx512fp16 = host Float16 miscompile workaround, see ci.yml).
+  JULIA_DEPOT_PATH: /lus/flare/projects/Julia/mschanen/ci/julia_depot
+  JULIA_CPU_TARGET: "generic;icelake-server;sapphirerapids,-avx512fp16"
+  # Single-target variant for the runtime JIT (-C): multi-target strings are image-only
+  # syntax and are rejected at runtime. Must subtract the same avx512fp16 feature as the
+  # image targets above (see ci.yml for the miscompile this works around).
+  JULIA_JIT_TARGET: "native,-avx512fp16"
+
+stages: [build, test, coverage]
+
+.aurora-env:
+  script:
+    # Site profile supplies julia 1.12, ONEAPI_LTS=1 and the ZE_* settings. errexit off
+    # across it — profile.d scripts reference unset variables and return non-zero freely.
+    - set +eu; source /etc/profile >/dev/null 2>&1; module load julia >/dev/null 2>&1; set -eu
+    - command -v julia || { echo "ERROR no julia on PATH after site profile" >&2; exit 1; }
+    - julia --version; echo "depot=$JULIA_DEPOT_PATH"
+
+# Build the support library from deps/src rather than using the registered
+# oneAPI_Support_jll artifact (parity with ci.yml). build_local.jl installs into the
+# depot's scratch space, so with the depot on flare the products persist across jobs at a
+# stable absolute path; the LocalPreferences.toml files it writes into the checkout just
+# point there and travel to the test job as artifacts.
+build:support:
+  stage: build
+  extends: .aurora-shell-runner
+  timeout: 3h
+  resource_group: aurora-ci-depot  # pipelines share the depot; serialize against test:lts
+  script:
+    - !reference [.aurora-env, script]
+    - julia --color=yes --project=deps deps/build_local.jl
+    # Instantiate (and thereby precompile) both environments here so the 1 h batch job
+    # spends its walltime on tests, not on Pkg. Manifests are gitignored, so the test
+    # env must be resolved here with oneAPI dev'ed at the checkout (path "..") — a plain
+    # instantiate would silently test the registry release instead — and the resulting
+    # test/Manifest.toml travels to test:lts as an artifact.
+    - julia --color=yes --project=. -e 'using Pkg; Pkg.instantiate()'
+    - julia --color=yes --project=test -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()'
+    # Coverage tooling into a depot-shared env here, where the network is available;
+    # test:lts only loads it.
+    - julia --color=yes --project=$JULIA_DEPOT_PATH/environments/coverage -e 'using Pkg; Pkg.add("CoverageTools")'
+    # Guard against silently falling back to the JLL and testing the wrong library
+    # (same check as ci.yml).
+    - julia --color=yes --project=. -e '
+          using TOML, oneAPI_Support_jll;
+          want = TOML.parsefile("LocalPreferences.toml")["oneAPI_Support_jll"]["liboneapi_support_path"];
+          got = oneAPI_Support_jll.liboneapi_support_path;
+          @info "support library" want got;
+          want == got || error("oneAPI_Support_jll is not using the locally-built library")'
+  artifacts:
+    paths:
+      - LocalPreferences.toml
+      - test/LocalPreferences.toml
+      - test/Manifest.toml
+    expire_in: 1 week
+
+test:lts:
+  stage: test
+  extends: .aurora-batch-runner
+  timeout: 5h  # must cover PBS queue wait, not just the 1 h walltime
+  resource_group: aurora-ci-depot
+  variables:
+    ANL_AURORA_SCHEDULER_PARAMETERS: "-A Julia -q debug -l select=1,walltime=01:00:00,filesystems=home:flare,place=scatter"
+  script:
+    - !reference [.aurora-env, script]
+    # Each tile of a Max 1550 is its own device; the test runner spreads workers over
+    # them (ONEAPI_TEST_SPREAD_GPUS, see test/runtests.jl). ONEAPI_SYNC_EACH_SUBMISSION
+    # works around the Aurora LTS NEO dropped-tail corruption (lib/level-zero/cmdlist.jl).
+    - export ZE_FLAT_DEVICE_HIERARCHY=FLAT ONEAPI_LTS=1 ONEAPI_TEST_SPREAD_GPUS=1 ONEAPI_SYNC_EACH_SUBMISSION=1
+    # -C on the command line, not just the JULIA_CPU_TARGET variable: the variable
+    # selects pkgimages but does not constrain the runtime JIT, and the AVX512-FP16
+    # host-Float16 miscompile (broadcast Float16 reference failure, see ci.yml) needs
+    # the JIT constrained too. Propagates to the parallel test workers via
+    # Base.julia_cmd().
+    # --code-coverage propagates to the parallel workers via Base.julia_cmd(), as the
+    # cpu-target does (see ci.yml). Every LTS-gated branch is unreachable on the rolling
+    # stack, so this job is the only coverage signal for those paths.
+    - julia -C "$JULIA_JIT_TARGET" --code-coverage=user --color=yes --project=test test/runtests.jl --quickfail
+    # Collapse the per-file .cov output into lcov.info for the upload job; directories
+    # match the buildkite julia-coverage plugin's.
+    - julia --color=yes --project=$JULIA_DEPOT_PATH/environments/coverage -e '
+          using CoverageTools;
+          cov = mapreduce(process_folder, vcat, ["src", "lib", "examples"]);
+          LCOV.writefile("lcov.info", cov)'
+  artifacts:
+    paths: [lcov.info]
+    expire_in: 1 week
+
+# Coverage reporting must not be able to fail the build (parity with ci.yml). Only
+# created when the CODECOV_TOKEN CI/CD variable is set on the GitLab project.
+coverage:upload:
+  stage: coverage
+  extends: .aurora-shell-runner
+  timeout: 30m
+  needs: [test:lts]
+  rules:
+    - if: $CODECOV_TOKEN
+  allow_failure: true
+  script:
+    - curl -sSf -o codecov https://cli.codecov.io/latest/linux/codecov && chmod +x codecov
+    - ./codecov upload-process -t "$CODECOV_TOKEN" --git-service github
+        --slug JuliaGPU/oneAPI.jl --commit-sha "$CI_COMMIT_SHA" -f lcov.info

docs/make.jl

@@ -7,7 +7,13 @@ using Documenter
 using Documenter.Remotes
 using oneAPI
 
-oneAPI.versioninfo()
+# Informational only; the docs build runs on GPU-less hosted runners, where loading
+# oneAPI works but querying the driver does not.
+try
+    oneAPI.versioninfo()
+catch err
+    @warn "oneAPI not functional on this host; skipping versioninfo" exception = err
+end
 
 makedocs(
     sitename = "oneAPI.jl",