diff --git a/.github/workflows/alcf.yml b/.github/workflows/alcf.yml new file mode 100644 index 00000000..a4ede851 --- /dev/null +++ b/.github/workflows/alcf.yml @@ -0,0 +1,125 @@ +name: ALCF CI + +# Bridges GitHub to ALCF GitLab CI (https://gitlab-ci.alcf.anl.gov/mschanen/oneAPI-jl), +# which pull-mirrors this repository and runs the LTS-stack test suite on Aurora (see +# .gitlab-ci.yml). This workflow makes the GitLab pipeline show up as a PR check: +# +# - push to main / same-repo PRs: the mirror already carries the branch; force a mirror +# sync (instead of waiting for the ~30 min schedule), then poll the pipeline for the +# head SHA and adopt its result. +# - fork PRs: never run automatically — GitLab CI jobs execute as an ALCF user on +# Aurora, so running fork code is opt-in. A maintainer applies the 'alcf-ci' label, +# which pushes the PR head to GitLab as branch gh-pr- and triggers a pipeline. +# Re-apply the label to re-run after new pushes. This workflow only moves refs; it +# never checks out or executes PR code. + +on: + push: + branches: [main] + pull_request: + pull_request_target: + types: [labeled] + +concurrency: + group: alcf-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + aurora: + name: LTS stack (ALCF GitLab CI, Aurora) + runs-on: ubuntu-latest + # PBS queue wait + 1 h walltime + build stage; generous but below the 6 h runner cap + timeout-minutes: 330 + if: >- + github.event_name == 'push' || + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'pull_request_target' && + github.event.label.name == 'alcf-ci' && + github.event.pull_request.head.repo.full_name != github.repository) + env: + GITLAB_HOST: gitlab-ci.alcf.anl.gov + GITLAB_PROJECT: mschanen/oneAPI-jl + API: https://gitlab-ci.alcf.anl.gov/api/v4/projects/238 + GITLAB_TOKEN: ${{ secrets.ALCF_GITLAB_TOKEN }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + steps: + - name: Push fork PR head to GitLab (label-gated) + if: github.event_name == 'pull_request_target' + env: + PR: ${{ github.event.pull_request.number }} + run: | + git init -q head && cd head + git fetch -q --depth=1 "https://github.com/$GITHUB_REPOSITORY" "pull/$PR/head" + if [ "$(git rev-parse FETCH_HEAD)" != "$HEAD_SHA" ]; then + echo "::error::PR head moved since labeling; re-apply the label" + exit 1 + fi + git push -q -f "https://oauth2:${GITLAB_TOKEN}@${GITLAB_HOST}/${GITLAB_PROJECT}.git" \ + "FETCH_HEAD:refs/heads/gh-pr-$PR" + + - name: Force mirror sync + if: github.event_name != 'pull_request_target' + run: | + # Best-effort: if this fails, the scheduled mirror sync still triggers the + # pipeline, just later; the polling step below tolerates the delay. + curl -sS -X POST -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$API/mirror/pull" \ + || echo "::warning::mirror sync request failed; relying on scheduled sync" + + - name: Wait for pipeline and adopt its result + run: | + # Transient API failures (network, rate limiting) must not fail the check: + # api() degrades to empty output, and the loops treat that as "try again". + api() { curl -sS --max-time 30 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } + + echo "waiting for a GitLab pipeline for $HEAD_SHA" + ID="" + for i in $(seq 1 80); do # up to 40 min for mirror sync + pipeline creation + ID=$(api "$API/pipelines?sha=$HEAD_SHA&order_by=id&sort=desc&per_page=1" \ + | jq -r '.[0].id // empty' 2>/dev/null || true) + if [ -n "$ID" ]; then break; fi + sleep 30 + done + if [ -z "$ID" ]; then + echo "::error::no GitLab pipeline appeared for $HEAD_SHA within 40 min (mirror not synced, or ref filtered by workflow rules)" + exit 1 + fi + URL="https://${GITLAB_HOST}/${GITLAB_PROJECT}/-/pipelines/$ID" + echo "pipeline: $URL" + echo "PIPELINE_ID=$ID" >> "$GITHUB_ENV" + echo "[ALCF GitLab pipeline $ID]($URL)" >> "$GITHUB_STEP_SUMMARY" + prev="" + while true; do + STATUS=$(api "$API/pipelines/$ID" | jq -r '.status // empty' 2>/dev/null || true) + if [ -n "$STATUS" ] && [ "$STATUS" != "$prev" ]; then echo "status: $STATUS"; prev="$STATUS"; fi + case "$STATUS" in + success) + exit 0 ;; + failed|canceled|skipped) + echo "failed jobs:" + api "$API/pipelines/$ID/jobs?per_page=100" \ + | jq -r '.[] | select(.status=="failed") | " \(.name): \(.web_url)"' 2>/dev/null \ + | tee -a "$GITHUB_STEP_SUMMARY" || true + echo "::error::GitLab pipeline $STATUS: $URL" + exit 1 ;; + esac + sleep 60 + done + + # The GitLab instance requires an ALCF login, so mirror the job logs into this + # run where any contributor can read them. + - name: Publish GitLab job logs + if: always() && env.PIPELINE_ID != '' + run: | + api() { curl -sS --max-time 60 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } + api "$API/pipelines/$PIPELINE_ID/jobs?per_page=100" \ + | jq -r '.[] | "\(.id)|\(.name)|\(.status)"' \ + | while IFS='|' read -r jid jname jstatus; do + echo "::group::${jname} — ${jstatus}" + api "$API/jobs/$jid/trace" + echo "" + echo "::endgroup::" + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 07c37f37..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: CI - -on: - push: - branches: - - main - tags: '*' - pull_request: - types: [opened, synchronize, reopened] - schedule: - - cron: '0 0 * * 0' - -permissions: - contents: read - -jobs: - self-runner: - # Spelled out so the PR checks box says which driver stack this exercises: this job is - # the LTS stack (ONEAPI_LTS=1, self-hosted Aurora runner), while buildkite covers the - # rolling stack. Without an explicit name GitHub derives one from the matrix - # ("self-runner (ubuntu-latest, 1, x64)"), which says nothing about the stack and - # misreports the runner as ubuntu-latest — `os` is unused, `runs-on` is self-hosted. - name: LTS stack (self-hosted, Julia ${{ matrix.julia-version }}) - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Pin each parallel test worker to a distinct GPU tile instead of - # oversubscribing device 0 (see test/runtests.jl). - ONEAPI_TEST_SPREAD_GPUS: '1' - ONEAPI_LTS: '1' - # Synchronize after every command-list submission to work around the - # Aurora LTS NEO dropped-tail corruption (see lib/level-zero/cmdlist.jl). - ONEAPI_SYNC_EACH_SUBMISSION: '1' - runs-on: [self-hosted, linux, X64] - strategy: - matrix: - os: [ubuntu-latest] - julia-version: ['1'] - julia-arch: [x64] - - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 1 - token: ${{ github.token }} - ssh-strict: false - - uses: julia-actions/setup-julia@latest - with: - version: ${{ matrix.julia-version }} - - uses: julia-actions/cache@v3 - # Build liboneapi_support.so from deps/src rather than using the registered - # oneAPI_Support_jll artifact, which lags behind the wrappers in this branch. - # Writes LocalPreferences.toml (plus a copy in test/, for Pkg.jl#2500) pointing - # oneAPI_Support_jll at the locally-built library. Not continue-on-error: if this - # fails we would silently fall back to the JLL and test the wrong library. - - name: Build the oneAPI support library - timeout-minutes: 90 - run: julia --color=yes --project=deps deps/build_local.jl - - uses: julia-actions/julia-buildpkg@latest - continue-on-error: true - # Must run after julia-buildpkg: Manifest.toml is gitignored, so the top-level - # project is not instantiated until then and `using oneAPI_Support_jll` would - # fail with "required but does not seem to be installed". - - name: Check the local support library is picked up - run: | - julia --color=yes --project=. -e ' - using TOML, oneAPI_Support_jll - want = TOML.parsefile("LocalPreferences.toml")["oneAPI_Support_jll"]["liboneapi_support_path"] - got = oneAPI_Support_jll.liboneapi_support_path - @info "support library" want got - want == got || error("oneAPI_Support_jll is not using the locally-built library")' - # Disable AVX512-FP16 host codegen on the Aurora Sapphire Rapids nodes. Under concurrent - # oneMKL load the native AVX512-FP16 path silently miscomputes *host* Float16 (e.g. the - # GPUArrays `A .* B .+ c` broadcast reference), failing tests even though the GPU result - # is correct (single-process clean; MXCSR clean; only the native-FP16 path, not Float32). - # `-C native,-avx512fp16` routes Float16 through Float32 and propagates to the Pkg.test - # subprocess and its parallel workers via Base.julia_cmd(). `julia-runtest` cannot pass a - # cpu-target, so invoke Pkg.test() directly. - - name: Run tests (AVX512-FP16 disabled) - run: julia -C "native,-avx512fp16" --color=yes --project=. -e 'import Pkg; Pkg.test(coverage=true)' - # Report coverage from this job too, not just from buildkite. Every LTS-gated branch - # (the queue registry, the coalesced reduction path, the :khronos codegen selection, - # the drain-before-free) is unreachable on the rolling stack, so without this upload - # Codecov counts all of it as uncovered. `--code-coverage` propagates to the Pkg.test - # subprocess and its parallel workers through Base.julia_cmd(), as the cpu-target - # above does. Directories match the buildkite julia-coverage plugin's. - - uses: julia-actions/julia-processcoverage@v1 - with: - directories: src,lib,examples - - uses: codecov/codecov-action@v5 - with: - files: lcov.info - token: ${{ secrets.CODECOV_TOKEN }} - # Coverage reporting must not be able to fail the build: this job is the only - # signal for the LTS stack, and a Codecov outage or a missing token is not a - # test failure. - fail_ci_if_error: false diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9043fe1e..c853602f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,7 +18,7 @@ jobs: DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} JULIA_DEBUG: Documenter - runs-on: [self-hosted, linux, X64] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..6e305ab0 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,131 @@ +# ALCF GitLab CI: LTS-stack tests on Aurora. +# +# This file only does something on the ALCF GitLab mirror +# (https://gitlab-ci.alcf.anl.gov/mschanen/oneAPI-jl), which pull-mirrors this repository +# and runs pipelines on mirrored branches. GitHub's .github/workflows/alcf.yml bridges the +# result back to PRs as a check. GitHub.com itself ignores this file. +# +# Layout: the Aurora PBS `debug` queue caps walltime at 1 h (the only 1-node queue), so +# everything that doesn't need a GPU — the support-library build and package +# instantiation/precompilation — runs on the login-node shell runner into a persistent +# depot on flare, and only the test run itself goes through PBS. + +include: + - project: 'anl/ci-resources/defaults' + ref: main + file: ['/runners.yml'] + +workflow: + rules: + - if: $CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + +default: + interruptible: true # force-pushes cancel stale runs instead of queueing behind them + +variables: + # Global (not per-script) for two reasons: the ~/.bashrc julia module load warns on + # stdout when JULIA_DEPOT_PATH is unset, corrupting the PBS job ID that Jacamar parses + # from qsub; and the CPU target must be identical in every job or the batch job rebuilds + # the package images the build job precompiled (icelake = login nodes, sapphirerapids = + # compute nodes, -avx512fp16 = host Float16 miscompile workaround, see ci.yml). + JULIA_DEPOT_PATH: /lus/flare/projects/Julia/mschanen/ci/julia_depot + JULIA_CPU_TARGET: "generic;icelake-server;sapphirerapids,-avx512fp16" + # Single-target variant for the runtime JIT (-C): multi-target strings are image-only + # syntax and are rejected at runtime. Must subtract the same avx512fp16 feature as the + # image targets above (see ci.yml for the miscompile this works around). + JULIA_JIT_TARGET: "native,-avx512fp16" + +stages: [build, test, coverage] + +.aurora-env: + script: + # Site profile supplies julia 1.12, ONEAPI_LTS=1 and the ZE_* settings. errexit off + # across it — profile.d scripts reference unset variables and return non-zero freely. + - set +eu; source /etc/profile >/dev/null 2>&1; module load julia >/dev/null 2>&1; set -eu + - command -v julia || { echo "ERROR no julia on PATH after site profile" >&2; exit 1; } + - julia --version; echo "depot=$JULIA_DEPOT_PATH" + +# Build the support library from deps/src rather than using the registered +# oneAPI_Support_jll artifact (parity with ci.yml). build_local.jl installs into the +# depot's scratch space, so with the depot on flare the products persist across jobs at a +# stable absolute path; the LocalPreferences.toml files it writes into the checkout just +# point there and travel to the test job as artifacts. +build:support: + stage: build + extends: .aurora-shell-runner + timeout: 3h + resource_group: aurora-ci-depot # pipelines share the depot; serialize against test:lts + script: + - !reference [.aurora-env, script] + - julia --color=yes --project=deps deps/build_local.jl + # Instantiate (and thereby precompile) both environments here so the 1 h batch job + # spends its walltime on tests, not on Pkg. Manifests are gitignored, so the test + # env must be resolved here with oneAPI dev'ed at the checkout (path "..") — a plain + # instantiate would silently test the registry release instead — and the resulting + # test/Manifest.toml travels to test:lts as an artifact. + - julia --color=yes --project=. -e 'using Pkg; Pkg.instantiate()' + - julia --color=yes --project=test -e 'using Pkg; Pkg.develop(path="."); Pkg.instantiate()' + # Coverage tooling into a depot-shared env here, where the network is available; + # test:lts only loads it. + - julia --color=yes --project=$JULIA_DEPOT_PATH/environments/coverage -e 'using Pkg; Pkg.add("CoverageTools")' + # Guard against silently falling back to the JLL and testing the wrong library + # (same check as ci.yml). + - julia --color=yes --project=. -e ' + using TOML, oneAPI_Support_jll; + want = TOML.parsefile("LocalPreferences.toml")["oneAPI_Support_jll"]["liboneapi_support_path"]; + got = oneAPI_Support_jll.liboneapi_support_path; + @info "support library" want got; + want == got || error("oneAPI_Support_jll is not using the locally-built library")' + artifacts: + paths: + - LocalPreferences.toml + - test/LocalPreferences.toml + - test/Manifest.toml + expire_in: 1 week + +test:lts: + stage: test + extends: .aurora-batch-runner + timeout: 5h # must cover PBS queue wait, not just the 1 h walltime + resource_group: aurora-ci-depot + variables: + ANL_AURORA_SCHEDULER_PARAMETERS: "-A Julia -q debug -l select=1,walltime=01:00:00,filesystems=home:flare,place=scatter" + script: + - !reference [.aurora-env, script] + # Each tile of a Max 1550 is its own device; the test runner spreads workers over + # them (ONEAPI_TEST_SPREAD_GPUS, see test/runtests.jl). ONEAPI_SYNC_EACH_SUBMISSION + # works around the Aurora LTS NEO dropped-tail corruption (lib/level-zero/cmdlist.jl). + - export ZE_FLAT_DEVICE_HIERARCHY=FLAT ONEAPI_LTS=1 ONEAPI_TEST_SPREAD_GPUS=1 ONEAPI_SYNC_EACH_SUBMISSION=1 + # -C on the command line, not just the JULIA_CPU_TARGET variable: the variable + # selects pkgimages but does not constrain the runtime JIT, and the AVX512-FP16 + # host-Float16 miscompile (broadcast Float16 reference failure, see ci.yml) needs + # the JIT constrained too. Propagates to the parallel test workers via + # Base.julia_cmd(). + # --code-coverage propagates to the parallel workers via Base.julia_cmd(), as the + # cpu-target does (see ci.yml). Every LTS-gated branch is unreachable on the rolling + # stack, so this job is the only coverage signal for those paths. + - julia -C "$JULIA_JIT_TARGET" --code-coverage=user --color=yes --project=test test/runtests.jl --quickfail + # Collapse the per-file .cov output into lcov.info for the upload job; directories + # match the buildkite julia-coverage plugin's. + - julia --color=yes --project=$JULIA_DEPOT_PATH/environments/coverage -e ' + using CoverageTools; + cov = mapreduce(process_folder, vcat, ["src", "lib", "examples"]); + LCOV.writefile("lcov.info", cov)' + artifacts: + paths: [lcov.info] + expire_in: 1 week + +# Coverage reporting must not be able to fail the build (parity with ci.yml). Only +# created when the CODECOV_TOKEN CI/CD variable is set on the GitLab project. +coverage:upload: + stage: coverage + extends: .aurora-shell-runner + timeout: 30m + needs: [test:lts] + rules: + - if: $CODECOV_TOKEN + allow_failure: true + script: + - curl -sSf -o codecov https://cli.codecov.io/latest/linux/codecov && chmod +x codecov + - ./codecov upload-process -t "$CODECOV_TOKEN" --git-service github + --slug JuliaGPU/oneAPI.jl --commit-sha "$CI_COMMIT_SHA" -f lcov.info diff --git a/docs/make.jl b/docs/make.jl index a5afba1d..0f2c6702 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -7,7 +7,13 @@ using Documenter using Documenter.Remotes using oneAPI -oneAPI.versioninfo() +# Informational only; the docs build runs on GPU-less hosted runners, where loading +# oneAPI works but querying the driver does not. +try + oneAPI.versioninfo() +catch err + @warn "oneAPI not functional on this host; skipping versioninfo" exception = err +end makedocs( sitename = "oneAPI.jl",