From 01b2636b488e06fdb148629e955161ae28a9b4b1 Mon Sep 17 00:00:00 2001 From: Pierre Granger Date: Thu, 5 Nov 2026 20:56:06 +0100 Subject: [PATCH] ci: GitHub Actions with HDF5 1.14 and 1.12 from the LCG views on CVMFS Stock Ubuntu runners only offer HDF5 1.10 through apt, which CMakeLists.txt refuses, so both jobs mount sft.cern.ch and source an LCG view: - hdf5-1_14: ubuntu-24.04, LCG_110 x86_64-ubuntu2404-gcc13-opt (HDF5 1.14.6) - hdf5-1_12: ubuntu-22.04, LCG_105 x86_64-ubuntu2204-gcc11-opt (HDF5 1.12.2) Each job runs ci/github-ci.sh, which configures with the new SPINE_READER_REQUIRE_PYTHON=ON (a missing Python 3 now fails the configuration instead of silently dropping the generated-model and golden-member checks), builds, and runs ctest. The same script is the local dry run, in CERN's SFT Ubuntu images on CVMFS (ci/EGM.md); both jobs pass there on this commit. ci/check_tag_version.sh refuses a release tag that differs from project(VERSION). It is meant to be run before pushing a tag; CI runs it again on tag pushes. The workflow has read-only permissions, no secrets, and pins both actions by commit SHA. --- .github/workflows/ci.yml | 38 ++++++++++++++++++++++ EGM.md | 22 +++++++++---- README.md | 6 ++++ ci/EGM.md | 70 +++++++++++++++++++++++++++++++++------- ci/check_tag_version.sh | 27 ++++++++++++++++ ci/github-ci.sh | 28 ++++++++++++++++ tests/CMakeLists.txt | 11 ++++++- 7 files changed, 183 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100755 ci/check_tag_version.sh create mode 100755 ci/github-ci.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..28d3db9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +# Build spine-reader and run ctest with HDF5 1.14 and 1.12, using the LCG views on CVMFS. +# Stock Ubuntu runners only offer HDF5 1.10 through apt, which CMakeLists.txt refuses. +# Each job runs ci/github-ci.sh, which can also be run locally (see ci/EGM.md). +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + hdf5-1_14: + name: HDF5 1.14 (LCG_110, Ubuntu 24.04) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: cvmfs-contrib/github-action-cvmfs@72f2a8d366043978d6aebee154bc9ed321c2f2a8 # v5.6 + with: + cvmfs_repositories: sft.cern.ch + - name: Build and test + run: ci/github-ci.sh /cvmfs/sft.cern.ch/lcg/views/LCG_110/x86_64-ubuntu2404-gcc13-opt/setup.sh + - name: Tag matches project(VERSION) + if: startsWith(github.ref, 'refs/tags/') + run: ci/check_tag_version.sh "$GITHUB_REF_NAME" + + hdf5-1_12: + name: HDF5 1.12 (LCG_105, Ubuntu 22.04) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: cvmfs-contrib/github-action-cvmfs@72f2a8d366043978d6aebee154bc9ed321c2f2a8 # v5.6 + with: + cvmfs_repositories: sft.cern.ch + - name: Build and test + run: ci/github-ci.sh /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-ubuntu2204-gcc11-opt/setup.sh diff --git a/EGM.md b/EGM.md index 692a6c3..8dc7228 100644 --- a/EGM.md +++ b/EGM.md @@ -26,13 +26,16 @@ The model is generated at build-maintenance time, not at configure time: `tools/gen/generate.py` -> `include/spine/model/`, `include/spine/model.hpp`, `src/model/`. The root folder holds no source files: only `CMakeLists.txt`, `cmake/SpineReaderConfig.cmake.in`, -`schema/spine-1.4.0.json`, `README.md`, `EGM.md`, `LICENSE` and `.gitignore`. +`schema/spine-1.4.0.json`, `README.md`, `EGM.md`, `LICENSE`, `.gitignore` and the GitHub Actions +workflow `.github/workflows/ci.yml` (documented in [./ci/EGM.md](./ci/EGM.md)). Build and packaging (`CMakeLists.txt`): - Options: `SPINE_READER_BUILD_TESTS` and `SPINE_READER_BUILD_TOOLS` (both default ON for a top-level build, OFF as a subproject) and `SPINE_READER_SANITIZE` (`OFF`/`ON`/`thread`). The - default build type, when none is given, is `RelWithDebInfo`. + default build type, when none is given, is `RelWithDebInfo`. `tests/CMakeLists.txt` adds + `SPINE_READER_REQUIRE_PYTHON` (default OFF; CI turns it on): a missing Python 3 then fails the + configuration instead of leaving out the Python-based checks. - `cmake --install` installs the library, `spine-dump` (when the tools are built) and the headers under `include/spine`. Only headers are installed: the `EGM.md` files are excluded. - It also exports the CMake package `SpineReader` (`SpineReaderTargets`, version compatibility @@ -66,8 +69,10 @@ Build and packaging (`CMakeLists.txt`): - [./tools/fixtures/EGM.md](./tools/fixtures/EGM.md) - Golden test files written by SPINE's own writer, plus their expected JSON and `tests/golden_members.inc`. - examples: [./examples/EGM.md](./examples/EGM.md) - The `spine-dump` command-line tool. -- ci: [./ci/EGM.md](./ci/EGM.md) - `sanitize.sh` (ASan/UBSan or TSan on EL9) and `sl7-ups.sh` - (SL7 container, UPS gcc 12 and HDF5 1.12.2). +- ci: [./ci/EGM.md](./ci/EGM.md) - the GitHub Actions workflow and `github-ci.sh` (HDF5 1.14 and + 1.12 from LCG views on CVMFS), `check_tag_version.sh` (release tag against + `project(VERSION)`), `sanitize.sh` (ASan/UBSan or TSan on EL9) and `sl7-ups.sh` (SL7 container, + UPS gcc 12 and HDF5 1.12.2). - tests (not documented by contract): doctest unit tests and the test data under `tests/data/`. ## Global Invariants @@ -102,7 +107,9 @@ Build and packaging (`CMakeLists.txt`): restoring it afterwards. - Developer tools only: `tools/gen` rewrites the generated model; `tools/fixtures` clones SPINE tags and rewrites `tests/data/fixtures/` and `tests/golden_members.inc`; `ci/` scripts - create `build-asan/`, `build-tsan/` and `build-sl7/`. + create `build-ci/`, `build-asan/`, `build-tsan/` and `build-sl7/`. +- CI: the GitHub Actions workflow builds and tests on GitHub's hosted runners, with read-only + access to the repository and no secrets. ## System Verification @@ -113,5 +120,8 @@ Build and packaging (`CMakeLists.txt`): `spine_reader_real_files` (skipped unless `SPINE_READER_REAL_FILES` names a directory). - `spine_reader_tests` excludes the `real files*` test cases, which `spine_reader_real_files` runs. `spine_reader_generated_model` and `spine_reader_golden_members` are registered only - when CMake finds Python 3, and `spine_dump_output` only when the tools are built. + when CMake finds Python 3 (with `SPINE_READER_REQUIRE_PYTHON=ON`, a missing Python fails the + configuration), and `spine_dump_output` only when the tools are built. +- CI: `.github/workflows/ci.yml` runs `ci/github-ci.sh` with HDF5 1.14 (LCG_110, Ubuntu 24.04) + and HDF5 1.12 (LCG_105, Ubuntu 22.04) on every push and pull request. - Other toolchains: `ci/sl7-ups.sh`, `ci/sanitize.sh`, `ci/sanitize.sh thread`. diff --git a/README.md b/README.md index 669b781..b8b58b6 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,12 @@ The test files in `tests/data/` come with the scripts that write them: `core/mak writer) and `tools/fixtures/` (golden fixtures). `ci/sl7-ups.sh` builds with the SL7 UPS toolchain (gcc 12, HDF5 1.12.2); `ci/sanitize.sh` builds with ASan and UBSan (`ci/sanitize.sh thread`: ThreadSanitizer). +CI (`.github/workflows/ci.yml`) builds and runs ctest on every push and pull request, with HDF5 +1.14 (LCG_110, Ubuntu 24.04) and HDF5 1.12 (LCG_105, Ubuntu 22.04) from the LCG views on CVMFS. +Both jobs run `ci/github-ci.sh`, which configures with `-DSPINE_READER_REQUIRE_PYTHON=ON`: a missing +Python 3 then fails the configuration instead of silently leaving out the Python-based checks. +`ci/EGM.md` shows how to run the same jobs locally. + ## Conventions - C++17 in everything installed; no HDF5 type appears in a public header. diff --git a/ci/EGM.md b/ci/EGM.md index a9c63ee..11d9e62 100644 --- a/ci/EGM.md +++ b/ci/EGM.md @@ -1,14 +1,41 @@ # ci ## Overview -Two bash scripts that build and test spine-reader in configurations that the -default build does not cover. `sanitize.sh` builds with sanitizers on EL9 using -the CVMFS LCG_110 view. `sl7-ups.sh` builds inside an SL7 container with the -UPS toolchain that the experiments' LArSoft stacks use. Both scripts find the -repository from their own location, so they can be run from any working +Bash scripts that build and test spine-reader in configurations the default build does not +cover, plus the pieces of the GitHub Actions workflow (`.github/workflows/ci.yml`). +- `github-ci.sh` is what each CI job runs, and what the local dry run runs. +- `check_tag_version.sh` checks a release tag against `project(VERSION)`. +- `sanitize.sh` builds with sanitizers on EL9, using the CVMFS LCG_110 view. +- `sl7-ups.sh` builds inside an SL7 container with the UPS toolchain that the experiments' + LArSoft stacks use. + +Every script finds the repository from its own location, so it can be run from any working directory. ## Interface +- `github-ci.sh [build directory]` + - Sources the given LCG view, then configures `/` (default `build-ci`) + with `-DSPINE_READER_REQUIRE_PYTHON=ON`, builds with `-j$(nproc)`, and runs + `ctest --output-on-failure`. No build type is given, so the project default + (`RelWithDebInfo`) applies, as in CI. + - Prints the OS, the compiler (`$CXX`, set by the LCG view), CMake, Python and the view. + - Exit codes: 2 on bad arguments; otherwise the first failing command's code. +- `check_tag_version.sh vX.Y.Z` + - Reads `project(spine-reader VERSION X.Y.Z ...)` from `CMakeLists.txt`. + - Exits 0 when the argument equals `vX.Y.Z`, 1 when it does not or when no version line is + found, and 2 on bad arguments. Messages name both versions. + - Run it before pushing a release tag. CI runs it again on every tag push. +- `.github/workflows/ci.yml` (in the repository's `.github/` folder) + - Runs on every push (branches and tags), every pull request and manual dispatch, with + `permissions: contents: read` and no secrets. + - Job `hdf5-1_14`: `ubuntu-24.04`, LCG_110 `x86_64-ubuntu2404-gcc13-opt` (HDF5 1.14.6, gcc 13 + from the runner, CMake 3.30, Python 3.13). On a tag push it also runs + `check_tag_version.sh "$GITHUB_REF_NAME"`. + - Job `hdf5-1_12`: `ubuntu-22.04`, LCG_105 `x86_64-ubuntu2204-gcc11-opt` (HDF5 1.12.2, gcc 11 + from the runner, CMake 3.26, Python 3.9). + - Each job checks out the repository, mounts `sft.cern.ch` with + `cvmfs-contrib/github-action-cvmfs`, and runs `github-ci.sh` with its view. The two actions + are pinned by commit SHA, with the version in a comment. - `sanitize.sh [mode]` - With no argument (`mode=address`, the default) it configures with `-DSPINE_READER_SANITIZE=ON`, which enables ASan and UBSan with @@ -31,9 +58,13 @@ directory. builds with `-j8`, and runs `build-sl7/tests/spine_reader_tests` directly. ## Invariants -- `sanitize.sh` runs `set -e` before it sources the LCG view, so a failed setup - stops the script. `sl7-ups.sh` runs `set -e` only after the UPS `setup` calls, - because those return non-zero even when they succeed. Keep that order. +- The LCG views for Ubuntu use the system compiler, which GitHub's runners provide + (`ubuntu-24.04`: gcc 13; `ubuntu-22.04`: gcc 11). +- `github-ci.sh` sources the view before it enables `set -euo pipefail`, because LCG setup + scripts are not `-e`/`-u` clean. +- `sanitize.sh` runs `set -e` before it sources the LCG view, so a failed setup stops the script. + `sl7-ups.sh` runs `set -e` only after the UPS `setup` calls, because those return non-zero + even when they succeed. Keep that order. - `sl7-ups.sh` runs only the `spine_reader_tests` binary. It does not run `ctest`, so ctest-only entries such as `spine_dump_output` and the Python-based generated-model checks are not run in the SL7 build. @@ -43,14 +74,29 @@ directory. - The toolchain versions in `sl7-ups.sh` (gcc 12.1 / e26, HDF5 1.12.2) are the targets the UPS stacks build against. The top-level `CMakeLists.txt` requires `HDF5 1.12` or newer. +- A release tag is never moved or deleted. If the CI tag check fails after a push, the fix is + a new patch release. ## Side Effects -- Create and overwrite the build directories `build-asan/`, `build-tsan/` and - `build-sl7/` in the repository root. -- Read from CVMFS: the LCG view, `setup_larsoft.sh` and UPS products, and the - SL7 container image. `sl7-ups.sh` needs `apptainer` on the host. +- Create and overwrite the build directories `build-ci/` (or the one given to `github-ci.sh`), + `build-asan/`, `build-tsan/` and `build-sl7/` in the repository root. `build*/` is ignored by + git. +- Read from CVMFS: the LCG views, `setup_larsoft.sh` and UPS products, and the container + images. `sl7-ups.sh` and the dry run below need `apptainer` on the host. +- The workflow runs on GitHub's hosted runners, with read-only access to the repository. ## Verification - Run: `ci/sanitize.sh` (ASan and UBSan), `ci/sanitize.sh thread` (TSan), and `ci/sl7-ups.sh` (SL7 UPS toolchain). +- Dry run of the two CI jobs, in CERN's SFT build images on CVMFS (GitHub's runner images + differ, so the first CI run is the final check): + + ```bash + img=/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/sft/docker + apptainer exec -B /cvmfs -B /tmp $img/ubuntu24:latest bash -e ci/github-ci.sh \ + /cvmfs/sft.cern.ch/lcg/views/LCG_110/x86_64-ubuntu2404-gcc13-opt/setup.sh build-ci-1.14 + apptainer exec -B /cvmfs -B /tmp $img/ubuntu22:latest bash -e ci/github-ci.sh \ + /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-ubuntu2204-gcc11-opt/setup.sh build-ci-1.12 + ``` +- Tag check: `ci/check_tag_version.sh v` exits 0; any other tag exits 1. - The default build these scripts vary: `source /cvmfs/sft.cern.ch/lcg/views/LCG_110/x86_64-el9-gcc13-opt/setup.sh && cmake -S . -B build && cmake --build build -j8 && ctest --test-dir build --output-on-failure` diff --git a/ci/check_tag_version.sh b/ci/check_tag_version.sh new file mode 100755 index 0000000..0ec9134 --- /dev/null +++ b/ci/check_tag_version.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Refuse a release tag that differs from project(VERSION) in CMakeLists.txt. +# Run it before pushing a tag; CI runs it again on every tag push, as a backstop. +# +# ci/check_tag_version.sh vX.Y.Z +# +# Exit codes: 0 the tag matches; 1 it does not (or no version was found); 2 usage. +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 vX.Y.Z" >&2 + exit 2 +fi +tag=$1 +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + +version=$(sed -n -E 's/^project\(spine-reader VERSION ([0-9]+\.[0-9]+\.[0-9]+)[ )].*/\1/p' \ + "$repo/CMakeLists.txt") +if [[ -z "$version" ]]; then + echo "check_tag_version: no 'project(spine-reader VERSION X.Y.Z' line in $repo/CMakeLists.txt" >&2 + exit 1 +fi +if [[ "$tag" != "v$version" ]]; then + echo "check_tag_version: tag $tag does not match project(VERSION) $version (expected v$version)" >&2 + exit 1 +fi +echo "check_tag_version: tag $tag matches project(VERSION) $version" diff --git a/ci/github-ci.sh b/ci/github-ci.sh new file mode 100755 index 0000000..6b9b56d --- /dev/null +++ b/ci/github-ci.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Build and test spine-reader the way the GitHub workflow (.github/workflows/ci.yml) does: +# source an LCG view, configure with SPINE_READER_REQUIRE_PYTHON=ON, build, run ctest. +# The same script is used for the local dry run (see ci/EGM.md). +# +# ci/github-ci.sh [build directory, default build-ci] +set -uo pipefail + +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "usage: $0 [build directory]" >&2 + exit 2 +fi +view=$1 +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +build=$repo/${2:-build-ci} + +# LCG setup scripts are not -e / -u clean: source the view before strict mode. +set +eu +# shellcheck disable=SC1090 +source "$view" +set -euo pipefail + +echo "== $(. /etc/os-release && echo "$PRETTY_NAME") | $("${CXX:-g++}" --version | head -1)" +echo "== $(cmake --version | head -1) | $(python3 --version) | view $view" + +cmake -S "$repo" -B "$build" -DSPINE_READER_REQUIRE_PYTHON=ON +cmake --build "$build" -j"$(nproc)" +ctest --test-dir "$build" --output-on-failure diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8711e4e..169dee2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -30,7 +30,16 @@ endif() # The generated model (include/spine/model, src/model) must be what tools/gen produces from the # committed schema, which was dumped from SPINE: this is what ties every default value, field # type and assign pointer of the model to SPINE. It needs only a Python 3 interpreter. -find_package(Python3 COMPONENTS Interpreter QUIET) +# SPINE_READER_REQUIRE_PYTHON (CI turns it on) makes a missing interpreter a configuration +# error instead of silently leaving these checks out. +option(SPINE_READER_REQUIRE_PYTHON + "Fail the configuration if Python 3 is missing, instead of skipping the Python-based checks" + OFF) +if(SPINE_READER_REQUIRE_PYTHON) + find_package(Python3 REQUIRED COMPONENTS Interpreter) +else() + find_package(Python3 COMPONENTS Interpreter QUIET) +endif() if(Python3_Interpreter_FOUND) add_test(NAME spine_reader_generated_model COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/gen/generate.py --check