diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..28d3db9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +# Build spine-reader and run ctest with HDF5 1.14 and 1.12, using the LCG views on CVMFS. +# Stock Ubuntu runners only offer HDF5 1.10 through apt, which CMakeLists.txt refuses. +# Each job runs ci/github-ci.sh, which can also be run locally (see ci/EGM.md). +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + hdf5-1_14: + name: HDF5 1.14 (LCG_110, Ubuntu 24.04) + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: cvmfs-contrib/github-action-cvmfs@72f2a8d366043978d6aebee154bc9ed321c2f2a8 # v5.6 + with: + cvmfs_repositories: sft.cern.ch + - name: Build and test + run: ci/github-ci.sh /cvmfs/sft.cern.ch/lcg/views/LCG_110/x86_64-ubuntu2404-gcc13-opt/setup.sh + - name: Tag matches project(VERSION) + if: startsWith(github.ref, 'refs/tags/') + run: ci/check_tag_version.sh "$GITHUB_REF_NAME" + + hdf5-1_12: + name: HDF5 1.12 (LCG_105, Ubuntu 22.04) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: cvmfs-contrib/github-action-cvmfs@72f2a8d366043978d6aebee154bc9ed321c2f2a8 # v5.6 + with: + cvmfs_repositories: sft.cern.ch + - name: Build and test + run: ci/github-ci.sh /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-ubuntu2204-gcc11-opt/setup.sh diff --git a/EGM.md b/EGM.md index 692a6c3..8dc7228 100644 --- a/EGM.md +++ b/EGM.md @@ -26,13 +26,16 @@ The model is generated at build-maintenance time, not at configure time: `tools/gen/generate.py` -> `include/spine/model/`, `include/spine/model.hpp`, `src/model/`. The root folder holds no source files: only `CMakeLists.txt`, `cmake/SpineReaderConfig.cmake.in`, -`schema/spine-1.4.0.json`, `README.md`, `EGM.md`, `LICENSE` and `.gitignore`. +`schema/spine-1.4.0.json`, `README.md`, `EGM.md`, `LICENSE`, `.gitignore` and the GitHub Actions +workflow `.github/workflows/ci.yml` (documented in [./ci/EGM.md](./ci/EGM.md)). Build and packaging (`CMakeLists.txt`): - Options: `SPINE_READER_BUILD_TESTS` and `SPINE_READER_BUILD_TOOLS` (both default ON for a top-level build, OFF as a subproject) and `SPINE_READER_SANITIZE` (`OFF`/`ON`/`thread`). The - default build type, when none is given, is `RelWithDebInfo`. + default build type, when none is given, is `RelWithDebInfo`. `tests/CMakeLists.txt` adds + `SPINE_READER_REQUIRE_PYTHON` (default OFF; CI turns it on): a missing Python 3 then fails the + configuration instead of leaving out the Python-based checks. - `cmake --install` installs the library, `spine-dump` (when the tools are built) and the headers under `include/spine`. Only headers are installed: the `EGM.md` files are excluded. - It also exports the CMake package `SpineReader` (`SpineReaderTargets`, version compatibility @@ -66,8 +69,10 @@ Build and packaging (`CMakeLists.txt`): - [./tools/fixtures/EGM.md](./tools/fixtures/EGM.md) - Golden test files written by SPINE's own writer, plus their expected JSON and `tests/golden_members.inc`. - examples: [./examples/EGM.md](./examples/EGM.md) - The `spine-dump` command-line tool. -- ci: [./ci/EGM.md](./ci/EGM.md) - `sanitize.sh` (ASan/UBSan or TSan on EL9) and `sl7-ups.sh` - (SL7 container, UPS gcc 12 and HDF5 1.12.2). +- ci: [./ci/EGM.md](./ci/EGM.md) - the GitHub Actions workflow and `github-ci.sh` (HDF5 1.14 and + 1.12 from LCG views on CVMFS), `check_tag_version.sh` (release tag against + `project(VERSION)`), `sanitize.sh` (ASan/UBSan or TSan on EL9) and `sl7-ups.sh` (SL7 container, + UPS gcc 12 and HDF5 1.12.2). - tests (not documented by contract): doctest unit tests and the test data under `tests/data/`. ## Global Invariants @@ -102,7 +107,9 @@ Build and packaging (`CMakeLists.txt`): restoring it afterwards. - Developer tools only: `tools/gen` rewrites the generated model; `tools/fixtures` clones SPINE tags and rewrites `tests/data/fixtures/` and `tests/golden_members.inc`; `ci/` scripts - create `build-asan/`, `build-tsan/` and `build-sl7/`. + create `build-ci/`, `build-asan/`, `build-tsan/` and `build-sl7/`. +- CI: the GitHub Actions workflow builds and tests on GitHub's hosted runners, with read-only + access to the repository and no secrets. ## System Verification @@ -113,5 +120,8 @@ Build and packaging (`CMakeLists.txt`): `spine_reader_real_files` (skipped unless `SPINE_READER_REAL_FILES` names a directory). - `spine_reader_tests` excludes the `real files*` test cases, which `spine_reader_real_files` runs. `spine_reader_generated_model` and `spine_reader_golden_members` are registered only - when CMake finds Python 3, and `spine_dump_output` only when the tools are built. + when CMake finds Python 3 (with `SPINE_READER_REQUIRE_PYTHON=ON`, a missing Python fails the + configuration), and `spine_dump_output` only when the tools are built. +- CI: `.github/workflows/ci.yml` runs `ci/github-ci.sh` with HDF5 1.14 (LCG_110, Ubuntu 24.04) + and HDF5 1.12 (LCG_105, Ubuntu 22.04) on every push and pull request. - Other toolchains: `ci/sl7-ups.sh`, `ci/sanitize.sh`, `ci/sanitize.sh thread`. diff --git a/README.md b/README.md index 669b781..b8b58b6 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,12 @@ The test files in `tests/data/` come with the scripts that write them: `core/mak writer) and `tools/fixtures/` (golden fixtures). `ci/sl7-ups.sh` builds with the SL7 UPS toolchain (gcc 12, HDF5 1.12.2); `ci/sanitize.sh` builds with ASan and UBSan (`ci/sanitize.sh thread`: ThreadSanitizer). +CI (`.github/workflows/ci.yml`) builds and runs ctest on every push and pull request, with HDF5 +1.14 (LCG_110, Ubuntu 24.04) and HDF5 1.12 (LCG_105, Ubuntu 22.04) from the LCG views on CVMFS. +Both jobs run `ci/github-ci.sh`, which configures with `-DSPINE_READER_REQUIRE_PYTHON=ON`: a missing +Python 3 then fails the configuration instead of silently leaving out the Python-based checks. +`ci/EGM.md` shows how to run the same jobs locally. + ## Conventions - C++17 in everything installed; no HDF5 type appears in a public header. diff --git a/ci/EGM.md b/ci/EGM.md index a9c63ee..11d9e62 100644 --- a/ci/EGM.md +++ b/ci/EGM.md @@ -1,14 +1,41 @@ # ci ## Overview -Two bash scripts that build and test spine-reader in configurations that the -default build does not cover. `sanitize.sh` builds with sanitizers on EL9 using -the CVMFS LCG_110 view. `sl7-ups.sh` builds inside an SL7 container with the -UPS toolchain that the experiments' LArSoft stacks use. Both scripts find the -repository from their own location, so they can be run from any working +Bash scripts that build and test spine-reader in configurations the default build does not +cover, plus the pieces of the GitHub Actions workflow (`.github/workflows/ci.yml`). +- `github-ci.sh` is what each CI job runs, and what the local dry run runs. +- `check_tag_version.sh` checks a release tag against `project(VERSION)`. +- `sanitize.sh` builds with sanitizers on EL9, using the CVMFS LCG_110 view. +- `sl7-ups.sh` builds inside an SL7 container with the UPS toolchain that the experiments' + LArSoft stacks use. + +Every script finds the repository from its own location, so it can be run from any working directory. ## Interface +- `github-ci.sh [build directory]` + - Sources the given LCG view, then configures `/` (default `build-ci`) + with `-DSPINE_READER_REQUIRE_PYTHON=ON`, builds with `-j$(nproc)`, and runs + `ctest --output-on-failure`. No build type is given, so the project default + (`RelWithDebInfo`) applies, as in CI. + - Prints the OS, the compiler (`$CXX`, set by the LCG view), CMake, Python and the view. + - Exit codes: 2 on bad arguments; otherwise the first failing command's code. +- `check_tag_version.sh vX.Y.Z` + - Reads `project(spine-reader VERSION X.Y.Z ...)` from `CMakeLists.txt`. + - Exits 0 when the argument equals `vX.Y.Z`, 1 when it does not or when no version line is + found, and 2 on bad arguments. Messages name both versions. + - Run it before pushing a release tag. CI runs it again on every tag push. +- `.github/workflows/ci.yml` (in the repository's `.github/` folder) + - Runs on every push (branches and tags), every pull request and manual dispatch, with + `permissions: contents: read` and no secrets. + - Job `hdf5-1_14`: `ubuntu-24.04`, LCG_110 `x86_64-ubuntu2404-gcc13-opt` (HDF5 1.14.6, gcc 13 + from the runner, CMake 3.30, Python 3.13). On a tag push it also runs + `check_tag_version.sh "$GITHUB_REF_NAME"`. + - Job `hdf5-1_12`: `ubuntu-22.04`, LCG_105 `x86_64-ubuntu2204-gcc11-opt` (HDF5 1.12.2, gcc 11 + from the runner, CMake 3.26, Python 3.9). + - Each job checks out the repository, mounts `sft.cern.ch` with + `cvmfs-contrib/github-action-cvmfs`, and runs `github-ci.sh` with its view. The two actions + are pinned by commit SHA, with the version in a comment. - `sanitize.sh [mode]` - With no argument (`mode=address`, the default) it configures with `-DSPINE_READER_SANITIZE=ON`, which enables ASan and UBSan with @@ -31,9 +58,13 @@ directory. builds with `-j8`, and runs `build-sl7/tests/spine_reader_tests` directly. ## Invariants -- `sanitize.sh` runs `set -e` before it sources the LCG view, so a failed setup - stops the script. `sl7-ups.sh` runs `set -e` only after the UPS `setup` calls, - because those return non-zero even when they succeed. Keep that order. +- The LCG views for Ubuntu use the system compiler, which GitHub's runners provide + (`ubuntu-24.04`: gcc 13; `ubuntu-22.04`: gcc 11). +- `github-ci.sh` sources the view before it enables `set -euo pipefail`, because LCG setup + scripts are not `-e`/`-u` clean. +- `sanitize.sh` runs `set -e` before it sources the LCG view, so a failed setup stops the script. + `sl7-ups.sh` runs `set -e` only after the UPS `setup` calls, because those return non-zero + even when they succeed. Keep that order. - `sl7-ups.sh` runs only the `spine_reader_tests` binary. It does not run `ctest`, so ctest-only entries such as `spine_dump_output` and the Python-based generated-model checks are not run in the SL7 build. @@ -43,14 +74,29 @@ directory. - The toolchain versions in `sl7-ups.sh` (gcc 12.1 / e26, HDF5 1.12.2) are the targets the UPS stacks build against. The top-level `CMakeLists.txt` requires `HDF5 1.12` or newer. +- A release tag is never moved or deleted. If the CI tag check fails after a push, the fix is + a new patch release. ## Side Effects -- Create and overwrite the build directories `build-asan/`, `build-tsan/` and - `build-sl7/` in the repository root. -- Read from CVMFS: the LCG view, `setup_larsoft.sh` and UPS products, and the - SL7 container image. `sl7-ups.sh` needs `apptainer` on the host. +- Create and overwrite the build directories `build-ci/` (or the one given to `github-ci.sh`), + `build-asan/`, `build-tsan/` and `build-sl7/` in the repository root. `build*/` is ignored by + git. +- Read from CVMFS: the LCG views, `setup_larsoft.sh` and UPS products, and the container + images. `sl7-ups.sh` and the dry run below need `apptainer` on the host. +- The workflow runs on GitHub's hosted runners, with read-only access to the repository. ## Verification - Run: `ci/sanitize.sh` (ASan and UBSan), `ci/sanitize.sh thread` (TSan), and `ci/sl7-ups.sh` (SL7 UPS toolchain). +- Dry run of the two CI jobs, in CERN's SFT build images on CVMFS (GitHub's runner images + differ, so the first CI run is the final check): + + ```bash + img=/cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/sft/docker + apptainer exec -B /cvmfs -B /tmp $img/ubuntu24:latest bash -e ci/github-ci.sh \ + /cvmfs/sft.cern.ch/lcg/views/LCG_110/x86_64-ubuntu2404-gcc13-opt/setup.sh build-ci-1.14 + apptainer exec -B /cvmfs -B /tmp $img/ubuntu22:latest bash -e ci/github-ci.sh \ + /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-ubuntu2204-gcc11-opt/setup.sh build-ci-1.12 + ``` +- Tag check: `ci/check_tag_version.sh v` exits 0; any other tag exits 1. - The default build these scripts vary: `source /cvmfs/sft.cern.ch/lcg/views/LCG_110/x86_64-el9-gcc13-opt/setup.sh && cmake -S . -B build && cmake --build build -j8 && ctest --test-dir build --output-on-failure` diff --git a/ci/check_tag_version.sh b/ci/check_tag_version.sh new file mode 100755 index 0000000..0ec9134 --- /dev/null +++ b/ci/check_tag_version.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Refuse a release tag that differs from project(VERSION) in CMakeLists.txt. +# Run it before pushing a tag; CI runs it again on every tag push, as a backstop. +# +# ci/check_tag_version.sh vX.Y.Z +# +# Exit codes: 0 the tag matches; 1 it does not (or no version was found); 2 usage. +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 vX.Y.Z" >&2 + exit 2 +fi +tag=$1 +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) + +version=$(sed -n -E 's/^project\(spine-reader VERSION ([0-9]+\.[0-9]+\.[0-9]+)[ )].*/\1/p' \ + "$repo/CMakeLists.txt") +if [[ -z "$version" ]]; then + echo "check_tag_version: no 'project(spine-reader VERSION X.Y.Z' line in $repo/CMakeLists.txt" >&2 + exit 1 +fi +if [[ "$tag" != "v$version" ]]; then + echo "check_tag_version: tag $tag does not match project(VERSION) $version (expected v$version)" >&2 + exit 1 +fi +echo "check_tag_version: tag $tag matches project(VERSION) $version" diff --git a/ci/github-ci.sh b/ci/github-ci.sh new file mode 100755 index 0000000..6b9b56d --- /dev/null +++ b/ci/github-ci.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Build and test spine-reader the way the GitHub workflow (.github/workflows/ci.yml) does: +# source an LCG view, configure with SPINE_READER_REQUIRE_PYTHON=ON, build, run ctest. +# The same script is used for the local dry run (see ci/EGM.md). +# +# ci/github-ci.sh [build directory, default build-ci] +set -uo pipefail + +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "usage: $0 [build directory]" >&2 + exit 2 +fi +view=$1 +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +build=$repo/${2:-build-ci} + +# LCG setup scripts are not -e / -u clean: source the view before strict mode. +set +eu +# shellcheck disable=SC1090 +source "$view" +set -euo pipefail + +echo "== $(. /etc/os-release && echo "$PRETTY_NAME") | $("${CXX:-g++}" --version | head -1)" +echo "== $(cmake --version | head -1) | $(python3 --version) | view $view" + +cmake -S "$repo" -B "$build" -DSPINE_READER_REQUIRE_PYTHON=ON +cmake --build "$build" -j"$(nproc)" +ctest --test-dir "$build" --output-on-failure diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8711e4e..169dee2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -30,7 +30,16 @@ endif() # The generated model (include/spine/model, src/model) must be what tools/gen produces from the # committed schema, which was dumped from SPINE: this is what ties every default value, field # type and assign pointer of the model to SPINE. It needs only a Python 3 interpreter. -find_package(Python3 COMPONENTS Interpreter QUIET) +# SPINE_READER_REQUIRE_PYTHON (CI turns it on) makes a missing interpreter a configuration +# error instead of silently leaving these checks out. +option(SPINE_READER_REQUIRE_PYTHON + "Fail the configuration if Python 3 is missing, instead of skipping the Python-based checks" + OFF) +if(SPINE_READER_REQUIRE_PYTHON) + find_package(Python3 REQUIRED COMPONENTS Interpreter) +else() + find_package(Python3 COMPONENTS Interpreter QUIET) +endif() if(Python3_Interpreter_FOUND) add_test(NAME spine_reader_generated_model COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/gen/generate.py --check