From a87c13b22cc757f876fd6d350acd1cc24cee6ed6 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 21 Sep 2026 04:11:24 +0000 Subject: [PATCH 1/3] eckitlib: Add version 2.1.1.26 Builds the compiled eckit C++ toolkit libraries from ecmwf/eckit at tag 2.1.1 and packages them as the eckitlib binary wrapper wheel, replaying upstream's own python/eckitlib/buildconfig CMAKE_PARAMS and post-build.sh (auditwheel repair). Upstream's cp310..cp314 wheels carry byte-identical payloads since nothing links the Python C API, so one py3-none wheel is built instead of a per-interpreter matrix. The wheel bundles libcurl and its dependency closure from the image, so the licence of every RPM auditwheel grafted in is collected from the SBOM and shipped in the dist-info. --- .github/workflows/build-eckitlib.yml | 262 +++++++++++++++++++++++++++ docs/packages/eckitlib.yaml | 5 + 2 files changed, 267 insertions(+) create mode 100644 .github/workflows/build-eckitlib.yml create mode 100644 docs/packages/eckitlib.yaml diff --git a/.github/workflows/build-eckitlib.yml b/.github/workflows/build-eckitlib.yml new file mode 100644 index 0000000000..2af0f52630 --- /dev/null +++ b/.github/workflows/build-eckitlib.yml @@ -0,0 +1,262 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# Upstream builds eckitlib via a private tool (ecmwf/reusable-workflows' +# python-wrapper-wheel.yml + the "wheelmaker" image), not a public workflow, so this +# replays ecmwf/eckit's own python/eckitlib/buildconfig CMAKE_PARAMS and +# post-build.sh directly. The built libraries use no Python C API, so this builds +# one py3-none wheel instead of upstream's byte-identical cp310..cp314 matrix. +# ENABLE_PYTHON is off: the Cython extension it adds is installed into the separate +# `eckit` distribution's source tree, never into this wheel. +name: Build eckitlib wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/eckitlib.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-eckitlib.yml' + - 'docs/packages/eckitlib.yaml' + push: + branches: [main] + paths: + - '.github/workflows/build-eckitlib.yml' + - 'docs/packages/eckitlib.yaml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + ECBUILD_VERSION: '3.14.2' + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: eckitlib + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Build eckitlib ${{ matrix.version }} py3-none-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 720 + env: + PACKAGE_VERSION: ${{ matrix.version }} + + steps: + - name: Derive the eckit git tag from the package version + run: echo "ECKIT_VERSION=$(echo "$PACKAGE_VERSION" | sed -E 's/\.[0-9]+$//')" >> "$GITHUB_ENV" + + - name: Checkout eckit ${{ env.ECKIT_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ecmwf/eckit + ref: ${{ env.ECKIT_VERSION }} + persist-credentials: false + + - name: Record the resolved commit + run: echo "ECKIT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + + - name: Write the wheel packaging scripts + run: | + cat > bundled_rpms.py <<'PY' + import glob + import json + import sys + + sbom = json.load(open(glob.glob(f"{sys.argv[1]}/*.dist-info/sboms/auditwheel.cdx.json")[0])) + names = {c["name"] for c in sbom["components"] if c["purl"].startswith("pkg:rpm/")} + print("\n".join(sorted(names))) + PY + + cat > pack_wheel.py <<'PY' + import hashlib + import os + import sys + import zipfile + from base64 import urlsafe_b64encode + + STAGE, OUT_DIR = sys.argv[1:3] + + dist_info = next(n for n in sorted(os.listdir(STAGE)) if n.endswith(".dist-info")) + name, version = dist_info[: -len(".dist-info")].split("-") + tags = [ + line.split(":", 1)[1].strip() + for line in open(os.path.join(STAGE, dist_info, "WHEEL")) + if line.startswith("Tag:") + ] + interpreter, abi = tags[0].split("-")[:2] + tag = f"{interpreter}-{abi}-{'.'.join(t.split('-', 2)[2] for t in tags)}" + + os.makedirs(OUT_DIR, exist_ok=True) + wheel_path = os.path.join(OUT_DIR, f"{name}-{version}-{tag}.whl") + record_name = f"{dist_info}/RECORD" + records = [] + + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf: + for root, _dirs, files in os.walk(STAGE): + for filename in sorted(files): + path = os.path.join(root, filename) + arcname = os.path.relpath(path, STAGE) + if arcname == record_name: + continue + with open(path, "rb") as f: + data = f.read() + info = zipfile.ZipInfo(arcname) + info.external_attr = (os.stat(path).st_mode & 0xFFFF) << 16 + zf.writestr(info, data, zipfile.ZIP_DEFLATED) + digest = urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + records.append(f"{arcname},sha256={digest},{len(data)}") + + zf.writestr(record_name, "\n".join(records + [f"{record_name},,"]) + "\n") + + print(wheel_path) + PY + + - name: Build eckit, then package and repair the wheel + run: | + docker run --rm \ + -v "$(pwd)":/workspace \ + --workdir /workspace \ + -e PACKAGE_VERSION="$PACKAGE_VERSION" \ + -e ECKIT_COMMIT="$ECKIT_COMMIT" \ + -e ECBUILD_VERSION="$ECBUILD_VERSION" \ + "$MANYLINUX_RISCV64_IMAGE" \ + bash -c ' + set -euo pipefail + dnf -y -q install libcurl-devel lz4-devel flex + DIST_INFO=/tmp/stage/eckitlib-$PACKAGE_VERSION.dist-info + mkdir -p /tmp/deps /tmp/build $DIST_INFO/licenses + + curl -fsSL "https://github.com/ecmwf/ecbuild/archive/refs/tags/$ECBUILD_VERSION.tar.gz" | tar xz -C /tmp/build + cmake -S "/tmp/build/ecbuild-$ECBUILD_VERSION" -B /tmp/build/ecbuild-build -D CMAKE_INSTALL_PREFIX=/tmp/deps + cmake --build /tmp/build/ecbuild-build -j "$(nproc)" + cmake --install /tmp/build/ecbuild-build + + cmake -S . -B /tmp/build/eckit-build \ + -D CMAKE_BUILD_TYPE=MinSizeRel \ + -D CMAKE_PREFIX_PATH=/tmp/deps \ + -D CMAKE_INSTALL_PREFIX=/tmp/stage/eckitlib \ + -D CMAKE_INSTALL_LIBDIR=lib64 \ + -D ENABLE_MPI=0 \ + -D ENABLE_ECKIT_GEO=1 \ + -D ENABLE_BUILD_TOOLS=OFF \ + -D ENABLE_AEC=0 \ + -D ENABLE_EIGEN=0 \ + -D ENABLE_LZ4=1 \ + -D ENABLE_PYTHON=0 + cmake --build /tmp/build/eckit-build -j "$(nproc)" + cmake --install /tmp/build/eckit-build + + printf "__version__ = \"%s\"\n__commit_hash__ = \"%s\"\nfindlibs_dependencies = []\n" \ + "$PACKAGE_VERSION" "$ECKIT_COMMIT" > /tmp/stage/eckitlib/__init__.py + + cp LICENSE AUTHORS $DIST_INFO/licenses/ + printf "eckitlib\n" > $DIST_INFO/top_level.txt + printf "Wheel-Version: 1.0\nGenerator: python-wheels-riscv64-port\nRoot-Is-Purelib: false\nTag: py3-none-linux_riscv64\n" > $DIST_INFO/WHEEL + printf "Metadata-Version: 2.4\nName: eckitlib\nVersion: %s\nSummary: Compiled eckit C++ toolkit libraries (ECMWF), no Python bindings\nHome-page: https://github.com/ecmwf/eckit\nLicense-Expression: Apache-2.0\n" \ + "$PACKAGE_VERSION" > $DIST_INFO/METADATA + + python3 pack_wheel.py /tmp/stage /tmp/unrepaired + auditwheel repair --plat manylinux_2_39_riscv64 -w /tmp/repaired /tmp/unrepaired/*.whl + + mkdir -p /tmp/final + cd /tmp/final && unzip -q /tmp/repaired/*.whl && cd /workspace + LICENSES=/tmp/final/eckitlib-$PACKAGE_VERSION.dist-info/licenses + for pkg in $(python3 bundled_rpms.py /tmp/final); do + mkdir -p "$LICENSES/$pkg" + for f in $(rpm -ql "$pkg" | grep "^/usr/share/licenses/"); do + if [ -f "$f" ]; then cp "$f" "$LICENSES/$pkg/"; fi + done + test -n "$(ls -A "$LICENSES/$pkg")" + done + + python3 pack_wheel.py /tmp/final wheelhouse + ' + + - name: Check the wheel ships the libraries and every bundled licence + run: | + python3 - wheelhouse/*.whl <<'EOF' + import json, sys, zipfile + + zf = zipfile.ZipFile(sys.argv[1]) + names = zf.namelist() + for lib in ("libeckit.so", "libeckit_geo.so", "libeckit_sql.so", "libeckit_codec.so"): + assert any(n.endswith(f"lib64/{lib}") for n in names), (lib, names) + for licence in ("LICENSE", "AUTHORS"): + assert any(n.endswith(f".dist-info/licenses/{licence}") for n in names), licence + + sbom = json.loads(zf.read(next(n for n in names if n.endswith("auditwheel.cdx.json")))) + for component in sbom["components"]: + if component["purl"].startswith("pkg:rpm/"): + prefix = f".dist-info/licenses/{component['name']}/" + assert any(prefix in n for n in names), component["name"] + EOF + + - name: Smoke-test the built libraries + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends python3-venv + python3 -m venv .venv + . .venv/bin/activate + # gotcha 234: stock Ubuntu 24.04 apt pip (24.0) has zero manylinux_*_riscv64 + # tags, so it rejects the riscv64 wheel outright without upgrading first. + pip install -q --upgrade pip + pip install -q findlibs wheelhouse/*.whl + python3 -c " + import ctypes, glob, os, subprocess + import eckitlib, findlibs + + print(findlibs.load('eckit')) + root = os.path.dirname(eckitlib.__file__) + for lib in sorted(glob.glob(os.path.join(root, 'lib64', '*.so'))): + ctypes.CDLL(lib) + print('loaded', os.path.basename(lib)) + tool = subprocess.run( + [os.path.join(root, 'bin', 'eckit-version'), '-long', '-build'], + check=True, capture_output=True, text=True, + ) + report = tool.stdout + tool.stderr + print(report) + assert eckitlib.__version__.rsplit('.', 1)[0] in report + assert 'riscv64' in report + " + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eckitlib-${{ matrix.version }}-py3-none-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish eckitlib ${{ matrix.version }} + needs: [setup, build_wheels] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: eckitlib-${{ matrix.version }}-*-manylinux_riscv64 diff --git a/docs/packages/eckitlib.yaml b/docs/packages/eckitlib.yaml new file mode 100644 index 0000000000..bf25596f5d --- /dev/null +++ b/docs/packages/eckitlib.yaml @@ -0,0 +1,5 @@ +package-name: eckitlib +source-code: https://github.com/ecmwf/eckit +license: Apache-2.0 +versions: +- version: 2.1.1.26 From 1429ec6775bf565ef8cbd59a111d3212577409c7 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 21 Sep 2026 04:36:21 +0000 Subject: [PATCH 2/3] eckitlib: Collect the bundled libraries' licences and GPL sources auditwheel grafts libcurl's whole dependency closure into the wheel, so every RPM its CycloneDX SBOM names now contributes its %license files to the dist-info, and the copyleft ones (keyutils, libssh, libidn2, libunistring, libxcrypt, systemd, libcap, pcre2) get a gpl_sources job beside the build. Two distro packages need more than their own %license: pcre2 keeps its text in the pcre2-syntax subpackage, and lz4-libs ships none at all, so the collector falls back to the other packages built from the same source RPM and then to installing that source package. Rocky's lz4 licence directory holds only the GPLv2 text covering its CLI tools, so liblz4's BSD-2-Clause text comes from the lz4 repository, as upstream's own pre-compile.sh does. --- .github/workflows/build-eckitlib.yml | 101 +++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-eckitlib.yml b/.github/workflows/build-eckitlib.yml index 2af0f52630..60158e3878 100644 --- a/.github/workflows/build-eckitlib.yml +++ b/.github/workflows/build-eckitlib.yml @@ -75,14 +75,54 @@ jobs: - name: Write the wheel packaging scripts run: | - cat > bundled_rpms.py <<'PY' + cat > collect_licences.py <<'PY' import glob import json + import os + import re + import shutil + import subprocess import sys - sbom = json.load(open(glob.glob(f"{sys.argv[1]}/*.dist-info/sboms/auditwheel.cdx.json")[0])) - names = {c["name"] for c in sbom["components"] if c["purl"].startswith("pkg:rpm/")} - print("\n".join(sorted(names))) + WHEEL_DIR = sys.argv[1] + dist_info = glob.glob(f"{WHEEL_DIR}/*.dist-info")[0] + sbom = json.load(open(f"{dist_info}/sboms/auditwheel.cdx.json")) + + + def rpm(*args): + return subprocess.run(["rpm", *args], capture_output=True, text=True).stdout + + + def licences_of(package): + return [ + path + for path in rpm("-ql", package).splitlines() + if path.startswith("/usr/share/licenses/") and os.path.isfile(path) + ] + + + by_source = [line.split() for line in rpm("-qa", "--qf", "%{SOURCERPM} %{NAME}\n").splitlines()] + + for name in sorted({c["name"] for c in sbom["components"] if c["purl"].startswith("pkg:rpm/")}): + source = rpm("-q", "--qf", "%{SOURCERPM}", name) + files = [ + path + for sibling_source, sibling in by_source + if sibling_source == source + for path in licences_of(sibling) + ] + if not files: + # lz4-libs is a runtime subpackage with no %license file and no installed + # sibling that has one, so its source package has to be pulled in. + base = re.sub(r"-[^-]+-[^-]+\.src\.rpm$", "", source) + subprocess.run(["dnf", "-y", "-q", "install", base], check=True) + files = licences_of(base) + if not files: + raise SystemExit(f"no licence file found for {name}") + dest = f"{dist_info}/licenses/{name}" + os.makedirs(dest, exist_ok=True) + for path in files: + shutil.copy(path, dest) PY cat > pack_wheel.py <<'PY' @@ -178,14 +218,14 @@ jobs: mkdir -p /tmp/final cd /tmp/final && unzip -q /tmp/repaired/*.whl && cd /workspace - LICENSES=/tmp/final/eckitlib-$PACKAGE_VERSION.dist-info/licenses - for pkg in $(python3 bundled_rpms.py /tmp/final); do - mkdir -p "$LICENSES/$pkg" - for f in $(rpm -ql "$pkg" | grep "^/usr/share/licenses/"); do - if [ -f "$f" ]; then cp "$f" "$LICENSES/$pkg/"; fi - done - test -n "$(ls -A "$LICENSES/$pkg")" - done + python3 collect_licences.py /tmp/final + # Rocky ships only the GPLv2 text covering the lz4 CLI tools, while the + # bundled liblz4 is BSD-2-Clause and no RPM carries that text; upstream + # pre-compile.sh fetches it from the lz4 repository for the same reason. + LZ4_LICENSES=/tmp/final/eckitlib-$PACKAGE_VERSION.dist-info/licenses/lz4-libs + LZ4_VERSION=$(rpm -q --qf "%{VERSION}" lz4-libs) + curl -fsSL "https://raw.githubusercontent.com/lz4/lz4/v$LZ4_VERSION/LICENSE" -o "$LZ4_LICENSES/LICENSE" + curl -fsSL "https://raw.githubusercontent.com/lz4/lz4/v$LZ4_VERSION/lib/LICENSE" -o "$LZ4_LICENSES/lib-LICENSE" python3 pack_wheel.py /tmp/final wheelhouse ' @@ -244,9 +284,42 @@ jobs: path: wheelhouse/*.whl if-no-files-found: error + gpl_sources: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Collect GPL sources for eckitlib ${{ matrix.version }} + runs-on: ubuntu-24.04-riscv + + env: + PACKAGE_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The copyleft (GPL/LGPL) libraries auditwheel vendors out of the build image + # along libcurl's dependency closure. + - uses: ./actions/collect-gpl-sources + with: + image: ${{ env.MANYLINUX_RISCV64_IMAGE }} + packages: gcc keyutils-libs libssh libidn2 libunistring libxcrypt systemd-libs libcap pcre2 + output: gpl-sources.tar + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eckitlib-${{ env.PACKAGE_VERSION }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + publish: name: Publish eckitlib ${{ matrix.version }} - needs: [setup, build_wheels] + needs: [setup, build_wheels, gpl_sources] if: needs.setup.outputs.versions != '[]' strategy: fail-fast: false @@ -260,3 +333,5 @@ jobs: app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} with: artifact-pattern: eckitlib-${{ matrix.version }}-*-manylinux_riscv64 + gpl-sources-artifact: eckitlib-${{ matrix.version }}-gpl-sources + gpl-sources-description: gcc and the copyleft libraries bundled in the wheel From b5133a7f104a4c816955407f154077eb93e6ea90 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Mon, 21 Sep 2026 05:19:02 +0000 Subject: [PATCH 3/3] eckitlib: Exercise the libraries through their C API, not the tool The eckit-version binary the wheel ships segfaults before printing anything, on upstream's own released x86_64 wheel as much as on this build, so it proves nothing about the libraries. Call libeckit's eckit_version_str/eckit_git_sha1 entry points through ctypes instead and check both against the metadata the wheel declares. --- .github/workflows/build-eckitlib.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-eckitlib.yml b/.github/workflows/build-eckitlib.yml index 60158e3878..02b762726a 100644 --- a/.github/workflows/build-eckitlib.yml +++ b/.github/workflows/build-eckitlib.yml @@ -260,22 +260,20 @@ jobs: pip install -q --upgrade pip pip install -q findlibs wheelhouse/*.whl python3 -c " - import ctypes, glob, os, subprocess + import ctypes, glob, os import eckitlib, findlibs - print(findlibs.load('eckit')) - root = os.path.dirname(eckitlib.__file__) - for lib in sorted(glob.glob(os.path.join(root, 'lib64', '*.so'))): + for lib in sorted(glob.glob(os.path.join(os.path.dirname(eckitlib.__file__), 'lib64', '*.so'))): ctypes.CDLL(lib) print('loaded', os.path.basename(lib)) - tool = subprocess.run( - [os.path.join(root, 'bin', 'eckit-version'), '-long', '-build'], - check=True, capture_output=True, text=True, - ) - report = tool.stdout + tool.stderr - print(report) - assert eckitlib.__version__.rsplit('.', 1)[0] in report - assert 'riscv64' in report + + eckit = findlibs.load('eckit') + for name in ('eckit_version', 'eckit_version_str', 'eckit_git_sha1'): + getattr(eckit, name).restype = ctypes.c_char_p + eckit.eckit_version_int.restype = ctypes.c_uint + print(eckit.eckit_version().decode(), eckit.eckit_version_int(), eckit.eckit_git_sha1().decode()) + assert eckit.eckit_version_str().decode() == eckitlib.__version__.rsplit('.', 1)[0] + assert eckit.eckit_git_sha1().decode() == eckitlib.__commit_hash__ " - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1