diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 655c506..b7787ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,9 @@ permissions: {} concurrency: group: release-${{ github.event_name }}-${{ github.ref_name || inputs.tag }} cancel-in-progress: false + # Keep every pending run. A newer push would otherwise cancel the pending run of a checksum merge, + # and only that run publishes. + queue: max env: CARGO_TERM_COLOR: always @@ -295,6 +298,36 @@ jobs: exit 1 fi + # The BEAM binds every symbol of a NIF when it loads it, and only the enif_* functions come from + # the BEAM. Any other symbol that no library of the NIF defines fails the load. ldd -r binds the + # same way, and nm lists the symbols that macOS leaves to the load. + - name: Check that the NIF loads + shell: bash + env: + NIF: native/whisper_cpp_native/target/${{ matrix.target }}/release/libwhisper_cpp_native.${{ matrix.src_ext || matrix.lib_ext }} + INSTALL_CUDA: ${{ matrix.install_cuda }} + INSTALL_ROCM: ${{ matrix.install_rocm }} + run: | + if [ "$INSTALL_CUDA" = true ]; then + # The runner has no NVIDIA driver. The toolkit's stub of its library stands in for it. + ln -s /usr/local/cuda/lib64/stubs/libcuda.so "$RUNNER_TEMP/libcuda.so.1" + export LD_LIBRARY_PATH="$RUNNER_TEMP" + elif [ "$INSTALL_ROCM" = true ]; then + export LD_LIBRARY_PATH="$ROCM_PATH/lib" + fi + if [ "$RUNNER_OS" = macOS ]; then + REPORT="$(nm -m -u "$NIF")" + MISSING="$(grep 'dynamically looked up' <<< "$REPORT" | grep -v ' _enif_' || true)" + else + REPORT="$(ldd -r "$NIF" 2>&1)" || { echo "$REPORT"; exit 1; } + MISSING="$(grep -E 'not found|undefined symbol' <<< "$REPORT" | grep -v 'undefined symbol: enif_' || true)" + fi + if [ -n "$MISSING" ]; then + echo "$MISSING" + echo "::error::The NIF does not load: it needs the libraries or symbols above" + exit 1 + fi + - name: Create tarball shell: bash env: @@ -325,7 +358,9 @@ jobs: tag: name: Create release tag runs-on: ubuntu-24.04 - needs: detect + # Tag only after every NIF build passes. A failed build then leaves no tag, so the next push to + # main releases the same version. + needs: [detect, build] if: github.event_name == 'push' && needs.detect.outputs.should_release == 'true' permissions: contents: write # create and push the release tag @@ -401,11 +436,17 @@ jobs: group: ${{ github.workflow }}-${{ github.ref }}-publish cancel-in-progress: false steps: + # The package holds the sources of the tag, which the NIF archives were built from, and the + # checksum file of the merge that publishes. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ needs.detect.outputs.sha }} + ref: refs/tags/${{ needs.detect.outputs.tag }} + fetch-depth: 0 persist-credentials: false + - name: Take the checksum file of the merge + run: git checkout "$GITHUB_SHA" -- checksum-Elixir.WhisperCpp.Native.exs + # Hex versions are immutable, so a package with checksums of other archives stays broken. - name: Compare the checksum file with the release env: @@ -424,12 +465,19 @@ jobs: otp-version: "29.1.2" - name: Install dependencies - run: | - mix local.hex --force - mix local.rebar --force - mix deps.get + run: mix deps.get + + # The compile downloads the precompiled NIF of this runner and checks it against the checksum + # file, as a consumer's build does. + - name: Load the precompiled NIF + run: mix run --no-start -e 'Code.ensure_loaded!(WhisperCpp.Native)' - # With no Hex credential in the job, Hex publishes through the workload identity. The docs - # compile downloads the precompiled NIF of this runner and checks it against the checksum file. + # With no Hex credential in the job, Hex publishes through the workload identity. A failed upload + # of the docs still exits 0, so the step reads the log. - name: Publish the package and its docs - run: mix hex.publish --yes + shell: bash + run: | + mix hex.publish --yes 2>&1 | tee "$RUNNER_TEMP/publish.log" + if grep -E 'Publishing (docs )?failed' "$RUNNER_TEMP/publish.log"; then + exit 1 + fi diff --git a/AGENTS.md b/AGENTS.md index 235b4bc..9eacf39 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,26 +49,31 @@ users never build Rust. 1. Bump `@version` in `mix.exs`, add the `CHANGELOG.md` entry, and push to `main`. 2. On each push to `main`, `release.yml` releases when no tag exists for the `mix.exs` version. It builds a tarball per - target and variant, creates the tag, and uploads the tarballs and `SHA256SUMS`. If a run stops before it creates the - tag, the next push retries the release. + target and variant, creates the tag, and uploads the tarballs and `SHA256SUMS`. A build fails when its NIF needs a + symbol that no library defines, because the BEAM could not load it. If a run stops before it creates the tag, the + next push retries the release. 3. After the tag exists, a manual dispatch builds the tag's commit. It fails when the tag is missing or its `mix.exs` version differs, and it uploads only the assets the release does not have yet. It never replaces a published tarball, because a new tarball breaks the checksum file in the Hex package. -4. Regenerate `checksum-Elixir.WhisperCpp.Native.exs` from the published assets, and open a PR with it. This keeps - the checksum of each tag reproducible from the repo: - - ```bash - mix rustler_precompiled.download WhisperCpp.Native --all --no-config --ignore-unavailable --print - ``` - +4. Regenerate `checksum-Elixir.WhisperCpp.Native.exs` from the published assets with `task checksum:download`, and + open a PR with it. This keeps the checksum of each tag reproducible from the repo. The task builds the NIF from + source first, because `mix rustler_precompiled.download` reads the version and the targets from the metadata of the + last compile. The task keeps that metadata in `_build`, where a compile in another checkout cannot change it. It + fails when an asset is missing. 5. Merge the checksum PR. A push to `main` that changes the checksum file, for a version that has its tag, starts the - `Publish to Hex` job of `release.yml`. It compares the file with the release's `SHA256SUMS`, stops when they differ, - and runs `mix hex.publish --yes` in the GitHub environment `hex`, which only `main` can use. The job keeps no Hex - key: the package has a workload identity on hex.pm for `rubas/whisper_cpp`, workflow `release.yml`, environment - `hex`, and Hex trades the job's OIDC token for a short token that can only publish `whisper_cpp`. + `Publish to Hex` job of `release.yml`. It publishes the sources of the tag, which the NIFs were built from, with the + checksum file of the merge. It compares the file with the release's `SHA256SUMS`, stops when they differ, loads the + precompiled NIF of its runner, and runs `mix hex.publish --yes` in the GitHub environment `hex`, which only `main` + can use. The job keeps no Hex key: the package has a workload identity on hex.pm for `rubas/whisper_cpp`, workflow + `release.yml`, environment `hex`, and Hex trades the job's OIDC token for a short token that can only publish + `whisper_cpp`. 6. The package has staged publishing on. The job ends with `Package staged for approval at `, and an owner - approves the release on hex.pm with their 2FA within 7 days. A failed docs upload prints `Publishing failed` but - leaves the job green, so read the end of its log. When the job fails, fix the cause and re-run it. + approves the release on hex.pm with their 2FA within 7 days. The job fails when the log says `Publishing failed` or + `Publishing docs failed`, because `mix hex.publish` exits 0 when only the docs upload fails. +7. When the job fails for a cause outside the repo, such as hex.pm or the workload identity, fix it and re-run the job. + It publishes the same tag. A cause in the tag needs a new patch version. A re-run keeps the workflow file of its run, + so after a fix of the job itself, release a new patch version, or a package owner runs `mix hex.publish` from the + tag with the checksum file of the merge. -Merge nothing else to `main` between the version bump and the checksum PR, or the package holds sources that the NIFs -were not built from. Hex versions are immutable, so change the checksum file only in the checksum PR of a release. +A change that merges between the version bump and the checksum PR ships only with the next version. Hex versions are +immutable, so change the checksum file only in the checksum PR of a release. diff --git a/Taskfile.yml b/Taskfile.yml index 18266fe..bb926b9 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -80,6 +80,18 @@ tasks: cmds: - mix hex.build + checksum:download: + desc: Write the checksum file from the NIF artefacts of the GitHub release of @version + # Without a source build, the compile fetches the NIF of this host, which the checksum file + # does not list yet. The download reads the version and the targets from the metadata of the + # last compile. A cache in this checkout keeps a compile in another checkout from changing it, + # and --force writes it. + env: + WHISPER_CPP_BUILD: "1" + RUSTLER_PRECOMPILED_GLOBAL_CACHE_PATH: "{{.ROOT_DIR}}/_build/rustler_precompiled" + cmds: + - mix do compile --force + rustler_precompiled.download WhisperCpp.Native --all --print + build:cpu: desc: Source-build NIF with pure CPU backend cmds: