diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..6771674 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,122 @@ +name: release + +# Publishes benchx to PyPI and creates the GitHub release, when the tag v is pushed. +# docs/releasing.md is the checklist. A tag such as v0.1.0rc1 makes a pre-release. +# +# The package is tested before anything is uploaded. Conda packages are not built here: they +# come from the benchx feedstock on conda-forge, which follows the releases on PyPI. +# +# Only a tag publishes. A pull request that changes the packaging or the version runs the build +# job and uploads nothing. +# +# This needs, on GitHub, the pypi environment with its required reviewers, and on PyPI a trusted +# publisher for this file and that environment. The actions are pinned to commits since they +# handle the uploads. + +on: + push: + tags: ["v*"] + pull_request: + branches: [main] + paths: + - ".github/workflows/release.yml" + - "pyproject.toml" + - "src/benchx/__init__.py" + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + prerelease: ${{ steps.version.outputs.prerelease }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + + - name: Build the sdist, and the wheel from it + run: uv build + + - name: Read the version from the sdist + id: version + run: | + version=$(basename dist/benchx-*.tar.gz .tar.gz) + version=${version#benchx-} + echo "version=$version" >> "$GITHUB_OUTPUT" + if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "prerelease=false" >> "$GITHUB_OUTPUT" + else + echo "prerelease=true" >> "$GITHUB_OUTPUT" + fi + + - name: Check that the tag names this version + if: github.ref_type == 'tag' + env: + TAG: ${{ github.ref_name }} + VERSION: ${{ steps.version.outputs.version }} + run: | + if [ "$TAG" != "v$VERSION" ]; then + echo "::error::tag $TAG does not match the package version $VERSION" + exit 1 + fi + + - name: Check the package metadata + run: uvx twine check --strict dist/* + + - name: Install the wheel into a clean environment and test it + run: | + uv venv --python 3.13 "$RUNNER_TEMP/venv" + source "$RUNNER_TEMP/venv/bin/activate" + uv pip install "$(ls dist/*.whl)[test]" + # Away from the checkout, so that nothing can import it instead of the installed package. + cd "$RUNNER_TEMP" + bx --help + python -m pytest --pyargs benchx -ra + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dist + path: dist/ + if-no-files-found: error + + publish-pypi: + needs: build + # Also what keeps the job after this one off pull requests and forks: it needs this one. + if: github.repository == 'benchx-dev/benchx' && github.ref_type == 'tag' + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/benchx/${{ needs.build.outputs.version }}/ + permissions: + id-token: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dist + path: dist + + - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + + github-release: + needs: [build, publish-pypi] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dist + path: release-files + + - name: Create the release + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + TAG: ${{ github.ref_name }} + PRERELEASE: ${{ needs.build.outputs.prerelease }} + run: gh release create "$TAG" release-files/* --verify-tag --generate-notes --prerelease="$PRERELEASE" diff --git a/docs/benchx.md b/docs/benchx.md index 9f3c56c..45c62c2 100644 --- a/docs/benchx.md +++ b/docs/benchx.md @@ -127,7 +127,7 @@ is a runnable walk-through. Environments that cannot share a process, such as two venvs with different numpy versions, are measured by separate `bx bench` calls that share a `--run-key`; `--label NAME=VALUE`, `--round N` and `--slot K` place each call in the session, and `bx compare --run KEY --profile environments ---label NAME --baseline VALUE` compares the sides ([UC-04](use-cases/UC-04-cross-env.md)). +--label NAME --baseline VALUE` compares the sides ([UC-04](https://benchx-dev.github.io/benchx/use-cases/UC-04-cross-env/)). `examples/numpy-venvs` in [benchx_examples](https://github.com/benchx-dev/benchx_examples) is the worked example, with a shell script that drives it. @@ -214,6 +214,12 @@ every result is pyperf. Results also record the installed project's version store: the ad hoc path. Condition measurements (`measurement.kind: condition`) are not compared, and deterministic quantities are deferred. +## Install + +```console +$ pip install benchx +``` + ## Run it ```console diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..93c7c98 --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,68 @@ +# Releasing benchx + +Pushing a tag `v` runs `.github/workflows/release.yml`. It builds and tests the +package, waits for approval, publishes to PyPI, and creates the GitHub release. Below, +`upstream` is your remote for `benchx-dev/benchx`, and `0.1.0` stands for the version. + +## Checklist + +1. **Check CI.** The release workflow runs its own tests, but it does not look at the other + workflows, so check that the latest commit on `main` is green. +2. **Prepare the release in a pull request, and merge it.** + - Set `__version__` in `src/benchx/__init__.py`; `pyproject.toml` reads it from there. + - Update `CHANGELOG.md`, including the list of contributors. +3. **Tag the merged commit and push the tag.** The tag must be `v` plus exactly the version in + `__version__`, or the workflow stops before uploading anything. + + ```console + $ git fetch upstream + $ git tag -a v0.1.0 -m "benchx 0.1.0" upstream/main + $ git push upstream v0.1.0 + ``` + +4. **Approve the `publish-pypi` job.** Once the tests pass, the + [workflow run](https://github.com/benchx-dev/benchx/actions/workflows/release.yml) waits and + shows a "Review deployments" button. A required reviewer of the `pypi` environment other than + whoever pushed the tag has to approve it, so a release takes two people. +5. **Check the result.** The run should be green, with the version on + [PyPI](https://pypi.org/project/benchx/) and in the + [GitHub releases](https://github.com/benchx-dev/benchx/releases). +6. **Test the install in a new environment.** Activate it, because the tests start benchmarks + with whichever `python` is on `PATH`. + + ```console + $ python -m venv venv && source venv/bin/activate + $ pip install 'benchx[test]==0.1.0' + $ bx --help + $ python -m pytest --pyargs benchx + ``` + +7. **Merge the conda-forge update.** Conda packages come from the benchx feedstock on + conda-forge, not from this repository. Once that feedstock exists, conda-forge's bot opens a + pull request on it after a new version reaches PyPI; review and merge that pull request. +8. **Start the next development version.** In a pull request, set `__version__` to the next + version with a `.dev0` suffix: after `0.1.0`, that is `0.2.0.dev0`. Results record this version, + so without this step, results from `main` would claim to come from the release. + +## Pre-releases + +A version such as `0.1.0rc1`, tagged `v0.1.0rc1`, follows steps 1 to 6. It is marked as a +pre-release on GitHub, and `pip install benchx` skips it unless the version is named or +`--pre` is given. + +## If something fails + +- **Before the upload:** nothing is published. Delete the tag, fix the problem, and tag again: + + ```console + $ git push upstream --delete v0.1.0 + $ git tag -d v0.1.0 + ``` + +- **During the upload to PyPI:** check the [PyPI project](https://pypi.org/project/benchx/) + first. If the version is not there, re-run the failed job. +- **After the upload to PyPI, in `github-release`:** the package is published, so re-run the + failed job. +- **A published version is wrong:** PyPI does not allow a version to be uploaded again, even + after it is deleted, so release a new version. Yanking the bad one on PyPI stops new installs + from picking it up by default. diff --git a/pyproject.toml b/pyproject.toml index eea6e24..0e76669 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "benchx" -version = "0.1.0" +dynamic = ["version"] description = "BenchX: a flexible and powerful benchmarking framework" license = "Apache-2.0" license-files = ["LICENSE"] @@ -33,6 +33,9 @@ bx = "benchx.cli:main" requires = ["hatchling>=1.26"] build-backend = "hatchling.build" +[tool.hatch.version] +path = "src/benchx/__init__.py" + [tool.hatch.build.targets.wheel] packages = ["src/benchx"] diff --git a/src/benchx/runner.py b/src/benchx/runner.py index 7e49039..95507d5 100644 --- a/src/benchx/runner.py +++ b/src/benchx/runner.py @@ -17,7 +17,7 @@ from .adapters.base import Unsupported, no_cases from .core import Refusal, Rejected -RUNNER = {"name": "benchx-prototype", "version": __version__} +RUNNER = {"name": "benchx", "version": __version__} # The runner's own probe: one condition measurement per attempt (schema ยง4.4). PROBE_PRODUCER = {"name": "benchx/runner-probe", "version": __version__, "mapping_version": "benchx-probe/v1"} LOAD = {"name": "load-average", "unit": "1"} diff --git a/uv.lock b/uv.lock index b2c03f4..65762fc 100644 --- a/uv.lock +++ b/uv.lock @@ -35,7 +35,6 @@ wheels = [ [[package]] name = "benchx" -version = "0.1.0" source = { editable = "." } dependencies = [ { name = "jsonpointer" }, diff --git a/zensical.toml b/zensical.toml index a3dc99f..fe06bbc 100644 --- a/zensical.toml +++ b/zensical.toml @@ -73,6 +73,7 @@ nav = [ { "One-off benchmarking" = "user_stories/one-off.md" }, ] }, { "benchx" = "benchx.md" }, + { "Releasing" = "releasing.md" }, ] [project.validation]