From 5bf7c96c5a55ed1ab2248912ee71fc4f651867fc Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Tue, 3 Nov 2026 17:08:28 +0000 Subject: [PATCH 1/7] CI: add a e2e run from benchx_examples --- .github/workflows/e2e.yml | 169 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..3ae2e6e --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,169 @@ +name: e2e examples + +# Runs the numpy-spin examples in benchx-dev/benchx_examples and prints their output: +# +# track.sh builds NumPy with spin at several revisions and benchmarks each +# num_threads.sh benchmarks one prebuilt revision at several OPENBLAS_NUM_THREADS +# +# Nothing is checked. The job fails only if a step fails (a clone, a build, a script that +# crashes); the scripts exit 0 even when a revision fails, so read the printed output. +# +# Runs on every pull request, and on demand. A run on demand can take another +# benchx_examples branch, tag or commit (examples_ref), which a pull request here cannot: +# the examples live in another repository. +# +# The benchmarks are pinned to two cores (BENCH_CPUS). num_threads.sh should use 1, 2 and 4 +# threads, so that the last of them oversubscribes the two cores. The NumPy builds are not +# pinned. +# +on: + workflow_dispatch: + inputs: + examples_ref: + description: "benchx_examples branch, tag or commit to run" + default: main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + numpy-spin: + name: numpy-spin (track.sh, num_threads.sh) + runs-on: ubuntu-latest + timeout-minutes: 150 + env: + NUMPY: ${{ github.workspace }}/numpy + EXAMPLE: ${{ github.workspace }}/benchx_examples/examples/numpy-spin + BENCH_CPUS: "0,1" + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_MAXSIZE: 500M + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + # The repository keeps the output of earlier runs next to the scripts. + # Start clean instead + - name: Remove the output from examples/numpy-spin + working-directory: ${{ env.EXAMPLE }} + run: rm -rf results store.parquet + + - name: Install system packages + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libopenblas-dev pkg-config ccache + + # The key is new for every run, because a cache cannot be overwritten; a run starts + # from the most recent one. It is saved at the end, also when a later step fails. + - name: Restore the ccache cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ccache-${{ runner.os }}- + + - name: Zero the ccache statistics + run: ccache --zero-stats + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/prototype/uv.lock + + # The scripts run plain `python`, `bx` and `spin`, and hard-code + # lib/python3.12/site-packages, so this is Python 3.12 on the PATH. The `test` + # extra is only for pyperf, which the benchmarks need. + - name: Install benchx on Python 3.12 + working-directory: benchx/prototype + run: | + uv python install 3.12 + uv sync --extra test --python 3.12 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + - name: Install NumPy's build tools + run: uv pip install --python benchx/prototype/.venv/bin/python spin ninja meson-python cython + + # The full history and the tags: track.sh checks out v2.4.3 and v2.4.0. + - name: Clone NumPy + run: | + git clone --quiet https://github.com/numpy/numpy.git "$NUMPY" + git -C "$NUMPY" submodule update --init + + # Two cores for the benchmarks, all of them for the builds, so only `bx` is pinned: + # a `bx` ahead of the real one on the PATH, which the scripts call without knowing. + - name: Pin the benchmarks to two cores + run: | + mkdir -p "$RUNNER_TEMP/pin" + cat > "$RUNNER_TEMP/pin/bx" <> "$GITHUB_PATH" + "$RUNNER_TEMP/pin/bx" --help > /dev/null + + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "bx: $(command -v bx)" + echo "ccache: $(ccache --version | head -1)" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + echo "numpy main: $(git -C "$NUMPY" rev-parse --short main)" + + # num_threads.sh assumes a prebuilt NumPy at main, checked out. + - name: Build NumPy main + working-directory: ${{ env.NUMPY }} + run: spin build + + # Before track.sh, which leaves the NumPy checkout at the last revision it built. + - name: Run num_threads.sh + working-directory: ${{ env.EXAMPLE }} + run: bash ./num_threads.sh + + - name: Print the thread sweep results + working-directory: ${{ env.EXAMPLE }} + run: bx head --table -n 100 + + - name: Run track.sh + working-directory: ${{ env.EXAMPLE }} + run: bash ./track.sh + + - name: Print the results across history + working-directory: ${{ env.EXAMPLE }} + run: bx head --table -n 100 + + - name: Show the ccache statistics + if: always() + run: ccache --show-stats + + - name: Save the ccache cache + if: always() + uses: actions/cache/save@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: numpy-spin-output + path: | + benchx_examples/examples/numpy-spin/results + benchx_examples/examples/numpy-spin/store.parquet + if-no-files-found: warn + retention-days: 14 From 8017a84f72b44b3ed962be6567ae189f857feac7 Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Tue, 3 Nov 2026 18:39:53 +0000 Subject: [PATCH 2/7] Move the Arrow examples to benchx_examples, and run them in the e2e workflow prototype/examples/{demo-suite,demo,archery-local} and prototype/demo.sh now live in benchx-dev/benchx_examples as examples/arrow/{demo-suite,demo,archery-local} (the demo script is examples/arrow/demo/run.sh). - Remove them from this repository, and the archery-local step from tests.yml. - e2e.yml gets a second job, arrow, that runs demo/run.sh and archery-local/run.sh from benchx_examples and uploads their output. The header now describes both jobs. - prototype/README.md and docs/design/prototype-design.md point to benchx_examples where they described these examples. The arrow job needs examples/arrow on the benchx_examples ref it clones (main by default), so it passes once that move is merged there. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/e2e.yml | 77 +++++++++- .github/workflows/tests.yml | 8 - docs/design/prototype-design.md | 7 +- prototype/README.md | 43 +----- prototype/demo.sh | 136 ----------------- prototype/examples/archery-local/.gitignore | 2 - prototype/examples/archery-local/README.md | 141 ------------------ prototype/examples/archery-local/activate.sh | 8 - .../examples/archery-local/benchx.conf.json | 13 -- prototype/examples/archery-local/provider.sh | 77 ---------- prototype/examples/archery-local/run.sh | 84 ----------- prototype/examples/demo-suite/CMakeLists.txt | 23 --- prototype/examples/demo-suite/bench.cpp | 44 ------ prototype/examples/demo/.gitignore | 2 - prototype/examples/demo/benchx.conf.json | 10 -- 15 files changed, 83 insertions(+), 592 deletions(-) delete mode 100644 prototype/demo.sh delete mode 100644 prototype/examples/archery-local/.gitignore delete mode 100644 prototype/examples/archery-local/README.md delete mode 100644 prototype/examples/archery-local/activate.sh delete mode 100644 prototype/examples/archery-local/benchx.conf.json delete mode 100755 prototype/examples/archery-local/provider.sh delete mode 100644 prototype/examples/archery-local/run.sh delete mode 100644 prototype/examples/demo-suite/CMakeLists.txt delete mode 100644 prototype/examples/demo-suite/bench.cpp delete mode 100644 prototype/examples/demo/.gitignore delete mode 100644 prototype/examples/demo/benchx.conf.json diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3ae2e6e..69be0a6 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,9 +1,13 @@ name: e2e examples -# Runs the numpy-spin examples in benchx-dev/benchx_examples and prints their output: +# Runs examples from benchx-dev/benchx_examples and prints their output. Two jobs: # -# track.sh builds NumPy with spin at several revisions and benchmarks each -# num_threads.sh benchmarks one prebuilt revision at several OPENBLAS_NUM_THREADS +# numpy-spin builds NumPy with spin and benchmarks it: +# track.sh several revisions, each built and benchmarked +# num_threads.sh one prebuilt revision at several OPENBLAS_NUM_THREADS +# arrow builds a small Google Benchmark suite with CMake and compares the builds: +# examples/arrow/demo/run.sh +# examples/arrow/archery-local/run.sh # # Nothing is checked. The job fails only if a step fails (a clone, a build, a script that # crashes); the scripts exit 0 even when a revision fails, so read the printed output. @@ -12,9 +16,9 @@ name: e2e examples # benchx_examples branch, tag or commit (examples_ref), which a pull request here cannot: # the examples live in another repository. # -# The benchmarks are pinned to two cores (BENCH_CPUS). num_threads.sh should use 1, 2 and 4 -# threads, so that the last of them oversubscribes the two cores. The NumPy builds are not -# pinned. +# In the numpy-spin job the benchmarks are pinned to two cores (BENCH_CPUS). num_threads.sh +# should use 1, 2 and 4 threads, so that the last of them oversubscribes the two cores. The +# NumPy builds are not pinned. # on: workflow_dispatch: @@ -167,3 +171,64 @@ jobs: benchx_examples/examples/numpy-spin/store.parquet if-no-files-found: warn retention-days: 14 + + arrow: + name: arrow (demo, archery-local) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/prototype/uv.lock + + # The scripts run plain `python` and `bx`. benchx is installed in editable mode, + # which is where it reads its schemas from. + - name: Install benchx + working-directory: benchx/prototype + run: | + uv python install 3.13 + uv sync --extra test --python 3.13 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + # cmake and a C++ compiler are preinstalled on the runner; CMake fetches + # Google Benchmark. + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "bx: $(command -v bx)" + echo "cmake: $(cmake --version | head -1)" + echo "c++: $(c++ --version | head -1)" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + + - name: Run the demo + working-directory: benchx_examples/examples/arrow/demo + run: bash ./run.sh + + - name: Run archery-local + working-directory: benchx_examples/examples/arrow/archery-local + run: bash ./run.sh + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: arrow-output + path: | + benchx_examples/examples/arrow/demo/store.parquet + benchx_examples/examples/arrow/archery-local/_work/results + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ff9f74..c25bc6c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,14 +54,6 @@ jobs: shell: bash run: uv run --extra test --python ${{ matrix.python-version }} bash examples/pyperf-bench/run.sh - # Needs cmake and a C++ compiler (preinstalled on the Linux and macOS - # runners) and fetches Google Benchmark; the example is bash and CMake - # with Unix paths, so it is not run on Windows. - - name: Run the archery-local example - if: runner.os != 'Windows' - shell: bash - run: uv run --extra test --python ${{ matrix.python-version }} bash examples/archery-local/run.sh - # Makes two venvs of its own with uv (Python 3.12, the newest with numpy 1.26.4 wheels), so # the matrix Python only runs the outer script and one cell is enough. Needs network access. - name: Run the numpy-venvs example diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index 173029d..1a26a5e 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -560,9 +560,10 @@ Allowing a local-only `revisions` comparison of a dirty tree is open Both profiles use one shape: two prepared build directories, one run key, R rounds, and `bx compare` in measure mode, which alternates the sides and -assigns `round` and `slot`. The demo script plays the user, and in that role +assigns `round` and `slot`. The demo script (`examples/arrow/demo/run.sh` in +the benchx_examples repository) plays the user, and in that role the target provider: it prepares the sources and build directories with git -and CMake, since benchx builds nothing. `prototype/examples/archery-local` +and CMake, since benchx builds nothing. `examples/arrow/archery-local` there shows the other route, where a provider script named in `benchx.conf.json` builds and describes the targets and the sides are given as `@plain` and `@hardened`. @@ -589,7 +590,7 @@ re-ingested to show idempotency, and compared again with `--results`; that document must equal the one from the local store (criterion 8). Every `bx run` delivers to the local store, so both shapes' results land in -the demo project's store (`prototype/examples/demo/store.parquet`), the ad hoc ones as thin results of project +the demo project's store (`examples/arrow/demo/store.parquet` in benchx_examples), the ad hoc ones as thin results of project `local/`. Run keys carry a timestamp, so successive demo runs add rows rather than colliding on `ingest_key`. The store and each series' history grow from run to run, and the demo ends with `bx head`. diff --git a/prototype/README.md b/prototype/README.md index 0b5ecab..6b8075f 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -65,7 +65,8 @@ declare `how_built` and `activation` text, which are copied into the work order and into every result's `provenance.info`, shown in the comparison, and never executed or checked: you activate the environment yourself, and the runner records the allowlisted variables it inherited beside your declaration. -`examples/archery-local` is a complete walk-through. +`examples/arrow/archery-local` in [benchx_examples](https://github.com/benchx-dev/benchx_examples) +is a complete walk-through. `bx bench` is the single-target form of `bx compare`: the same way of naming a target (`@CONFIG[=SOURCE_REF]`, or a path), the same options, the same code; it writes one @@ -181,39 +182,14 @@ every result is pyperf. Results also record the installed project's version ```console $ uv venv && uv pip install -e '.[test]' # editable: schemas are read from ../schemas $ .venv/bin/python -m pytest tests # 344 tests, fake Google Benchmark binary, no compiler -$ ./demo.sh # real builds; needs cmake, a C++ compiler, network once ``` -`demo.sh` prints every command before running it, as you would type it -(`bx` for `.venv/bin/python -m benchx.cli`, `$WORK` for its scratch -directory), so it doubles as a walkthrough. It prepares a git source with two -commits and three CMake build directories of `examples/demo-suite` (base, -head, and head with `DEMO_HARDENED=ON`), shows the first work order of each -part, then: - -1. **revisions, ad hoc:** `bx compare` in measure mode alternates base and head - over five rounds with orders that name no project, then read mode compares - the same run key through a throwaway store; -2. **environments, tracked:** `bx compare` alternates plain and hardened, labeled - `build: plain|hardened`, compares from the local store, re-ingests - (no-op), and rejects a unit conflict; -3. checks that the throwaway store gives the same comparison document as the - persistent one, lists series and history, and ends with `bx head`. - -Every `bx run` and every measure-mode `bx compare` delivers to the local store, so each demo run adds its -results, ad hoc and tracked, to `examples/demo/store.parquet` (the demo's project root, set -in its `benchx.conf.json`). Run keys carry a -timestamp, so runs accumulate and each series' history grows. Look at it any -time: - -```console -$ (cd examples/demo && ../../.venv/bin/python -m benchx.cli head -n 20) -$ rm examples/demo/store.parquet # start over -``` - -Both changes add two to three percent of work. The quiet benchmark flags them -as regressed; the noisy one, with about ten percent spread between -repetitions, does not. +The end-to-end examples that build real software are in +[benchx_examples](https://github.com/benchx-dev/benchx_examples), and the `e2e` +workflow runs them. `examples/arrow/demo/run.sh` there is a walkthrough of both comparison +profiles on a small Google Benchmark suite, with real CMake builds (it needs cmake, a +C++ compiler, and network access once). It prints every command before running it, +as you would type it, so it doubles as a walkthrough. ## Layout @@ -243,10 +219,7 @@ benchx/ compare.py run mode over a profile's projection, comparison document (comparator.md) cli.py bx (compare has a read mode and a measure mode) ../schemas/ the result and work-order schemas, read in place ($BENCHX_SCHEMAS overrides) -examples/demo-suite a two-benchmark Google Benchmark suite for the demo -examples/demo the demo's project root: its benchx.conf.json, and its persistent store examples/pyperf-bench Python functions under `bx bench` (own README) -examples/archery-local an end-to-end, Arrow-local-shaped example with a target provider (own README) examples/numpy-venvs UC-04 with two venvs and two numpy versions, driven by run.sh (own README) tests/ test_prototype.py: the success criteria and the runner and store rules, on a fake Google Benchmark binary; test_comparator.py: the diff --git a/prototype/demo.sh b/prototype/demo.sh deleted file mode 100644 index ed46383..0000000 --- a/prototype/demo.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env bash -# End-to-end demonstration of prototype-design.md §6: both profiles, both shapes. -# -# The script plays the user, and in that role the target provider: it prepares -# the environment with its own tools, a git repository with two commits and -# three CMake build directories (benchx builds nothing). Then `bx compare` in -# measure mode does the workbench's part (system-decomposition.md §3.4): it -# alternates the sides over the rounds, assigns round and slot, delivers every -# result to the local store, and compares. -# -# Every command is printed before it runs, as you would type it: `bx` stands -# for `$PYTHON -m benchx.cli`, and $WORK for the scratch directory. -# -# Needs cmake, a C++ compiler, and network access the first time, to fetch -# Google Benchmark unless it is installed. ROUNDS=5 by default. -# -# `bx compare` (measure mode) and `bx run` deliver every result to the one local store, -# examples/demo/store.parquet (paths.store in examples/demo/benchx.conf.json, the demo's -# project file, from whose directory every `bx` command runs), so the store and -# each series' history grow from run to run. `bx head` shows the latest rows -# at any time. Delete the file to start over. -set -euo pipefail -cd "$(dirname "$0")" -PROTOTYPE=$PWD -PYTHON=${PYTHON:-$PROTOTYPE/.venv/bin/python} -ROUNDS=${ROUNDS:-5} -WORK=$(mktemp -d) -DEMO_ROOT=$PROTOTYPE/examples/demo -STORE=$DEMO_ROOT/store.parquet -STAMP=$(date -u +%Y%m%dT%H%M%SZ) # run keys are unique per demo run, so runs accumulate -RUN_REV="rev-adhoc-$STAMP" -RUN_ENV="env-tracked-$STAMP" -export GIT_AUTHOR_NAME=demo GIT_AUTHOR_EMAIL=demo@example.org -export GIT_COMMITTER_NAME=demo GIT_COMMITTER_EMAIL=demo@example.org - -step() { printf '\n\033[1m== %s\033[0m\n' "$*"; } - -# say : print a command as you would type it -quote() { - if [[ $1 =~ ^[A-Za-z0-9_./:=@%+,^-]+$ ]]; then printf '%s' "$1" - else printf "'%s'" "${1//\'/\'\\\'\'}"; fi -} -SAY_SUFFIX="" -say() { # to stderr, so a command's own redirected output never contains it - local line="" arg - for arg in "$@"; do line+=" $(quote "$arg")"; done - line="${line# }${SAY_SUFFIX:+ $SAY_SUFFIX}" - printf '\033[36m$ %s\033[0m\n' "${line//"$WORK"/\$WORK}" >&2 -} -show() { say "$@"; "$@"; } # print, then run -quiet() { say "$@"; "$@" >/dev/null; } # print, then run without its output -bx() { say bx "$@"; (cd "$DEMO_ROOT" && "$PYTHON" -m benchx.cli "$@"); } -into() { # into : run with stdout to a file, printed with its redirect - SAY_SUFFIX="> $1"; local out=$1; shift - "$@" > "$out" - SAY_SUFFIX="" -} - -printf '\033[36m$ WORK=%s\033[0m\n' "$WORK" >&2 - -step "user: a source with two commits, a worktree at each" -show cp -R examples/demo-suite "$WORK/src" -show git -C "$WORK/src" init -q -show git -C "$WORK/src" add -A -show git -C "$WORK/src" commit -qm "base" -BASE=$(git -C "$WORK/src" rev-parse HEAD) -show sed -i.bak 's/kWork = 100/kWork = 102/' "$WORK/src/bench.cpp" -rm "$WORK/src/bench.cpp.bak" -show git -C "$WORK/src" commit -qam "head: 2% more work" -HEAD=$(git -C "$WORK/src" rev-parse HEAD) -show git -C "$WORK/src" worktree add -q --detach "$WORK/wt-base" "$BASE" -show git -C "$WORK/src" worktree add -q --detach "$WORK/wt-head" "$HEAD" -echo "base $BASE" -echo "head $HEAD" - -step "user: three build directories (base, head, head hardened)" -build() { # build [cmake args] - quiet cmake -S "$1" -B "$2" -DCMAKE_BUILD_TYPE=Release -DFETCHCONTENT_BASE_DIR="$WORK/_deps" "${@:3}" - quiet cmake --build "$2" -j -} -build "$WORK/wt-base" "$WORK/build-base" -build "$WORK/wt-head" "$WORK/build-head" -build "$WORK/wt-head" "$WORK/build-hardened" -DDEMO_HARDENED=ON - -step "revisions, ad hoc: no project, measured and compared in one command" -echo "# measure mode: the sides alternate over $ROUNDS rounds, one work order each" -bx compare "$WORK/build-base:$WORK/wt-base" "$WORK/build-head:$WORK/wt-head" \ - --rounds "$ROUNDS" --run-key "$RUN_REV" --out "$WORK/results-rev" # adapter, suite and profile: benchx.conf.json -echo "# the first work order it wrote:" -show cat "$WORK/results-rev/orders/order-0000.json" -echo -echo "# read mode: the same run key, compared again through a throwaway store" -bx compare --run "$RUN_REV" --baseline "${BASE:0:10}" --results "$WORK/results-rev" -echo "# bx compare also delivered these to the local store, as thin results of project local/" - -step "environments, tracked: project demo, into the local store" -bx compare "$WORK/build-head:$WORK/wt-head" "$WORK/build-hardened:$WORK/wt-head" --profile environments \ - --label build=plain,hardened --rounds "$ROUNDS" --project demo \ - --run-key "$RUN_ENV" --out "$WORK/results-env" - -step "re-ingest is a visible no-op" -bx ingest "$WORK/results-env" - -step "a conflicting unit is rejected with its taxonomy code" -echo "# \$WORK/bad.json: a copy of one result, its unit changed from s to ms" -"$PYTHON" - "$WORK" <<'EOF' -import json, pathlib, sys -work = pathlib.Path(sys.argv[1]) -doc = json.loads(next(p for p in sorted((work / "results-env").glob("*_wall-time_*.json"))).read_text()) -doc["ingest_key"] += ":in-ms" -doc["measurement"]["quantity"]["unit"] = "ms" -(work / "bad.json").write_text(json.dumps(doc)) -EOF -bx ingest "$WORK/bad.json" || true - -step "compare from the local store" -bx compare --run "$RUN_ENV" --profile environments --label build --baseline plain - -step "the same comparison through a throwaway store gives the same document" -into "$WORK/a.json" bx compare --run "$RUN_ENV" --profile environments --label build --baseline plain --json -into "$WORK/b.json" bx compare --run "$RUN_ENV" --profile environments --label build --baseline plain --json \ - --results "$WORK/results-env" -show cmp "$WORK/a.json" "$WORK/b.json" && echo "identical" - -step "series and history (a query; history mode is deferred)" -into "$WORK/series.txt" bx series --workload BM_Quiet --quantity wall-time -show head -6 "$WORK/series.txt" -SERIES=$(awk '/demo .*est=benchx\/median/ {print $1; exit}' "$WORK/series.txt") -into "$WORK/history.txt" bx history "$SERIES" -show head -1 "$WORK/history.txt" -show tail -n 4 "$WORK/history.txt" - -step "the store is one Parquet file, and keeps growing" -show ls -l "$STORE" -bx head -n 8 -printf '\ndemo artifacts in %s\n' "$WORK" diff --git a/prototype/examples/archery-local/.gitignore b/prototype/examples/archery-local/.gitignore deleted file mode 100644 index 6acf7e7..0000000 --- a/prototype/examples/archery-local/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -_work/ -results/ diff --git a/prototype/examples/archery-local/README.md b/prototype/examples/archery-local/README.md deleted file mode 100644 index dfbbe50..0000000 --- a/prototype/examples/archery-local/README.md +++ /dev/null @@ -1,141 +0,0 @@ -# archery-local: an end-to-end example - -One commit, built two ways (plain and hardened), compared with noise-aware -verdicts, where the comparison says what was built and how. It follows the -Arrow local story, "Measure what a build option costs" -([`docs/user_stories/apache-arrow-local-benchmarking.md`](../../../docs/user_stories/apache-arrow-local-benchmarking.md)), -on a small stand-in suite (`../demo-suite`: a quiet and a noisy Google -Benchmark case, where hardening adds about 3% of work). - -It exercises the target-provider contract of -[`system-decomposition.md`](../../../docs/design/system-decomposition.md) §3.4: - -- **You are the user, and the provider is your build script.** `provider.sh` - configures and builds with CMake, as `archery` or a contributor would. benchx - builds nothing. -- **The provider tells benchx what it built** in a *target description*: the - build directory and its source, plus two pieces of text you own, `how_built` - (the exact commands) and `activation` (what to run to enter the environment, - with the `shell` it is written for). Either may be empty. -- **You activate the environment yourself** (`source activate.sh`). benchx never - runs `activation`. It records the text, as given, in the work order and in - every result's `provenance.info`, beside the allowlisted environment variables - the runner actually inherited. If the two disagree, both stay and nothing - judges. - -## Run it - -You need `cmake`, a C++ compiler, `git` and `python3` on your `PATH`, and network -access the first time (CMake fetches Google Benchmark v1.9.1 unless it is -installed). From the repository root: - -```console -$ cd prototype -$ python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like -$ pip install -e . -$ examples/archery-local/run.sh -``` - -It takes about two minutes the first time (two CMake builds and a measured -run) and prints every command before running it, as you would type it. The -script runs `bx` and `python` from the active environment. More rounds: -`ROUNDS=10 ./run.sh`. - -Everything it creates is under `_work/` in this directory, including the local -store (`_work/store.parquet`, set by `paths.store` in `benchx.conf.json`), so nothing -outside this directory changes. `rm -rf _work` starts over. - -## What you will see - -1. **Activate.** `source activate.sh` puts a stand-in dependencies directory on - `LD_LIBRARY_PATH`. This is you, not benchx. -2. **`bx target describe @plain`** fails with exit 3 from the provider: nothing is - built yet. `describe` builds nothing, by contract. -3. **`bx target prepare @plain` / `@hardened`** make the provider build. Its CMake - output is the build's own, on stderr; the target description it prints on - stdout is the answer. `how_built` is the very commands the provider ran. -4. **`bx target describe @hardened`** now works. The same document is stored in - the build directory as `.benchx-target.json`, so a long-lived build directory - carries its own description (point `bx compare` at a bare directory and it - reads that file; no provider needed). -5. **`bx compare @plain @hardened ...`** is the workbench's session loop: it - asks the provider to prepare each side (a no-op build now), then alternates - the sides over the rounds, one work order each, delivers the results to the - local store, and compares: - - ``` - run hardening-... profile environments baseline plain contender hardened - plain: how built: cmake -S .../_work/src -B .../_work/build-plain ... (+1 lines) - plain: activation: source .../activate.sh - plain: observed env: LD_LIBRARY_PATH=.../_work/deps/lib - hardened: how built: cmake -S ... -B .../_work/build-hardened ... -DDEMO_HARDENED=ON (+1 lines) - ... - REGRESSED (1) - BM_Quiet/1024 wall-time effect +2.42% noise 0.57% rounds=5 - NO CHANGE DETECTED (1) - BM_Noisy/1024 wall-time effect +3.43% noise 3.04% rounds=5 - ``` - - Compare that with the story's pains: the comparison now describes itself - (what was built, how, in what environment), and the verdict uses each - benchmark's own noise instead of a flat 5%: the quiet case flags a 2.4% - change that is four times its noise, the noisy case does not flag a 3.4% - swing inside its noise. The numbers vary from run to run; with few rounds the - noisy case may flag too. -6. **The work order and a result**, printed: the order's `target` carries - `how_built`, `activation`, `shell` and `provider`; the result's - `provenance.info` carries them, next to `coordinates.environment.system.process.environment` and - `coordinates.subject.configuration` (read by the runner from - `CMakeCache.txt`). -7. **Forget to activate.** The same comparison with `LD_LIBRARY_PATH` removed - from the environment: the declaration still says `source .../activate.sh`, - the observed environment no longer lists `LD_LIBRARY_PATH`, and nothing - refuses: declarations are recorded as given (`benchmark-environments.md` - §3.2), never checked. - -## Try it by hand - -After `./run.sh` (the build directories and the store stay in `_work/`). From -this directory, with the environment still active: - -```console -$ source activate.sh - -$ bx target describe @hardened # what exists, nothing built -$ bx target prepare @hardened # incremental build, then the description -$ bx target describe _work/build-hardened # a bare directory: its own sidecar - -# the story's own knobs: scope by filter, precision by repetitions -$ bx compare @plain @hardened --label build=plain,hardened --filter BM_Quiet --rounds 10 \ - --repetitions 5 --run-key mine # the adapter, suite, project and profile come from benchx.conf.json - -$ bx compare --run mine --label build --baseline plain # read mode -$ bx head -n 4 # the latest stored results -$ bx head --json -n 1 | python3 -m json.tool | less # provenance.info, environment.system -``` - -Edit `provider.sh`'s `hardened)` case or `activate.sh` and run again to see how -each change shows up in the next comparison. Empty text is fine: set -`"activation": ""` (or `null`, or leave it out) in a description and the -runner simply omits it from the results. - -## What is in this directory - -| File | Role | -|---|---| -| `run.sh` | the end-to-end script | -| `provider.sh` | the target provider: your build script, speaking the `describe`/`prepare` contract | -| `benchx.conf.json` | the project file: the provider command, the harness and its suite, the project name, the comparison profile, and where the store and results go | -| `activate.sh` | the environment you activate before calling `bx` | -| `_work/` | everything generated: source copy, build directories, results, store (git-ignored) | - -The documents involved are in `../../../schemas/`: `target-provider-request`, -`target-description`, and `work-order` (its `target` takes the declared text). - -## What this does not show - -The provider here builds one source tree, `WORKSPACE`. Comparing against a tag -or a past revision (`@plain=v1.0`) is the provider's business and is not -implemented in this example; the prototype's own `demo.sh` shows revisions -with two prebuilt worktrees. A declared `build` object in a description is -refused by the prototype's runner (it reads `CMakeCache.txt` instead). diff --git a/prototype/examples/archery-local/activate.sh b/prototype/examples/archery-local/activate.sh deleted file mode 100644 index cd10b8b..0000000 --- a/prototype/examples/archery-local/activate.sh +++ /dev/null @@ -1,8 +0,0 @@ -# The environment this example's benchmarks are meant to run in. In the Arrow -# story this is where you would load a toolchain or point at the dependencies -# you built; here it only puts a stand-in "deps" directory on the library -# search path. You source it yourself, before invoking bx: benchx never runs -# it, it records the text that says you did (the target description's -# `activation`) next to the environment it actually inherited. -HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -export LD_LIBRARY_PATH="$HERE/_work/deps/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" diff --git a/prototype/examples/archery-local/benchx.conf.json b/prototype/examples/archery-local/benchx.conf.json deleted file mode 100644 index 9029306..0000000 --- a/prototype/examples/archery-local/benchx.conf.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema_version": "benchx/config/0.1.0", - "project": "archery-local", - "provider": { "command": ["./provider.sh"] }, - "adapter": { - "name": "google-benchmark", - "suite": "demo-bench", - "quantities": ["wall-time"], - "precision": { "repetitions": 5 } - }, - "paths": { "store": "_work/store.parquet", "results": "_work/results" }, - "compare": { "profile": "environments" } -} diff --git a/prototype/examples/archery-local/provider.sh b/prototype/examples/archery-local/provider.sh deleted file mode 100755 index a5226e0..0000000 --- a/prototype/examples/archery-local/provider.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash -# The target provider of this example (system-decomposition.md §3.4). -# -# It is the user's own build script, speaking the provider contract: one JSON -# request on stdin (schemas/target-provider-request/0.1.0), one target -# description on stdout (schemas/target-description/0.1.0). Everything else, -# above all the build's own output, goes to stderr. -# -# describe reports a target that exists and builds nothing; exit 3 if none -# prepare configures and builds it, with CMake, as Arrow's contributors do -# (an incremental no-op when it is already built), then reports it -# -# Configurations: `plain`, and `hardened` (-DDEMO_HARDENED=ON), the same -# commit built two ways, like the -DARROW_HARDENING=ON story. -# -# `how_built` is assembled from the very commands this script runs, so the -# text cannot drift from what happened. benchx never runs any of it. -set -euo pipefail -HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -WORK=${BENCHX_EXAMPLE_WORK:-$HERE/_work} -export GIT_AUTHOR_NAME=demo GIT_AUTHOR_EMAIL=demo@example.org -export GIT_COMMITTER_NAME=demo GIT_COMMITTER_EMAIL=demo@example.org - -request=$(cat) -field() { python3 -c 'import json,sys; print(json.load(sys.stdin).get(sys.argv[1]) or "")' "$1" <<<"$request"; } -operation=$(field operation) -configuration=$(field configuration) -case "$configuration" in - plain) extra=() ;; - hardened) extra=(-DDEMO_HARDENED=ON) ;; - *) echo "provider: unknown configuration '$configuration' (plain or hardened)" >&2; exit 2 ;; -esac -SRC=$WORK/src -BUILD=$WORK/build-$configuration - -describe() { - [[ -f $BUILD/.benchx-target.json ]] || { echo "provider: $configuration is not built; use prepare" >&2; exit 3; } - cat "$BUILD/.benchx-target.json" -} - -if [[ $operation == describe ]]; then describe; exit 0; fi - -HOW="" -step() { # step : record it in how_built, show it, run it (output to stderr) - local line="$*" - HOW+="$line"$'\n' - printf '\033[2m provider$ %s\033[0m\n' "$line" >&2 - "$@" >&2 -} - -if [[ ! -d $SRC/.git ]]; then - mkdir -p "$WORK" - cp -R "$HERE/../demo-suite" "$SRC" - git -C "$SRC" init -q - git -C "$SRC" remote add origin https://example.org/archery-local/demo-suite.git - git -C "$SRC" add -A - git -C "$SRC" commit -qm "demo-suite" -fi -mkdir -p "$WORK/deps/lib" # stand-in for the dependencies activate.sh points at - -step cmake -S "$SRC" -B "$BUILD" -DCMAKE_BUILD_TYPE=Release -DFETCHCONTENT_BASE_DIR="$WORK/_deps" ${extra[@]+"${extra[@]}"} -step cmake --build "$BUILD" -j - -BUILD=$BUILD SRC=$SRC HOW=$HOW CONFIGURATION=$configuration HERE=$HERE python3 - >"$BUILD/.benchx-target.json" <<'PY' -import json, os -print(json.dumps({ - "schema_version": "benchx/target-description/0.1.0", - "target": {"kind": "build", "path": os.environ["BUILD"], - "source": {"uri": "https://example.org/archery-local/demo-suite.git", "type": "git"}, - "source_dir": os.environ["SRC"]}, - "how_built": os.environ["HOW"].rstrip("\n"), - "activation": "source " + os.path.join(os.environ["HERE"], "activate.sh"), - "shell": "bash", - "provider": {"name": "provider.sh", "version": "1"}, -}, indent=1)) -PY -describe # the sidecar just written, which stays with the build directory diff --git a/prototype/examples/archery-local/run.sh b/prototype/examples/archery-local/run.sh deleted file mode 100644 index 6389898..0000000 --- a/prototype/examples/archery-local/run.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash -# End to end, in the shape of the Arrow local story ("Measure what a build -# option costs"): one commit built two ways, plain and hardened, compared with -# noise-aware verdicts, where the comparison describes itself. -# -# You play the user. The target provider (provider.sh, named in -# benchx.conf.json) is your own build script: it builds with CMake and -# tells benchx what it built, how, and what environment you need. benchx builds -# nothing, and runs none of that text: you activate the environment yourself -# (activate.sh) and benchx records the text beside the environment it inherited. -# -# Every command is printed before it runs, as you would type it. Run it in an -# activated environment that has benchx installed (from prototype/: -# `pip install -e .`, or `uv sync --extra test`). Needs cmake and a C++ -# compiler on PATH, and network access the first time, to fetch Google -# Benchmark unless installed. -# -# ROUNDS=5 alternation rounds (default 5) -# -# benchx.conf.json is this directory's project file, so the local store is kept -# inside it too (_work/store.parquet). Delete _work/ to start over. -set -euo pipefail -cd "$(dirname "$0")" -HERE=$PWD -ROUNDS=${ROUNDS:-5} -STAMP=$(date -u +%Y%m%dT%H%M%SZ) -RUN="hardening-$STAMP" - -for tool in cmake c++ bx python; do - command -v "$tool" >/dev/null || { echo "run.sh: $tool not found on PATH (see README.md)" >&2; exit 1; } -done -bx --help >/dev/null 2>&1 || { echo "run.sh: benchx is not installed in the active environment (see README.md)" >&2; exit 1; } - -step() { printf '\n\033[1m== %s\033[0m\n' "$*"; } -say() { printf '\033[36m$ %s\033[0m\n' "$*" >&2; } -bx() { say bx "$@"; command bx "$@"; } - -step "user: activate the environment (benchx will not do this for you)" -say source activate.sh -source activate.sh -echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" - -step "ask the provider about a target that is not built yet: describe builds nothing" -bx target describe @plain || echo "(exit $?: nothing to describe yet, as expected)" - -step "prepare both configurations: the provider builds (its build output is the dimmed lines)" -bx target prepare @plain > /dev/null -bx target prepare @hardened - -step "describe now works, and the description lives in the build directory too" -bx target describe @hardened | head -3 -say head -3 _work/build-hardened/.benchx-target.json -head -3 _work/build-hardened/.benchx-target.json -echo "(the same document: a build directory carries its own description)" - -step "compare: the workbench asks the provider to prepare each side (a no-op), then alternates them" -bx compare @plain @hardened --label build=plain,hardened --rounds "$ROUNDS" --run-key "$RUN" --out _work/results - -step "the comparison names what was built and how; before, it was two opaque directory paths" -echo "# the target of work order 0001 (round 1, the hardened side), as the runner received it:" -python -c ' -import json, sys -print(json.dumps(json.load(open(sys.argv[1]))["target"], indent=1))' _work/results/orders/order-0001.json - -step "and every result carries it, next to what the runner saw (declared, observed: nothing judges)" -say bx head --json -n 1 '| …' -command bx head --json -n 1 | python -c ' -import json, sys -doc = json.loads(sys.stdin.readline()) -info = doc["provenance"]["info"] -print("provenance.info.how_built :", info["how_built"].replace("\n", "\n" + " " * 29)) -print("provenance.info.activation :", info["activation"]) -print("provenance.info.shell :", info["shell"]) -print("provenance.info.target_provider:", info["target_provider"]) -system = doc["coordinates"]["environment"]["system"] -print("environment.system.process.environment:", system.get("process", {}).get("environment")) -print("coordinates.subject.configuration:", doc["coordinates"]["subject"].get("configuration"))' - -step "forget to activate: the declaration still says what you should have run, the observation says what you did" -say env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build --rounds 3 ... -env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build \ - --label build=plain,hardened --rounds 3 --run-key "$RUN-unactivated" --out _work/results-unactivated - -printf '\nartifacts: %s/_work (results, orders, build directories, the local store)\n' "$HERE" diff --git a/prototype/examples/demo-suite/CMakeLists.txt b/prototype/examples/demo-suite/CMakeLists.txt deleted file mode 100644 index 503ff46..0000000 --- a/prototype/examples/demo-suite/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -cmake_minimum_required(VERSION 3.20) -project(demo CXX) - -option(DEMO_HARDENED "Simulated hardening checks, the one option the environments demo changes" OFF) -set(CMAKE_CXX_STANDARD 17) - -find_package(benchmark QUIET) -if(NOT benchmark_FOUND) - include(FetchContent) - set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) - set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "" FORCE) - FetchContent_Declare(benchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.1 - GIT_SHALLOW TRUE) - FetchContent_MakeAvailable(benchmark) -endif() - -add_executable(demo-bench bench.cpp) -target_link_libraries(demo-bench benchmark::benchmark_main) -if(DEMO_HARDENED) - target_compile_definitions(demo-bench PRIVATE DEMO_HARDENED) -endif() diff --git a/prototype/examples/demo-suite/bench.cpp b/prototype/examples/demo-suite/bench.cpp deleted file mode 100644 index 01d5938..0000000 --- a/prototype/examples/demo-suite/bench.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Two benchmarks for the prototype demo: a quiet one and a noisy one. -// -// kWork is what the revisions demo changes between commits; DEMO_HARDENED is -// what the environments demo changes between two builds of one commit. Both -// add a few percent of work, so the expected verdicts are known: the quiet -// benchmark flags the change, the noisy one does not. -#include - -#include -#include - -constexpr int kWork = 100; -#ifdef DEMO_HARDENED -constexpr int kExtra = 3; // simulated cost of hardening checks, in percent -#else -constexpr int kExtra = 0; -#endif - -static double Sum(const std::vector& v, long rounds) { - double s = 0; - for (long r = 0; r < rounds; ++r) { - for (double x : v) s += x; - benchmark::DoNotOptimize(s); - } - return s; -} - -static void BM_Quiet(benchmark::State& state) { - std::vector v(state.range(0), 1.0); - const long rounds = kWork * (100 + kExtra); - for (auto _ : state) benchmark::DoNotOptimize(Sum(v, rounds)); -} -BENCHMARK(BM_Quiet)->Arg(1024); - -// Each repetition draws its own load level, so repetitions disagree by -// around ten percent, as they do for a benchmark at the mercy of its machine. -static void BM_Noisy(benchmark::State& state) { - std::vector v(state.range(0), 1.0); - std::random_device device; - std::uniform_real_distribution level(1.0, 1.25); - const long rounds = static_cast(kWork * (100 + kExtra) * level(device)); - for (auto _ : state) benchmark::DoNotOptimize(Sum(v, rounds)); -} -BENCHMARK(BM_Noisy)->Arg(1024); diff --git a/prototype/examples/demo/.gitignore b/prototype/examples/demo/.gitignore deleted file mode 100644 index 13eecef..0000000 --- a/prototype/examples/demo/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -store.parquet -results/ diff --git a/prototype/examples/demo/benchx.conf.json b/prototype/examples/demo/benchx.conf.json deleted file mode 100644 index fae92f0..0000000 --- a/prototype/examples/demo/benchx.conf.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "schema_version": "benchx/config/0.1.0", - "adapter": { - "name": "google-benchmark", - "suite": "demo-bench", - "precision": { "repetitions": 5 } - }, - "paths": { "store": "store.parquet", "results": "results" }, - "compare": { "profile": "revisions" } -} From 10be34b2798eb00b870b30bd3941030009d3e868 Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Tue, 3 Nov 2026 18:53:04 +0000 Subject: [PATCH 3/7] Move the pyperf example to benchx_examples, and run it in the e2e workflow prototype/examples/pyperf-bench now lives in benchx-dev/benchx_examples as examples/pyperf-suite. - Remove it from this repository, and its step from tests.yml. - e2e.yml gets a third job, pyperf-suite, that runs run.sh from benchx_examples and uploads its out/ directory. The header lists all three jobs. - prototype/README.md points to benchx_examples where it described the example. The job needs examples/pyperf-suite on the benchx_examples ref it clones (main by default). benchx_examples PR #3 adds the same files unchanged, which do not run from there; the port with adapted instructions is needed. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/e2e.yml | 57 ++++++- .github/workflows/tests.yml | 4 - prototype/README.md | 4 +- prototype/examples/pyperf-bench/.gitignore | 1 - prototype/examples/pyperf-bench/README.md | 151 ------------------ .../examples/pyperf-bench/bench_suite.py | 39 ----- .../examples/pyperf-bench/benchx.conf.json | 13 -- prototype/examples/pyperf-bench/run.sh | 47 ------ 8 files changed, 58 insertions(+), 258 deletions(-) delete mode 100644 prototype/examples/pyperf-bench/.gitignore delete mode 100644 prototype/examples/pyperf-bench/README.md delete mode 100644 prototype/examples/pyperf-bench/bench_suite.py delete mode 100644 prototype/examples/pyperf-bench/benchx.conf.json delete mode 100644 prototype/examples/pyperf-bench/run.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 69be0a6..b243c29 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,6 +1,6 @@ name: e2e examples -# Runs examples from benchx-dev/benchx_examples and prints their output. Two jobs: +# Runs examples from benchx-dev/benchx_examples and prints their output. Three jobs: # # numpy-spin builds NumPy with spin and benchmarks it: # track.sh several revisions, each built and benchmarked @@ -8,6 +8,8 @@ name: e2e examples # arrow builds a small Google Benchmark suite with CMake and compares the builds: # examples/arrow/demo/run.sh # examples/arrow/archery-local/run.sh +# pyperf-suite runs a suite of Python functions with `bx bench`: +# examples/pyperf-suite/run.sh # # Nothing is checked. The job fails only if a step fails (a clone, a build, a script that # crashes); the scripts exit 0 even when a revision fails, so read the printed output. @@ -232,3 +234,56 @@ jobs: benchx_examples/examples/arrow/archery-local/_work/results if-no-files-found: warn retention-days: 14 + + pyperf-suite: + name: pyperf-suite (run.sh) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/prototype/uv.lock + + # The script runs plain `python` and `bx`, and the workers need pyperf, which the + # `test` extra brings. benchx is installed in editable mode, which is where it reads + # its schemas from. + - name: Install benchx + working-directory: benchx/prototype + run: | + uv python install 3.13 + uv sync --extra test --python 3.13 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "bx: $(command -v bx)" + echo "pyperf: $(python -c 'import pyperf; print(pyperf.__version__)')" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + + - name: Run the pyperf suite + working-directory: benchx_examples/examples/pyperf-suite + run: bash ./run.sh + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: pyperf-suite-output + path: benchx_examples/examples/pyperf-suite/out + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c25bc6c..1e64d68 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -50,10 +50,6 @@ jobs: - name: Run tests run: uv run --extra test --python ${{ matrix.python-version }} python -m pytest tests -ra - - name: Run the pyperf-bench example - shell: bash - run: uv run --extra test --python ${{ matrix.python-version }} bash examples/pyperf-bench/run.sh - # Makes two venvs of its own with uv (Python 3.12, the newest with numpy 1.26.4 wheels), so # the matrix Python only runs the outer script and one cell is enough. Needs network access. - name: Run the numpy-venvs example diff --git a/prototype/README.md b/prototype/README.md index 6b8075f..ec3dace 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -88,7 +88,8 @@ pyperf. functions: the module's git checkout is the target, its path the suite, and the functions are exact case names. `bx run` is the layer underneath: it executes a finished order, so `bx run results/KEY/orders/order.json` replays any bench run. -`examples/pyperf-bench` is a runnable walk-through. +`examples/pyperf-suite` in [benchx_examples](https://github.com/benchx-dev/benchx_examples) +is a runnable walk-through. Environments that cannot share a process, such as two venvs with different numpy versions, are measured by separate `bx bench` calls that share a `--run-key`; `--label NAME=VALUE`, `--round N` @@ -219,7 +220,6 @@ benchx/ compare.py run mode over a profile's projection, comparison document (comparator.md) cli.py bx (compare has a read mode and a measure mode) ../schemas/ the result and work-order schemas, read in place ($BENCHX_SCHEMAS overrides) -examples/pyperf-bench Python functions under `bx bench` (own README) examples/numpy-venvs UC-04 with two venvs and two numpy versions, driven by run.sh (own README) tests/ test_prototype.py: the success criteria and the runner and store rules, on a fake Google Benchmark binary; test_comparator.py: the diff --git a/prototype/examples/pyperf-bench/.gitignore b/prototype/examples/pyperf-bench/.gitignore deleted file mode 100644 index 89f9ac0..0000000 --- a/prototype/examples/pyperf-bench/.gitignore +++ /dev/null @@ -1 +0,0 @@ -out/ diff --git a/prototype/examples/pyperf-bench/README.md b/prototype/examples/pyperf-bench/README.md deleted file mode 100644 index af112cf..0000000 --- a/prototype/examples/pyperf-bench/README.md +++ /dev/null @@ -1,151 +0,0 @@ -# Example: benchmarking Python functions with `bx bench` - -A runnable example of a benchmark suite. `bench_suite.py` holds five small -functions; `bx bench` runs each one under [pyperf](https://pyperf.readthedocs.io), -writes benchx result files, delivers them to a local store, and prints a table. - -## Run it - -```sh -cd prototype -python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like -pip install -e . pyperf -examples/pyperf-bench/run.sh -``` - -The script runs `bx` and `python` from the active environment, so benchx and pyperf must -be installed there; the workers use that same `python`. Extra arguments go to -`bx bench` in step 1 and win over the config file, e.g. `./run.sh --processes 20`. -It takes about 20 seconds. Nothing outside this directory changes: the local store is -`out/store.parquet` (`paths.store` in `benchx.conf.json`), and `out/` is deleted at the start -of each run. - -## What goes in - -| Input | Where | What it does | -|---|---|---| -| `.` | first argument | The target: this directory, a working tree. pyperf reads a bare path that way. | -| `benchx.conf.json` | found by walking up from where you run `bx`; its directory is the project root | The harness (`adapter.name`), the module (`adapter.suite`, `bench_suite.py`: each function named `bench_*` or `benchmark_*` is a case), and the precision: worker processes, values per process, calibration, warmup. Here: 5 processes x 3 values, calibrate to 0.05 s per value. | -| Function names | after the target | `bx bench . bench_sorted bench_join` runs only those, any function, whatever its name. `bx bench bench_suite.py bench_sorted` still works as a shorthand: the module's git checkout is the target and its path the suite. | -| Flags | command line | `--processes`, `--values`, `--min-time`, `--loops`, `--warmups` override the file, and so does `--suite`. | -| `-e NAME=VALUE` | command line | Sets an environment variable for the workers. The workers otherwise inherit your whole environment. | - -Precision is resolved as: **flags, then the config file, then pyperf's defaults** -(20 processes x 3 values, 0.1 s calibration, 1 warmup). A level you leave out of the -file takes pyperf's default. `bx config init` writes a `benchx.conf.json` that states -the defaults, as a template to edit, and `bx config show` says where each value comes from. The resolved values are written into the work order, -so a result records the precision that applied. A bad config file is reported before -anything runs. - -The benchmark functions are timed bodies: pyperf calls each one many times, and does -the calibration (how many calls make up one value) itself. Put setup at module level. - -## The config file - -`benchx.conf.json` is the example's project file. Its `adapter.precision` is the work-order -`precision` block; `adapter.name` and `adapter.suite` say which harness and which module: - -```json -{ - "schema_version": "benchx/config/0.1.0", - "adapter": { - "name": "pyperf", - "suite": "bench_suite.py", - "precision": { - "repetitions": { - "mode": "fixed", - "levels": [{"unit": "process", "n": 5}, {"unit": "value", "n": 3}] - }, - "calibration": {"mode": "adaptive", "minimum_sample_seconds": 0.05}, - "warmup": {"mode": "count", "n_warmup": 1} - } - }, - "paths": {"store": "out/store.parquet", "results": "out"} -} -``` - -- `repetitions.levels`: `process` is the number of worker processes, `value` the values - each one records. Leave a level out and it takes pyperf's default (20 and 3). -- `calibration`: `adaptive` with `minimum_sample_seconds` lets pyperf pick how many calls - make up one value; `{"mode": "fixed", "n_iterations": N}` fixes it. -- `warmup`: `{"mode": "count", "n_warmup": N}` discards N values per process, or `{"mode": "none"}`. -- Every key is optional; pyperf's defaults (20 x 3, adaptive 0.1 s, 1 warmup) fill the rest. - Run `bx config init` in your own project to get a file that states them all. - -## What `run.sh` does - -1. `bx bench .` runs all five cases with the config file's suite and precision. -2. `bx bench . bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2` - runs one case, sets a variable for the workers, and overrides the file's process count. -3. `bx bench . bench_join --run-key example-again` runs one case a second time - under a new run key, so its series gets a second point. -4. `ls -l` and `bx head` show the local store: one Parquet file, and its latest rows. -5. `bx series` lists the series in the store. -6. `bx history` lists the points of the `bench_join` series, one per run. The script fails - if there are not two, which is what CI relies on. -7. It lists the files written. - -## What comes out - -The table, for step 1 (your numbers will differ): - -``` -case median mean ± sd min .. max n status ------------------ -------- ------------------- -------------------- -- ------- -bench_dict_build 18.29 us 18.56 us ± 680.9 ns 17.71 us .. 19.83 us 15 success -bench_join 3.585 us 3.543 us ± 272.4 ns 3.236 us .. 3.997 us 15 success -``` - -`n` is processes x values (5 x 3 = 15). Each value is the time of one call, in seconds -underneath. A `[warning]` after the status means the result carries a quality warning, -such as `env-mismatch`. - -Files, under `out/all/` and `out/env/`: - -- `__wall-time.json`: one benchx result per case (schema `measurement-result`; - with `-e` the name also carries the percent-encoded parameters, as in - `example-env_bench_env_var_%7B%22EXAMPLE_N%22%3A%22100000%22%7D_wall-time.json`): - the observations grouped by worker process, the pyperf protocol that applied, the - environment, the revision of the tree, and warnings. -- `__load-average.json`: beside each case's result, the 1-minute load - average the runner read before the case ran, a condition measurement with the same - `attempt_key` (schema §4.4). `bx series` lists its series too; `--quantity wall-time` - leaves them out. -- `workorder-.json`: the work order that was run, and `orders/order.json`, the - same order as written for you to read. Results refer to it by hash. -- `artifacts//NNNN.native.json`: pyperf's own output; `.driver.py`: the generated - script that ran the function; `.worker-env.json`: the environment a worker really had; - `.stdout` and `.stderr`. - -The local store, `out/home/store.parquet`, holds the results. `run.sh` reads it back -(your numbers and fingerprints will differ): - -``` -$ bx series --quantity wall-time -d780c9564c72 local/be8372c04606 bench_dict_build wall-time [s] est=benchx/median/v1 -e6eaa02c8309 local/be8372c04606 bench_env_var wall-time [s] est=benchx/median/v1 -4be59ef6debc local/be8372c04606 bench_join wall-time [s] est=benchx/median/v1 -... -85a6aa9c6996 local/be8372c04606 bench_env_var wall-time [s] est=benchx/median/v1 - -$ bx history 4be59ef6debc -revision order unavailable without a revision graph; listed by measurement start -d83d37b69010 2026-10-13T11:04:28.656515Z 3.32107e-06 (server) example-all:bench_join:wall-time -d83d37b69010 2026-10-13T11:04:39.045277Z 3.76987e-06 (server) example-again:bench_join:wall-time -``` - -`bench_env_var` has two series because step 2 sets `EXAMPLE_N`, which changes the -environment the case ran in. `bench_join` has one series with two points, one for each -run key. You can also `bx ingest` more result files; comparing two runs is not part of -this step. - -## Notes - -- The tree measured is the git checkout containing the module (here, this repository), as - found, uncommitted changes included. The result records the revision and whether the tree - was dirty. Because that tree is the whole repository, an uncommitted change anywhere in it, - in a doc or another example as much as in `bench_suite.py`, makes the result `dirty` and - changes its tree id; commit or stash first for a clean result. Outside git, it records a hash of the directory's contents. `--tree DIR` picks - the tree. -- pyperf prints advice when the machine is noisy; it is in `.stderr`. On a laptop, - close other programs. `python -m pyperf system tune` (needs root) quiets the CPU further. diff --git a/prototype/examples/pyperf-bench/bench_suite.py b/prototype/examples/pyperf-bench/bench_suite.py deleted file mode 100644 index cf6589d..0000000 --- a/prototype/examples/pyperf-bench/bench_suite.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Example benchmark suite: plain Python functions, one case each. - -`bx bench bench_suite.py` runs every function named bench_* or benchmark_*. -Name functions on the command line to run only those: - bx bench bench_suite.py bench_sorted bench_join - -Each function is the timed body; pyperf calls it many times. Do your setup -at module level (it runs once per worker process, outside the timing). -""" - -import os - -DATA = list(range(2000, 0, -1)) -WORDS = [f"word{i}" for i in range(500)] - - -def bench_sorted(): - sorted(DATA) - - -def bench_join(): - ",".join(WORDS) - - -def bench_dict_build(): - {w: len(w) for w in WORDS} - - -def bench_sum_squares(): - sum(i * i for i in range(1000)) - - -def bench_env_var(): - """Reads EXAMPLE_N from the environment, to show `-e NAME=VALUE` reaching the workers.""" - int(os.environ.get("EXAMPLE_N", "100")) ** 2 - - -def helper_not_a_case(): - """Not named bench_*, so only run if you name it explicitly.""" diff --git a/prototype/examples/pyperf-bench/benchx.conf.json b/prototype/examples/pyperf-bench/benchx.conf.json deleted file mode 100644 index bc59544..0000000 --- a/prototype/examples/pyperf-bench/benchx.conf.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema_version": "benchx/config/0.1.0", - "adapter": { - "name": "pyperf", - "suite": "bench_suite.py", - "precision": { - "repetitions": { "mode": "fixed", "levels": [{ "unit": "process", "n": 5 }, { "unit": "value", "n": 3 }] }, - "calibration": { "mode": "adaptive", "minimum_sample_seconds": 0.05 }, - "warmup": { "mode": "count", "n_warmup": 1 } - } - }, - "paths": { "store": "out/store.parquet", "results": "out" } -} diff --git a/prototype/examples/pyperf-bench/run.sh b/prototype/examples/pyperf-bench/run.sh deleted file mode 100644 index 21a2fb9..0000000 --- a/prototype/examples/pyperf-bench/run.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Run the example suite with `bx bench`. See README.md for what each step does. -# -# ./run.sh uses benchx.conf.json here (5 processes x 3 values) -# ./run.sh --processes 20 extra arguments go to `bx bench` and win over the file -# -# Run it in an activated environment that has benchx installed with pyperf -# (from prototype/: `pip install -e . pyperf`, or `uv sync --extra test`). -# Results go to ./out, and the local store to ./out/store.parquet (both set in -# benchx.conf.json), so nothing outside this directory changes. -set -euo pipefail -cd "$(dirname "$0")" -bx() { printf '\033[36m$ bx %s\033[0m\n' "$*" >&2; command bx "$@"; } - -command bx --help >/dev/null 2>&1 && python -c "import pyperf" 2>/dev/null || { - echo "missing dependencies: install benchx and pyperf into the active environment (see README.md)" >&2 - exit 1; } -rm -rf out; mkdir -p out - -echo "== 1. all bench_* functions, suite and precision from benchx.conf.json" -bx bench . --run-key example-all --out out/all "$@" - -echo; echo "== 2. one function, an environment variable set for the workers, flags over the file" -bx bench . bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2 \ - --run-key example-env --out out/env - -echo; echo "== 3. the same case again under a new run key, so its series gets a second point" -bx bench . bench_join --run-key example-again --out out/again - -echo; echo "== 4. what is in the local store: one Parquet file" -ls -l out/store.parquet -bx head -n 10 - -echo; echo "== 5. its series (bench_env_var has two: the -e setting changes the environment)" -bx series --quantity wall-time > out/series.txt -cat out/series.txt - -echo; echo "== 6. the history of bench_join: one point per run" -bx series --workload bench_join --quantity wall-time > out/join-series.txt -SERIES=$(awk 'NR == 1 {print $1}' out/join-series.txt) -[ -n "$SERIES" ] || { echo "no bench_join series in the store" >&2; exit 1; } -bx history "$SERIES" > out/history.txt -cat out/history.txt -[ "$(grep -c example- out/history.txt)" -eq 2 ] || { echo "expected two history points for $SERIES" >&2; exit 1; } - -echo; echo "== 7. what was written" -find out/all out/env out/again -type f | sort | sed 's/^/ /' From c028a4a45913dec9dc0be200167bf4955a04782a Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 4 Nov 2026 18:44:55 +0000 Subject: [PATCH 4/7] CI: run the matmul-params example in the e2e workflow A fourth job, matmul-params, runs the example from benchx_examples: it installs benchx, numpy and the CPU build of torch, and runs the quick version from the example's README (--processes 2 --values 2 --min-time 0.01). It uploads out/, which holds the result files and the store. The job's run steps were replayed locally against benchx_examples main (3a6b668): six cases, twelve results, all successful. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/e2e.yml | 72 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b243c29..ae82a40 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,6 +1,6 @@ name: e2e examples -# Runs examples from benchx-dev/benchx_examples and prints their output. Three jobs: +# Runs examples from benchx-dev/benchx_examples and prints their output. Four jobs: # # numpy-spin builds NumPy with spin and benchmarks it: # track.sh several revisions, each built and benchmarked @@ -10,6 +10,9 @@ name: e2e examples # examples/arrow/archery-local/run.sh # pyperf-suite runs a suite of Python functions with `bx bench`: # examples/pyperf-suite/run.sh +# matmul-params runs one parametrized benchmark, numpy and torch at three sizes, with +# `bx bench`; the quick version from the example's README: +# examples/matmul-params # # Nothing is checked. The job fails only if a step fails (a clone, a build, a script that # crashes); the scripts exit 0 even when a revision fails, so read the printed output. @@ -287,3 +290,70 @@ jobs: path: benchx_examples/examples/pyperf-suite/out if-no-files-found: warn retention-days: 14 + + matmul-params: + name: matmul-params (quick version) + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + # Keep the result store with the output, as the README does, not in the home directory. + BENCHX_HOME: ${{ github.workspace }}/benchx_examples/examples/matmul-params/out/home + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/prototype/uv.lock + + # The workers need pyperf, which the `test` extra brings. benchx is installed in + # editable mode, which is where it reads its schemas from. + - name: Install benchx + working-directory: benchx/prototype + run: | + uv python install 3.13 + uv sync --extra test --python 3.13 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + # The README's install lines. torch comes from PyTorch's CPU index: the default + # build is several GB of CUDA libraries that the runner has no use for. + - name: Install NumPy and PyTorch + run: | + uv pip install --python benchx/prototype/.venv/bin/python numpy + uv pip install --python benchx/prototype/.venv/bin/python torch --index-url https://download.pytorch.org/whl/cpu + + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "bx: $(command -v bx)" + echo "pyperf: $(python -c 'import pyperf; print(pyperf.__version__)')" + echo "numpy: $(python -c 'import numpy; print(numpy.__version__)')" + echo "torch: $(python -c 'import torch; print(torch.__version__)')" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + + # The quick version of the README: 2 processes x 2 values per case, 0.01 s of + # calibration. With pyperf's defaults it takes several minutes, mostly importing torch + # in every worker. `bx bench` prints one row per case. + - name: Run the quick version + working-directory: benchx_examples/examples/matmul-params + run: bx bench bench_matmul.py --processes 2 --values 2 --min-time 0.01 --out out + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: matmul-params-output + path: benchx_examples/examples/matmul-params/out + if-no-files-found: warn + retention-days: 14 From 482d9feec3cac8db7675987d963be28a09692cc7 Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Wed, 4 Nov 2026 18:45:03 +0000 Subject: [PATCH 5/7] Remove examples/matmul-params, which moved to benchx_examples The example lives in benchx-dev/benchx_examples now, and the e2e workflow runs it from there. The files in this directory were the same, except for the README's install lines. Co-Authored-By: Claude Sonnet 5.5 --- prototype/examples/matmul-params/.gitignore | 2 - prototype/examples/matmul-params/README.md | 74 ------------------- .../examples/matmul-params/bench_matmul.py | 29 -------- 3 files changed, 105 deletions(-) delete mode 100644 prototype/examples/matmul-params/.gitignore delete mode 100644 prototype/examples/matmul-params/README.md delete mode 100644 prototype/examples/matmul-params/bench_matmul.py diff --git a/prototype/examples/matmul-params/.gitignore b/prototype/examples/matmul-params/.gitignore deleted file mode 100644 index f1e3ac4..0000000 --- a/prototype/examples/matmul-params/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -out/ -results/ diff --git a/prototype/examples/matmul-params/README.md b/prototype/examples/matmul-params/README.md deleted file mode 100644 index 2ee0f77..0000000 --- a/prototype/examples/matmul-params/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Example: one benchmark, several parameter combinations - -`bench_matmul.py` times `a @ b` for square float64 arrays, for N = 10, 50, 100 and -for numpy and torch: one function, six cases. `bx bench` runs each combination as its own -case, under [pyperf](https://pyperf.readthedocs.io). - -## Run it - -From `prototype/`, in an environment of your choice: - -```sh -pip install -e . pyperf numpy torch -cd examples/matmul-params -export BENCHX_HOME=$PWD/out/home # optional: keep the result store here, not in ~ -bx bench bench_matmul.py --out out -``` - -This uses pyperf's defaults (20 worker processes x 3 values, 0.1 s calibration) and takes -several minutes, mostly importing torch in every worker. - -## A quick version - -```sh -bx bench bench_matmul.py --processes 2 --values 2 --min-time 0.01 --out out -``` - -About 15 seconds. The timings are rougher: each case has 4 values. - -## Output - -One row per combination, named by the function and its arguments: - -``` -case median mean ± sd ... -bench_matmul [n=10, xp='numpy'] 1.627 us ... -bench_matmul [n=10, xp='torch'] 2.387 us ... -... -bench_matmul [n=100, xp='torch'] 23.88 us ... -``` - -The result files are in `out/`, and `out/home/store.parquet` is the store. `bx` reports -twice as many results as rows: each case also records `load-average`. - -## Choosing cases - -```sh -bx bench bench_matmul.py bench_matmul # a function's name: all six -bx bench bench_matmul.py "bench_matmul(100, 'torch')" # one combination -bx bench bench_matmul.py --filter "'torch'" # a regex: the three torch cases -``` - -A regex is searched in `bench_matmul(10, 'torch')`, the function's name and the argument -values (not their names). `--filter 'bench_matmul$'` is therefore refused, since `$` leaves -no room for the arguments; the refusal says so. - -## How the parameters are declared - -```python -bench_matmul.params = [[10, 50, 100], ["numpy", "torch"]] # one list per argument -bench_matmul.param_names = ["n", "xp"] -``` - -Combinations are the product of the lists, in this order, and the function is called with -one of them. A value is recorded as its `repr`, so the library is given by its name and -imported on first use, not passed as a module. - -## What is timed - -The whole body of `bench_matmul` is timed: the call to `operands(n, xp)` as well as `a @ b`. -`operands` is cached, so the arrays are made once per worker process, on its first call (a -calibration or warmup call), not in every call. What every call still pays is the cache -lookup, about 60 ns here: roughly 4% of the numpy time at N=10 and under 0.2% at N=100. -The timings include it, for numpy and torch alike. The result array of `a @ b` is also -allocated inside the timed call. diff --git a/prototype/examples/matmul-params/bench_matmul.py b/prototype/examples/matmul-params/bench_matmul.py deleted file mode 100644 index 2e71eeb..0000000 --- a/prototype/examples/matmul-params/bench_matmul.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Example: one benchmark function, six cases (3 sizes x 2 array libraries). - - bx bench bench_matmul.py # every combination - bx bench bench_matmul.py bench_matmul # the same, by name - bx bench bench_matmul.py "bench_matmul(100, 'torch')" # one combination - -`params` holds one list of values per argument of the function and `param_names` names -them; each combination is its own case. Values are recorded as their repr, so the -library is given by name and imported when first used. Both libraries use float64 -(torch would default to float32). -""" - -import functools -import importlib - - -@functools.cache -def operands(n, xp): - lib = importlib.import_module(xp) - return lib.ones((n, n), dtype=lib.float64), lib.ones((n, n), dtype=lib.float64) - - -def bench_matmul(n, xp): - a, b = operands(n, xp) - a @ b - - -bench_matmul.params = [[10, 50, 100], ["numpy", "torch"]] -bench_matmul.param_names = ["n", "xp"] From 6e5544e4c1e05f5bdbc7fb034f103cf14a13af6b Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Thu, 5 Nov 2026 05:18:56 +0000 Subject: [PATCH 6/7] CI: run the numpy-venvs example in the e2e workflow A fifth job, numpy-venvs, runs the example from benchx_examples: it installs benchx in editable mode, from which run.sh installs benchx in the two venvs it makes (numpy 1.26.4 and 2.0.2, three rounds), then it runs run.sh. It uploads out/. The job's run steps were replayed locally against the example as it is on the numpy-venvs-example branch of benchx_examples: all steps passed. That branch is not on main there yet, and the job clones main, so the job fails until it is; a run on demand can name the branch (examples_ref). Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/e2e.yml | 58 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ae82a40..eead548 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,6 +1,6 @@ name: e2e examples -# Runs examples from benchx-dev/benchx_examples and prints their output. Four jobs: +# Runs examples from benchx-dev/benchx_examples and prints their output. Five jobs: # # numpy-spin builds NumPy with spin and benchmarks it: # track.sh several revisions, each built and benchmarked @@ -13,6 +13,9 @@ name: e2e examples # matmul-params runs one parametrized benchmark, numpy and torch at three sizes, with # `bx bench`; the quick version from the example's README: # examples/matmul-params +# numpy-venvs measures the same benchmarks under two numpy versions, each in its own venv, +# with a `bx bench` call per side and round, and compares them: +# examples/numpy-venvs/run.sh # # Nothing is checked. The job fails only if a step fails (a clone, a build, a script that # crashes); the scripts exit 0 even when a revision fails, so read the printed output. @@ -357,3 +360,56 @@ jobs: path: benchx_examples/examples/matmul-params/out if-no-files-found: warn retention-days: 14 + + numpy-venvs: + name: numpy-venvs (run.sh) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/prototype/uv.lock + + # The script makes two venvs of its own with uv (Python 3.12, the newest with numpy + # 1.26.4 wheels) and installs benchx in both, from the checkout that the benchx of the + # active environment runs from. So this environment needs benchx in editable mode, + # which is also where it reads its schemas from. The venvs bring their own pyperf. + - name: Install benchx + working-directory: benchx/prototype + run: | + uv python install 3.13 + uv sync --python 3.13 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "uv: $(uv --version)" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + + - name: Run the numpy-venvs example + working-directory: benchx_examples/examples/numpy-venvs + run: bash ./run.sh + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: numpy-venvs-output + path: benchx_examples/examples/numpy-venvs/out + if-no-files-found: warn + retention-days: 14 From 8854e134e4bf338ff5c0c473020f75364f65bd70 Mon Sep 17 00:00:00 2001 From: ev-br's Claude Code Agent Date: Thu, 5 Nov 2026 05:19:40 +0000 Subject: [PATCH 7/7] Remove examples/numpy-venvs, which moved to benchx_examples The example lives in benchx-dev/benchx_examples now, and the e2e workflow runs it from there, so the tests workflow no longer runs it, and the README and prototype-design.md point at the new place. The files there are the ones removed here, except that run.sh no longer assumes the benchx package is two levels up, the README says how to install benchx and links to UC-04 in this repository, and .gitignore also ignores __pycache__/. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/tests.yml | 9 -- docs/design/prototype-design.md | 4 +- prototype/README.md | 4 +- prototype/examples/numpy-venvs/.gitignore | 2 - prototype/examples/numpy-venvs/README.md | 102 ------------------ prototype/examples/numpy-venvs/bench_suite.py | 20 ---- .../examples/numpy-venvs/benchx.conf.json | 22 ---- .../examples/numpy-venvs/describe_env.py | 12 --- prototype/examples/numpy-venvs/provider.py | 45 -------- prototype/examples/numpy-venvs/run.sh | 55 ---------- 10 files changed, 4 insertions(+), 271 deletions(-) delete mode 100644 prototype/examples/numpy-venvs/.gitignore delete mode 100644 prototype/examples/numpy-venvs/README.md delete mode 100644 prototype/examples/numpy-venvs/bench_suite.py delete mode 100644 prototype/examples/numpy-venvs/benchx.conf.json delete mode 100644 prototype/examples/numpy-venvs/describe_env.py delete mode 100755 prototype/examples/numpy-venvs/provider.py delete mode 100755 prototype/examples/numpy-venvs/run.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1e64d68..ef67412 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,12 +49,3 @@ jobs: - name: Run tests run: uv run --extra test --python ${{ matrix.python-version }} python -m pytest tests -ra - - # Makes two venvs of its own with uv (Python 3.12, the newest with numpy 1.26.4 wheels), so - # the matrix Python only runs the outer script and one cell is enough. Needs network access. - - name: Run the numpy-venvs example - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' - shell: bash - run: >- - uv run --extra test --python ${{ matrix.python-version }} - bash examples/numpy-venvs/run.sh diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index 1a26a5e..1e29e9a 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -75,8 +75,8 @@ environment kind: a context collector (§3) prints facts about the environment i under `environment.system.`, a part of the varying coordinate; a setup file does the same for declared facts. A collector with `verify: how_built` and `on_fail: warn` checks the facts a target's `how_built` object declares against what it found, and the comparison lists the warning beside the -facts that differ. `prototype/examples/numpy-venvs` is the worked example, with the map from UC-04's -spellings to these. +facts that differ. `examples/numpy-venvs` in benchx_examples is the worked example, with the map from +UC-04's spellings to these. A bare path is read by the harness. For Google Benchmark it is `BUILD_DIR[:SOURCE_DIR]`, a `build` target. For pyperf it is a working tree, a diff --git a/prototype/README.md b/prototype/README.md index ec3dace..e4facca 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -95,7 +95,8 @@ Environments that cannot share a process, such as two venvs with different numpy measured by separate `bx bench` calls that share a `--run-key`; `--label NAME=VALUE`, `--round N` and `--slot K` place each call in the session, and `bx compare --run KEY --profile environments --label NAME --baseline VALUE` compares the sides ([UC-04](../docs/use-cases/UC-04-cross-env.md)). -`examples/numpy-venvs` is the worked example, with a shell script that drives it. +`examples/numpy-venvs` in [benchx_examples](https://github.com/benchx-dev/benchx_examples) +is the worked example, with a shell script that drives it. `bx bench PATH --adapter asv` runs an [asv](https://asv.readthedocs.io) benchmark suite unchanged, reading asv's benchmark format as a language and timing it with pyperf; @@ -220,7 +221,6 @@ benchx/ compare.py run mode over a profile's projection, comparison document (comparator.md) cli.py bx (compare has a read mode and a measure mode) ../schemas/ the result and work-order schemas, read in place ($BENCHX_SCHEMAS overrides) -examples/numpy-venvs UC-04 with two venvs and two numpy versions, driven by run.sh (own README) tests/ test_prototype.py: the success criteria and the runner and store rules, on a fake Google Benchmark binary; test_comparator.py: the comparator's invariants and method on hand-built results; diff --git a/prototype/examples/numpy-venvs/.gitignore b/prototype/examples/numpy-venvs/.gitignore deleted file mode 100644 index 388f608..0000000 --- a/prototype/examples/numpy-venvs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.venv-*/ -out/ diff --git a/prototype/examples/numpy-venvs/README.md b/prototype/examples/numpy-venvs/README.md deleted file mode 100644 index 2425bc7..0000000 --- a/prototype/examples/numpy-venvs/README.md +++ /dev/null @@ -1,102 +0,0 @@ -# numpy-venvs: UC-04 with two virtual environments - -The same code, measured under two numpy versions, by separate `bx bench` calls that alternate -round by round, then compared. This is -[UC-04](../../../docs/use-cases/UC-04-cross-env.md), "fixed branch, comparing environments": -`numpy==1.26.4` and `numpy==2.0.2` cannot be imported in one process, so each side runs in its -own environment, and a shell script drives the loop. - -You prepare the environments and enter them; benchx builds nothing and activates nothing. - -## Run it - -You need [`uv`](https://docs.astral.sh/uv/) (it fetches Python 3.12, the newest Python with numpy -1.26.4 wheels) and network access the first time. Then, from this directory: - -```console -$ ./run.sh -``` - -The versions (`NUMPY_A`, `NUMPY_B`), the Python and the number of rounds are set at the top of -`run.sh`. Everything it creates is in `.venv-a`, `.venv-b` and `out/`, all git-ignored. - -## What it does - -1. **Prepares two venvs**, `.venv-a` and `.venv-b`, each with its numpy, pyperf and benchx, and - `pyarrow<25`: benchx imports pyarrow, and pyarrow 25 refuses to import under numpy 1.x. -2. **Runs the caller's loop.** For each of the 3 rounds, one `bx bench` call per side, each in its - venv (`run.sh` sources the venv's `activate`; activating the other venv replaces it, so nothing - needs deactivating). The calls share a `--run-key`; each says which side it is - (`--label env=numpy-1.26.4`), which round, and which slot of the alternation. -3. **Compares.** `bx compare --run S --profile environments --label env --baseline numpy-1.26.4`. - The sides differ because the collector read a different numpy in each. The comparison says - what differed (`differs: ...deps.numpy`), shows what each side declared, and gives a verdict per - benchmark: - - ``` - run numpy-1793121294 profile environments baseline numpy-2.3.4 contender numpy-2.4.3 - numpy-2.3.4: how built: {"summary":{"numpy":"2.3.4"}} - numpy-2.4.3: how built: {"summary":{"numpy":"2.4.3"}} - differs: coordinates.environment.system.deps.numpy, coordinates.environment.system.deps.sha256 - - NO CHANGE DETECTED (3) - bench_matmul wall-time effect -16.28% noise 7.43% rounds=3 - ... - ``` - - (The numbers vary from run to run; `[local-only]` appears while the demo's files are - uncommitted.) -4. **Checks a declaration.** It declares numpy 2.0.2 for a run in the venv that has 1.26.4. The - collector disagrees, so every result of that run carries `collector-mismatch`. The run goes - ahead, because the collector has `on_fail: warn`, and the recorded numpy is the one found. - -## How the sides are told apart - -- `describe_env.py` is a **context collector** (`harness-adapter.md` §5.1), named in - `benchx.conf.json`. benchx runs it once per call, in the environment the call runs in. It prints - `{"summary": {"numpy": ""}}`, and that lands in each result's `environment.system.deps`, - which the `environments` comparison treats as the varying coordinate. benchx knows nothing about - venvs, conda or pixi: any command that prints facts about an environment does the same. -- `provider.py` is the **target provider**. For configuration `a` or `b` it declares, in - `how_built`, the numpy that side is meant to have. The collector has `verify: how_built`, so it - compares the declaration with what it found. The declared object has the same shape as the - collector's output, because the check compares key by key from the top. -- A check that fails refuses the run by default; `on_fail: warn` records a warning instead, which - is what the design asks for (a mismatch is recorded, not a gate). - -## UC-04's spellings, and the prototype's - -[UC-04](../../../docs/use-cases/UC-04-cross-env.md) is kept as written, and uses spellings the -prototype does not have. This is how each corresponds: - -- `benchx new-session` (`SESSION=$(...)`) → no command. The session is the run key, chosen by the - caller: `S=numpy-$(date +%s)`. -- `--session-id "$SESSION"` → `--run-key "$S"`. It lands in `provenance.run_key`. -- `benchx run run_benchmark.py` → `bx bench`. With a module and functions, - `bx bench MODULE.py [FUNCTION ...]`; with a prepared target, `bx bench @CONFIG --suite NAME`. - `bx run` takes an order file, not a script. -- `--env-label numpy-1.26.4` → `--label env=numpy-1.26.4`. The field `env_label` is - `provenance.labels.env`. -- `--round "$round"` → `--round N` and `--slot K`. UC-04 uses one number for two things. The - prototype separates the round, which pairs one attempt of each side (`procedure.round`), from - the slot, the realized position in the interleaved sequence (`procedure.slot`). UC-04's - `interleave_position` is the slot. The caller's loop keeps the slot counter. -- `benchx compare-environments --session-id S --baseline A --variant B` → - `bx compare --run S --profile environments --label env --baseline A`. The profile is UC-04's - `cross-environment-comparison`. There are exactly two sides, so the variant is the other one. -- `.venv-baseline/bin/benchx` → the `bx` of that environment, such as `.venv-a/bin/bx`. -- The output table (`benchmark_name`, `env`, `time`, `vs. baseline`) → the comparison output: - verdict groups with the effect, the noise and the rounds, and the facts that differ. -- UC-04's `session_id`, `env_label` and `interleave_position` fields → `provenance.run_key`, - `provenance.labels.env` and `procedure.slot`. The schema adds no fields for a profile - (`benchmark-result-schema.md` §5.5). - -## What is in this directory - -| File | Role | -|---|---| -| `run.sh` | the driver: venvs, the loop, the comparison, the check | -| `benchx.conf.json` | the project file: the adapter and suite, the provider, the `deps` collector | -| `provider.py` | the target provider: declares each side's numpy in `how_built` (`run.sh` exports the versions) | -| `describe_env.py` | the collector: prints the numpy this Python imports | -| `bench_suite.py` | three small numpy benchmarks | diff --git a/prototype/examples/numpy-venvs/bench_suite.py b/prototype/examples/numpy-venvs/bench_suite.py deleted file mode 100644 index 866f15a..0000000 --- a/prototype/examples/numpy-venvs/bench_suite.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Three small numpy benchmarks. pyperf calls each bench_* function many times and times the -calls, so the arrays are made once, at module level.""" - -import numpy as np - -_rng = np.random.default_rng(0) -_square = _rng.standard_normal((200, 200)) -_vector = _rng.standard_normal(100_000) - - -def bench_matmul(): - _square @ _square - - -def bench_sort(): - np.sort(_vector) - - -def bench_sum(): - _vector.sum() diff --git a/prototype/examples/numpy-venvs/benchx.conf.json b/prototype/examples/numpy-venvs/benchx.conf.json deleted file mode 100644 index 94146f8..0000000 --- a/prototype/examples/numpy-venvs/benchx.conf.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "schema_version": "benchx/config/0.1.0", - "project": "numpy-venvs", - "adapter": { - "name": "pyperf", - "suite": "bench_suite.py", - "precision": { - "repetitions": { "mode": "fixed", "levels": [{ "unit": "process", "n": 3 }, { "unit": "value", "n": 3 }] }, - "calibration": { "mode": "adaptive", "minimum_sample_seconds": 0.02 }, - "warmup": { "mode": "count", "n_warmup": 1 } - } - }, - "provider": { "command": ["./provider.py"] }, - "collectors": { - "deps": { - "command": ["python", "{root}/describe_env.py"], - "verify": "how_built", - "on_fail": "warn" - } - }, - "paths": { "store": "out/store.parquet", "results": "out" } -} diff --git a/prototype/examples/numpy-venvs/describe_env.py b/prototype/examples/numpy-venvs/describe_env.py deleted file mode 100644 index 7363bbb..0000000 --- a/prototype/examples/numpy-venvs/describe_env.py +++ /dev/null @@ -1,12 +0,0 @@ -"""The collector: print what the numpy that this Python imports says about itself. - -benchx.conf.json runs it once per call, in the environment the call runs in, so it can only -differ between the two sides if their environments do. It prints the same text for the same -environment: no paths, no timestamps. -""" - -import json - -import numpy - -print(json.dumps({"summary": {"numpy": numpy.__version__}}, sort_keys=True)) diff --git a/prototype/examples/numpy-venvs/provider.py b/prototype/examples/numpy-venvs/provider.py deleted file mode 100755 index 2fdbd5f..0000000 --- a/prototype/examples/numpy-venvs/provider.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -"""The target provider of this example (system-decomposition.md §3.4): one JSON request on stdin, -one target description on stdout. It builds nothing, so `describe` and `prepare` answer alike. - -Configuration `a` or `b` names a side. The description declares, in how_built, the numpy the -user means that side to have, in the shape describe_env.py prints. The collector in -benchx.conf.json compares the two: a side whose environment is not the one it was declared -with gets the warning collector-mismatch on every result. -""" - -import json -import os -import sys -from pathlib import Path - -HERE = Path(__file__).resolve().parent -# The canonical URI this working tree's results are filed under, as in archery-local. -URI = "https://example.org/numpy-venvs/demo.git" - - -def main(): - versions = {"a": os.environ.get("NUMPY_A"), "b": os.environ.get("NUMPY_B")} - if not all(versions.values()): - sys.exit("provider: run.sh sets NUMPY_A and NUMPY_B") - - # An absent or null configuration is refused below, like an unknown one. - configuration = json.load(sys.stdin).get("configuration") - if configuration not in versions: - print(f"provider: unknown configuration {configuration!r} (a or b)", file=sys.stderr) - sys.exit(2) - - json.dump({ - "schema_version": "benchx/target-description/0.1.0", - "target": { - "kind": "working_tree", - "path": str(HERE), - "source": {"uri": URI, "type": "git"}, - }, - "how_built": {"summary": {"numpy": versions[configuration]}}, - "provider": {"name": "provider.py", "version": "1"}, - }, sys.stdout, indent=1) - print() - - -main() diff --git a/prototype/examples/numpy-venvs/run.sh b/prototype/examples/numpy-venvs/run.sh deleted file mode 100755 index 7b67bf2..0000000 --- a/prototype/examples/numpy-venvs/run.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# UC-04 with two venvs: the same code under two numpy versions, measured by separate `bx bench` -# calls that alternate round by round, then compared. See README.md for what each step shows. -# -# Needs `uv`. Everything it creates is here: .venv-a, .venv-b and out/ (git-ignored). -# benchx builds nothing. -set -euo pipefail -cd "$(dirname "$0")" - -# provider.py reads the numpy versions from the environment. numpy 1.26.4 has no wheels beyond -# Python 3.12, so both venvs use it. Three rounds are the fewest that give a verdict. -export NUMPY_A=1.26.4 NUMPY_B=2.0.2 -PYTHON_VERSION=3.12 -ROUNDS=3 -SIDES=("a $NUMPY_A" "b $NUMPY_B") -RUN_KEY=numpy-$(date +%s) -SLOT=0 - -rm -rf out .venv-a .venv-b -mkdir -p out - -echo "== 1. the user prepares both environments; benchx builds nothing" -for SIDE_VERSION in "${SIDES[@]}"; do - read -r SIDE VERSION <<<"$SIDE_VERSION" - uv venv -q --python "$PYTHON_VERSION" ".venv-$SIDE" - # benchx imports pyarrow, and pyarrow 25 refuses to import under numpy 1.x. - uv pip install -q --python ".venv-$SIDE/bin/python" "numpy==$VERSION" "pyarrow<25" pyperf -e ../.. - echo " .venv-$SIDE: numpy $VERSION" -done - -echo "== 2. the caller's loop: alternate the sides, round by round, one slot per call" -for ROUND in $(seq 0 $((ROUNDS - 1))); do - for SIDE_VERSION in "${SIDES[@]}"; do - read -r SIDE VERSION <<<"$SIDE_VERSION" - # Activating a venv replaces the one activated before; no deactivate is needed. - source ".venv-$SIDE/bin/activate" - bx bench "@$SIDE" --run-key "$RUN_KEY" --label "env=numpy-$VERSION" \ - --round "$ROUND" --slot "$SLOT" --out "out/$RUN_KEY" > /dev/null - echo " round $ROUND, slot $SLOT: side $SIDE" - SLOT=$((SLOT + 1)) - done -done - -echo "== 3. compare: the sides differ in the numpy the collector read" -bx compare --run "$RUN_KEY" --profile environments --label env \ - --baseline "numpy-$NUMPY_A" | tee out/compare.txt -grep -q 'differs:.*deps\.numpy' out/compare.txt # the differing fact is shown -if grep -q 'failed invariants' out/compare.txt; then echo "the comparison was refused" >&2; exit 1; fi - -echo "== 4. the check: declare numpy $NUMPY_B for a run in the venv that has $NUMPY_A" -source .venv-a/bin/activate -bx bench @b --run-key "$RUN_KEY-mismatch" --label env=wrong \ - --round 0 --slot 0 --out "out/$RUN_KEY-mismatch" > /dev/null -grep -rq collector-mismatch "out/$RUN_KEY-mismatch" -echo " the results carry collector-mismatch; the run went ahead, with the numpy it found"