diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..d467dc8 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,415 @@ +name: e2e examples + +# Runs examples from benchx-dev/benchx_examples and prints their output. Five jobs: +# +# numpy-spin builds NumPy with spin and benchmarks it: +# track.sh several revisions, each built and benchmarked +# num_threads.sh one prebuilt revision at several OPENBLAS_NUM_THREADS +# arrow builds a small Google Benchmark suite with CMake and compares the builds: +# examples/arrow/demo/run.sh +# examples/arrow/archery-local/run.sh +# pyperf-suite runs a suite of Python functions with `bx bench`: +# examples/pyperf-suite/run.sh +# matmul-params runs one parametrized benchmark, numpy and torch at three sizes, with +# `bx bench`; the quick version from the example's README: +# examples/matmul-params +# numpy-venvs measures the same benchmarks under two numpy versions, each in its own venv, +# with a `bx bench` call per side and round, and compares them: +# examples/numpy-venvs/run.sh +# +# Nothing is checked. The job fails only if a step fails (a clone, a build, a script that +# crashes); the scripts exit 0 even when a revision fails, so read the printed output. +# +# Runs on every pull request, and on demand. A run on demand can take another +# benchx_examples branch, tag or commit (examples_ref), which a pull request here cannot: +# the examples live in another repository. +# +# In the numpy-spin job the benchmarks are pinned to two cores (BENCH_CPUS). num_threads.sh +# should use 1, 2 and 4 threads, so that the last of them oversubscribes the two cores. The +# NumPy builds are not pinned. +# +on: + workflow_dispatch: + inputs: + examples_ref: + description: "benchx_examples branch, tag or commit to run" + default: main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + numpy-spin: + name: numpy-spin (track.sh, num_threads.sh) + runs-on: ubuntu-latest + timeout-minutes: 150 + env: + NUMPY: ${{ github.workspace }}/numpy + EXAMPLE: ${{ github.workspace }}/benchx_examples/examples/numpy-spin + BENCH_CPUS: "0,1" + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_MAXSIZE: 500M + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + # The repository keeps the output of earlier runs next to the scripts. + # Start clean instead + - name: Remove the output from examples/numpy-spin + working-directory: ${{ env.EXAMPLE }} + run: rm -rf results store.parquet + + - name: Install system packages + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libopenblas-dev pkg-config ccache + + # The key is new for every run, because a cache cannot be overwritten; a run starts + # from the most recent one. It is saved at the end, also when a later step fails. + - name: Restore the ccache cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ccache-${{ runner.os }}- + + - name: Zero the ccache statistics + run: ccache --zero-stats + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/benchx/uv.lock + + # The scripts run plain `python`, `bx` and `spin`, and hard-code + # lib/python3.12/site-packages, so this is Python 3.12 on the PATH. The `test` + # extra is only for pyperf, which the benchmarks need. + - name: Install benchx on Python 3.12 + working-directory: benchx/benchx + run: | + uv python install 3.12 + uv sync --extra test --python 3.12 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + - name: Install NumPy's build tools + run: uv pip install --python benchx/benchx/.venv/bin/python spin ninja meson-python cython + + # The full history and the tags: track.sh checks out v2.4.3 and v2.4.0. + - name: Clone NumPy + run: | + git clone --quiet https://github.com/numpy/numpy.git "$NUMPY" + git -C "$NUMPY" submodule update --init + + # Two cores for the benchmarks, all of them for the builds, so only `bx` is pinned: + # a `bx` ahead of the real one on the PATH, which the scripts call without knowing. + - name: Pin the benchmarks to two cores + run: | + mkdir -p "$RUNNER_TEMP/pin" + cat > "$RUNNER_TEMP/pin/bx" <> "$GITHUB_PATH" + "$RUNNER_TEMP/pin/bx" --help > /dev/null + + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "bx: $(command -v bx)" + echo "ccache: $(ccache --version | head -1)" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + echo "numpy main: $(git -C "$NUMPY" rev-parse --short main)" + + # num_threads.sh assumes a prebuilt NumPy at main, checked out. + - name: Build NumPy main + working-directory: ${{ env.NUMPY }} + run: spin build + + # Before track.sh, which leaves the NumPy checkout at the last revision it built. + - name: Run num_threads.sh + working-directory: ${{ env.EXAMPLE }} + run: bash ./num_threads.sh + + - name: Print the thread sweep results + working-directory: ${{ env.EXAMPLE }} + run: bx head --table -n 100 + + - name: Run track.sh + working-directory: ${{ env.EXAMPLE }} + run: bash ./track.sh + + - name: Print the results across history + working-directory: ${{ env.EXAMPLE }} + run: bx head --table -n 100 + + - name: Show the ccache statistics + if: always() + run: ccache --show-stats + + - name: Save the ccache cache + if: always() + uses: actions/cache/save@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: numpy-spin-output + path: | + benchx_examples/examples/numpy-spin/results + benchx_examples/examples/numpy-spin/store.parquet + if-no-files-found: warn + retention-days: 14 + + arrow: + name: arrow (demo, archery-local) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/benchx/uv.lock + + # The scripts run plain `python` and `bx`. benchx is installed in editable mode, + # which is where it reads its schemas from. + - name: Install benchx + working-directory: benchx/benchx + run: | + uv python install 3.13 + uv sync --extra test --python 3.13 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + # cmake and a C++ compiler are preinstalled on the runner; CMake fetches + # Google Benchmark. + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "bx: $(command -v bx)" + echo "cmake: $(cmake --version | head -1)" + echo "c++: $(c++ --version | head -1)" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + + - name: Run the demo + working-directory: benchx_examples/examples/arrow/demo + run: bash ./run.sh + + - name: Run archery-local + working-directory: benchx_examples/examples/arrow/archery-local + run: bash ./run.sh + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: arrow-output + path: | + benchx_examples/examples/arrow/demo/store.parquet + benchx_examples/examples/arrow/archery-local/_work/results + if-no-files-found: warn + retention-days: 14 + + pyperf-suite: + name: pyperf-suite (run.sh) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/benchx/uv.lock + + # The script runs plain `python` and `bx`, and the workers need pyperf, which the + # `test` extra brings. benchx is installed in editable mode, which is where it reads + # its schemas from. + - name: Install benchx + working-directory: benchx/benchx + run: | + uv python install 3.13 + uv sync --extra test --python 3.13 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "bx: $(command -v bx)" + echo "pyperf: $(python -c 'import pyperf; print(pyperf.__version__)')" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + + - name: Run the pyperf suite + working-directory: benchx_examples/examples/pyperf-suite + run: bash ./run.sh + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: pyperf-suite-output + path: benchx_examples/examples/pyperf-suite/out + if-no-files-found: warn + retention-days: 14 + + matmul-params: + name: matmul-params (quick version) + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + # Keep the result store with the output, as the README does, not in the home directory. + BENCHX_HOME: ${{ github.workspace }}/benchx_examples/examples/matmul-params/out/home + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/benchx/uv.lock + + # The workers need pyperf, which the `test` extra brings. benchx is installed in + # editable mode, which is where it reads its schemas from. + - name: Install benchx + working-directory: benchx/benchx + run: | + uv python install 3.13 + uv sync --extra test --python 3.13 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + # The README's install lines. torch comes from PyTorch's CPU index: the default + # build is several GB of CUDA libraries that the runner has no use for. + - name: Install NumPy and PyTorch + run: | + uv pip install --python benchx/benchx/.venv/bin/python numpy + uv pip install --python benchx/benchx/.venv/bin/python torch --index-url https://download.pytorch.org/whl/cpu + + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "bx: $(command -v bx)" + echo "pyperf: $(python -c 'import pyperf; print(pyperf.__version__)')" + echo "numpy: $(python -c 'import numpy; print(numpy.__version__)')" + echo "torch: $(python -c 'import torch; print(torch.__version__)')" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + + # The quick version of the README: 2 processes x 2 values per case, 0.01 s of + # calibration. With pyperf's defaults it takes several minutes, mostly importing torch + # in every worker. `bx bench` prints one row per case. + - name: Run the quick version + working-directory: benchx_examples/examples/matmul-params + run: bx bench bench_matmul.py --processes 2 --values 2 --min-time 0.01 --out out + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: matmul-params-output + path: benchx_examples/examples/matmul-params/out + if-no-files-found: warn + retention-days: 14 + + numpy-venvs: + name: numpy-venvs (run.sh) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Check out benchx + uses: actions/checkout@v4 + with: + path: benchx + + - name: Clone benchx_examples + uses: actions/checkout@v4 + with: + repository: benchx-dev/benchx_examples + ref: ${{ inputs.examples_ref || 'main' }} + path: benchx_examples + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: benchx/benchx/uv.lock + + # The script makes two venvs of its own with uv (Python 3.12, the newest with numpy + # 1.26.4 wheels) and installs benchx in both, from the checkout that the benchx of the + # active environment runs from. So this environment needs benchx in editable mode, + # which is also where it reads its schemas from. The venvs bring their own pyperf. + - name: Install benchx + working-directory: benchx/benchx + run: | + uv python install 3.13 + uv sync --python 3.13 + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + + - name: Show what is under test + run: | + echo "python: $(python --version) ($(command -v python))" + echo "uv: $(uv --version)" + echo "benchx: $(git -C benchx rev-parse --short HEAD)" + echo "benchx_examples: $(git -C benchx_examples rev-parse --short HEAD)" + + - name: Run the numpy-venvs example + working-directory: benchx_examples/examples/numpy-venvs + run: bash ./run.sh + + - name: Upload the output + if: always() + uses: actions/upload-artifact@v4 + with: + name: numpy-venvs-output + path: benchx_examples/examples/numpy-venvs/out + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e6a35d3..b00fc31 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,24 +49,3 @@ jobs: - name: Run tests run: uv run --extra test --python ${{ matrix.python-version }} python -m pytest tests -ra - - - name: Run the pyperf-bench example - shell: bash - run: uv run --extra test --python ${{ matrix.python-version }} bash examples/pyperf-bench/run.sh - - # Needs cmake and a C++ compiler (preinstalled on the Linux and macOS - # runners) and fetches Google Benchmark; the example is bash and CMake - # with Unix paths, so it is not run on Windows. - - name: Run the archery-local example - if: runner.os != 'Windows' - shell: bash - run: uv run --extra test --python ${{ matrix.python-version }} bash examples/archery-local/run.sh - - # Makes two venvs of its own with uv (Python 3.12, the newest with numpy 1.26.4 wheels), so - # the matrix Python only runs the outer script and one cell is enough. Needs network access. - - name: Run the numpy-venvs example - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' - shell: bash - run: >- - uv run --extra test --python ${{ matrix.python-version }} - bash examples/numpy-venvs/run.sh diff --git a/benchx/README.md b/benchx/README.md index 0d8ddf7..161bf2f 100644 --- a/benchx/README.md +++ b/benchx/README.md @@ -65,7 +65,8 @@ declare `how_built` and `activation` text, which are copied into the work order and into every result's `provenance.info`, shown in the comparison, and never executed or checked: you activate the environment yourself, and the runner records the allowlisted variables it inherited beside your declaration. -`examples/archery-local` is a complete walk-through. +`examples/arrow/archery-local` in [benchx_examples](https://github.com/benchx-dev/benchx_examples) +is a complete walk-through. `bx bench` is the single-target form of `bx compare`: the same way of naming a target (`@CONFIG[=SOURCE_REF]`, or a path), the same options, the same code; it writes one @@ -87,13 +88,15 @@ pyperf. functions: the module's git checkout is the target, its path the suite, and the functions are exact case names. `bx run` is the layer underneath: it executes a finished order, so `bx run results/KEY/orders/order.json` replays any bench run. -`examples/pyperf-bench` is a runnable walk-through. +`examples/pyperf-suite` in [benchx_examples](https://github.com/benchx-dev/benchx_examples) +is a runnable walk-through. Environments that cannot share a process, such as two venvs with different numpy versions, are measured by separate `bx bench` calls that share a `--run-key`; `--label NAME=VALUE`, `--round N` and `--slot K` place each call in the session, and `bx compare --run KEY --profile environments --label NAME --baseline VALUE` compares the sides ([UC-04](../docs/use-cases/UC-04-cross-env.md)). -`examples/numpy-venvs` is the worked example, with a shell script that drives it. +`examples/numpy-venvs` in [benchx_examples](https://github.com/benchx-dev/benchx_examples) +is the worked example, with a shell script that drives it. `bx bench PATH --adapter asv` runs an [asv](https://asv.readthedocs.io) benchmark suite unchanged, reading asv's benchmark format as a language and timing it with pyperf; @@ -181,39 +184,14 @@ every result is pyperf. Results also record the installed project's version ```console $ uv venv && uv pip install -e '.[test]' # editable: schemas are read from ../schemas $ .venv/bin/python -m pytest tests # 344 tests, fake Google Benchmark binary, no compiler -$ ./demo.sh # real builds; needs cmake, a C++ compiler, network once ``` -`demo.sh` prints every command before running it, as you would type it -(`bx` for `.venv/bin/python -m benchx.cli`, `$WORK` for its scratch -directory), so it doubles as a walkthrough. It prepares a git source with two -commits and three CMake build directories of `examples/demo-suite` (base, -head, and head with `DEMO_HARDENED=ON`), shows the first work order of each -part, then: - -1. **revisions, ad hoc:** `bx compare` in measure mode alternates base and head - over five rounds with orders that name no project, then read mode compares - the same run key through a throwaway store; -2. **environments, tracked:** `bx compare` alternates plain and hardened, labeled - `build: plain|hardened`, compares from the local store, re-ingests - (no-op), and rejects a unit conflict; -3. checks that the throwaway store gives the same comparison document as the - persistent one, lists series and history, and ends with `bx head`. - -Every `bx run` and every measure-mode `bx compare` delivers to the local store, so each demo run adds its -results, ad hoc and tracked, to `examples/demo/store.parquet` (the demo's project root, set -in its `benchx.conf.json`). Run keys carry a -timestamp, so runs accumulate and each series' history grows. Look at it any -time: - -```console -$ (cd examples/demo && ../../.venv/bin/python -m benchx.cli head -n 20) -$ rm examples/demo/store.parquet # start over -``` - -Both changes add two to three percent of work. The quiet benchmark flags them -as regressed; the noisy one, with about ten percent spread between -repetitions, does not. +The end-to-end examples that build real software are in +[benchx_examples](https://github.com/benchx-dev/benchx_examples), and the `e2e` +workflow runs them. `examples/arrow/demo/run.sh` there is a walkthrough of both comparison +profiles on a small Google Benchmark suite, with real CMake builds (it needs cmake, a +C++ compiler, and network access once). It prints every command before running it, +as you would type it, so it doubles as a walkthrough. ## Layout @@ -243,11 +221,6 @@ benchx/ compare.py run mode over a profile's projection, comparison document (comparator.md) cli.py bx (compare has a read mode and a measure mode) ../schemas/ the result and work-order schemas, read in place ($BENCHX_SCHEMAS overrides) -examples/demo-suite a two-benchmark Google Benchmark suite for the demo -examples/demo the demo's project root: its benchx.conf.json, and its persistent store -examples/pyperf-bench Python functions under `bx bench` (own README) -examples/archery-local an end-to-end, Arrow-local-shaped example with a target provider (own README) -examples/numpy-venvs UC-04 with two venvs and two numpy versions, driven by run.sh (own README) tests/ test_prototype.py: the success criteria and the runner and store rules, on a fake Google Benchmark binary; test_comparator.py: the comparator's invariants and method on hand-built results; diff --git a/benchx/demo.sh b/benchx/demo.sh deleted file mode 100644 index ed46383..0000000 --- a/benchx/demo.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env bash -# End-to-end demonstration of prototype-design.md §6: both profiles, both shapes. -# -# The script plays the user, and in that role the target provider: it prepares -# the environment with its own tools, a git repository with two commits and -# three CMake build directories (benchx builds nothing). Then `bx compare` in -# measure mode does the workbench's part (system-decomposition.md §3.4): it -# alternates the sides over the rounds, assigns round and slot, delivers every -# result to the local store, and compares. -# -# Every command is printed before it runs, as you would type it: `bx` stands -# for `$PYTHON -m benchx.cli`, and $WORK for the scratch directory. -# -# Needs cmake, a C++ compiler, and network access the first time, to fetch -# Google Benchmark unless it is installed. ROUNDS=5 by default. -# -# `bx compare` (measure mode) and `bx run` deliver every result to the one local store, -# examples/demo/store.parquet (paths.store in examples/demo/benchx.conf.json, the demo's -# project file, from whose directory every `bx` command runs), so the store and -# each series' history grow from run to run. `bx head` shows the latest rows -# at any time. Delete the file to start over. -set -euo pipefail -cd "$(dirname "$0")" -PROTOTYPE=$PWD -PYTHON=${PYTHON:-$PROTOTYPE/.venv/bin/python} -ROUNDS=${ROUNDS:-5} -WORK=$(mktemp -d) -DEMO_ROOT=$PROTOTYPE/examples/demo -STORE=$DEMO_ROOT/store.parquet -STAMP=$(date -u +%Y%m%dT%H%M%SZ) # run keys are unique per demo run, so runs accumulate -RUN_REV="rev-adhoc-$STAMP" -RUN_ENV="env-tracked-$STAMP" -export GIT_AUTHOR_NAME=demo GIT_AUTHOR_EMAIL=demo@example.org -export GIT_COMMITTER_NAME=demo GIT_COMMITTER_EMAIL=demo@example.org - -step() { printf '\n\033[1m== %s\033[0m\n' "$*"; } - -# say : print a command as you would type it -quote() { - if [[ $1 =~ ^[A-Za-z0-9_./:=@%+,^-]+$ ]]; then printf '%s' "$1" - else printf "'%s'" "${1//\'/\'\\\'\'}"; fi -} -SAY_SUFFIX="" -say() { # to stderr, so a command's own redirected output never contains it - local line="" arg - for arg in "$@"; do line+=" $(quote "$arg")"; done - line="${line# }${SAY_SUFFIX:+ $SAY_SUFFIX}" - printf '\033[36m$ %s\033[0m\n' "${line//"$WORK"/\$WORK}" >&2 -} -show() { say "$@"; "$@"; } # print, then run -quiet() { say "$@"; "$@" >/dev/null; } # print, then run without its output -bx() { say bx "$@"; (cd "$DEMO_ROOT" && "$PYTHON" -m benchx.cli "$@"); } -into() { # into : run with stdout to a file, printed with its redirect - SAY_SUFFIX="> $1"; local out=$1; shift - "$@" > "$out" - SAY_SUFFIX="" -} - -printf '\033[36m$ WORK=%s\033[0m\n' "$WORK" >&2 - -step "user: a source with two commits, a worktree at each" -show cp -R examples/demo-suite "$WORK/src" -show git -C "$WORK/src" init -q -show git -C "$WORK/src" add -A -show git -C "$WORK/src" commit -qm "base" -BASE=$(git -C "$WORK/src" rev-parse HEAD) -show sed -i.bak 's/kWork = 100/kWork = 102/' "$WORK/src/bench.cpp" -rm "$WORK/src/bench.cpp.bak" -show git -C "$WORK/src" commit -qam "head: 2% more work" -HEAD=$(git -C "$WORK/src" rev-parse HEAD) -show git -C "$WORK/src" worktree add -q --detach "$WORK/wt-base" "$BASE" -show git -C "$WORK/src" worktree add -q --detach "$WORK/wt-head" "$HEAD" -echo "base $BASE" -echo "head $HEAD" - -step "user: three build directories (base, head, head hardened)" -build() { # build [cmake args] - quiet cmake -S "$1" -B "$2" -DCMAKE_BUILD_TYPE=Release -DFETCHCONTENT_BASE_DIR="$WORK/_deps" "${@:3}" - quiet cmake --build "$2" -j -} -build "$WORK/wt-base" "$WORK/build-base" -build "$WORK/wt-head" "$WORK/build-head" -build "$WORK/wt-head" "$WORK/build-hardened" -DDEMO_HARDENED=ON - -step "revisions, ad hoc: no project, measured and compared in one command" -echo "# measure mode: the sides alternate over $ROUNDS rounds, one work order each" -bx compare "$WORK/build-base:$WORK/wt-base" "$WORK/build-head:$WORK/wt-head" \ - --rounds "$ROUNDS" --run-key "$RUN_REV" --out "$WORK/results-rev" # adapter, suite and profile: benchx.conf.json -echo "# the first work order it wrote:" -show cat "$WORK/results-rev/orders/order-0000.json" -echo -echo "# read mode: the same run key, compared again through a throwaway store" -bx compare --run "$RUN_REV" --baseline "${BASE:0:10}" --results "$WORK/results-rev" -echo "# bx compare also delivered these to the local store, as thin results of project local/" - -step "environments, tracked: project demo, into the local store" -bx compare "$WORK/build-head:$WORK/wt-head" "$WORK/build-hardened:$WORK/wt-head" --profile environments \ - --label build=plain,hardened --rounds "$ROUNDS" --project demo \ - --run-key "$RUN_ENV" --out "$WORK/results-env" - -step "re-ingest is a visible no-op" -bx ingest "$WORK/results-env" - -step "a conflicting unit is rejected with its taxonomy code" -echo "# \$WORK/bad.json: a copy of one result, its unit changed from s to ms" -"$PYTHON" - "$WORK" <<'EOF' -import json, pathlib, sys -work = pathlib.Path(sys.argv[1]) -doc = json.loads(next(p for p in sorted((work / "results-env").glob("*_wall-time_*.json"))).read_text()) -doc["ingest_key"] += ":in-ms" -doc["measurement"]["quantity"]["unit"] = "ms" -(work / "bad.json").write_text(json.dumps(doc)) -EOF -bx ingest "$WORK/bad.json" || true - -step "compare from the local store" -bx compare --run "$RUN_ENV" --profile environments --label build --baseline plain - -step "the same comparison through a throwaway store gives the same document" -into "$WORK/a.json" bx compare --run "$RUN_ENV" --profile environments --label build --baseline plain --json -into "$WORK/b.json" bx compare --run "$RUN_ENV" --profile environments --label build --baseline plain --json \ - --results "$WORK/results-env" -show cmp "$WORK/a.json" "$WORK/b.json" && echo "identical" - -step "series and history (a query; history mode is deferred)" -into "$WORK/series.txt" bx series --workload BM_Quiet --quantity wall-time -show head -6 "$WORK/series.txt" -SERIES=$(awk '/demo .*est=benchx\/median/ {print $1; exit}' "$WORK/series.txt") -into "$WORK/history.txt" bx history "$SERIES" -show head -1 "$WORK/history.txt" -show tail -n 4 "$WORK/history.txt" - -step "the store is one Parquet file, and keeps growing" -show ls -l "$STORE" -bx head -n 8 -printf '\ndemo artifacts in %s\n' "$WORK" diff --git a/benchx/examples/archery-local/.gitignore b/benchx/examples/archery-local/.gitignore deleted file mode 100644 index 6acf7e7..0000000 --- a/benchx/examples/archery-local/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -_work/ -results/ diff --git a/benchx/examples/archery-local/README.md b/benchx/examples/archery-local/README.md deleted file mode 100644 index d6f7546..0000000 --- a/benchx/examples/archery-local/README.md +++ /dev/null @@ -1,141 +0,0 @@ -# archery-local: an end-to-end example - -One commit, built two ways (plain and hardened), compared with noise-aware -verdicts, where the comparison says what was built and how. It follows the -Arrow local story, "Measure what a build option costs" -([`docs/user_stories/apache-arrow-local-benchmarking.md`](../../../docs/user_stories/apache-arrow-local-benchmarking.md)), -on a small stand-in suite (`../demo-suite`: a quiet and a noisy Google -Benchmark case, where hardening adds about 3% of work). - -It exercises the target-provider contract of -[`system-decomposition.md`](../../../docs/design/system-decomposition.md) §3.4: - -- **You are the user, and the provider is your build script.** `provider.sh` - configures and builds with CMake, as `archery` or a contributor would. benchx - builds nothing. -- **The provider tells benchx what it built** in a *target description*: the - build directory and its source, plus two pieces of text you own, `how_built` - (the exact commands) and `activation` (what to run to enter the environment, - with the `shell` it is written for). Either may be empty. -- **You activate the environment yourself** (`source activate.sh`). benchx never - runs `activation`. It records the text, as given, in the work order and in - every result's `provenance.info`, beside the allowlisted environment variables - the runner actually inherited. If the two disagree, both stay and nothing - judges. - -## Run it - -You need `cmake`, a C++ compiler, `git` and `python3` on your `PATH`, and network -access the first time (CMake fetches Google Benchmark v1.9.1 unless it is -installed). From the repository root: - -```console -$ cd benchx -$ python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like -$ pip install -e . -$ examples/archery-local/run.sh -``` - -It takes about two minutes the first time (two CMake builds and a measured -run) and prints every command before running it, as you would type it. The -script runs `bx` and `python` from the active environment. More rounds: -`ROUNDS=10 ./run.sh`. - -Everything it creates is under `_work/` in this directory, including the local -store (`_work/store.parquet`, set by `paths.store` in `benchx.conf.json`), so nothing -outside this directory changes. `rm -rf _work` starts over. - -## What you will see - -1. **Activate.** `source activate.sh` puts a stand-in dependencies directory on - `LD_LIBRARY_PATH`. This is you, not benchx. -2. **`bx target describe @plain`** fails with exit 3 from the provider: nothing is - built yet. `describe` builds nothing, by contract. -3. **`bx target prepare @plain` / `@hardened`** make the provider build. Its CMake - output is the build's own, on stderr; the target description it prints on - stdout is the answer. `how_built` is the very commands the provider ran. -4. **`bx target describe @hardened`** now works. The same document is stored in - the build directory as `.benchx-target.json`, so a long-lived build directory - carries its own description (point `bx compare` at a bare directory and it - reads that file; no provider needed). -5. **`bx compare @plain @hardened ...`** is the workbench's session loop: it - asks the provider to prepare each side (a no-op build now), then alternates - the sides over the rounds, one work order each, delivers the results to the - local store, and compares: - - ``` - run hardening-... profile environments baseline plain contender hardened - plain: how built: cmake -S .../_work/src -B .../_work/build-plain ... (+1 lines) - plain: activation: source .../activate.sh - plain: observed env: LD_LIBRARY_PATH=.../_work/deps/lib - hardened: how built: cmake -S ... -B .../_work/build-hardened ... -DDEMO_HARDENED=ON (+1 lines) - ... - REGRESSED (1) - BM_Quiet/1024 wall-time effect +2.42% noise 0.57% rounds=5 - NO CHANGE DETECTED (1) - BM_Noisy/1024 wall-time effect +3.43% noise 3.04% rounds=5 - ``` - - Compare that with the story's pains: the comparison now describes itself - (what was built, how, in what environment), and the verdict uses each - benchmark's own noise instead of a flat 5%: the quiet case flags a 2.4% - change that is four times its noise, the noisy case does not flag a 3.4% - swing inside its noise. The numbers vary from run to run; with few rounds the - noisy case may flag too. -6. **The work order and a result**, printed: the order's `target` carries - `how_built`, `activation`, `shell` and `provider`; the result's - `provenance.info` carries them, next to `coordinates.environment.system.process.environment` and - `coordinates.subject.configuration` (read by the runner from - `CMakeCache.txt`). -7. **Forget to activate.** The same comparison with `LD_LIBRARY_PATH` removed - from the environment: the declaration still says `source .../activate.sh`, - the observed environment no longer lists `LD_LIBRARY_PATH`, and nothing - refuses: declarations are recorded as given (`benchmark-environments.md` - §3.2), never checked. - -## Try it by hand - -After `./run.sh` (the build directories and the store stay in `_work/`). From -this directory, with the environment still active: - -```console -$ source activate.sh - -$ bx target describe @hardened # what exists, nothing built -$ bx target prepare @hardened # incremental build, then the description -$ bx target describe _work/build-hardened # a bare directory: its own sidecar - -# the story's own knobs: scope by filter, precision by repetitions -$ bx compare @plain @hardened --label build=plain,hardened --filter BM_Quiet --rounds 10 \ - --repetitions 5 --run-key mine # the adapter, suite, project and profile come from benchx.conf.json - -$ bx compare --run mine --label build --baseline plain # read mode -$ bx head -n 4 # the latest stored results -$ bx head --json -n 1 | python3 -m json.tool | less # provenance.info, environment.system -``` - -Edit `provider.sh`'s `hardened)` case or `activate.sh` and run again to see how -each change shows up in the next comparison. Empty text is fine: set -`"activation": ""` (or `null`, or leave it out) in a description and the -runner simply omits it from the results. - -## What is in this directory - -| File | Role | -|---|---| -| `run.sh` | the end-to-end script | -| `provider.sh` | the target provider: your build script, speaking the `describe`/`prepare` contract | -| `benchx.conf.json` | the project file: the provider command, the harness and its suite, the project name, the comparison profile, and where the store and results go | -| `activate.sh` | the environment you activate before calling `bx` | -| `_work/` | everything generated: source copy, build directories, results, store (git-ignored) | - -The documents involved are in `../../../schemas/`: `target-provider-request`, -`target-description`, and `work-order` (its `target` takes the declared text). - -## What this does not show - -The provider here builds one source tree, `WORKSPACE`. Comparing against a tag -or a past revision (`@plain=v1.0`) is the provider's business and is not -implemented in this example; the implementation's own `demo.sh` shows revisions -with two prebuilt worktrees. A declared `build` object in a description is -refused by the runner (it reads `CMakeCache.txt` instead). diff --git a/benchx/examples/archery-local/activate.sh b/benchx/examples/archery-local/activate.sh deleted file mode 100644 index cd10b8b..0000000 --- a/benchx/examples/archery-local/activate.sh +++ /dev/null @@ -1,8 +0,0 @@ -# The environment this example's benchmarks are meant to run in. In the Arrow -# story this is where you would load a toolchain or point at the dependencies -# you built; here it only puts a stand-in "deps" directory on the library -# search path. You source it yourself, before invoking bx: benchx never runs -# it, it records the text that says you did (the target description's -# `activation`) next to the environment it actually inherited. -HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -export LD_LIBRARY_PATH="$HERE/_work/deps/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" diff --git a/benchx/examples/archery-local/benchx.conf.json b/benchx/examples/archery-local/benchx.conf.json deleted file mode 100644 index 9029306..0000000 --- a/benchx/examples/archery-local/benchx.conf.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema_version": "benchx/config/0.1.0", - "project": "archery-local", - "provider": { "command": ["./provider.sh"] }, - "adapter": { - "name": "google-benchmark", - "suite": "demo-bench", - "quantities": ["wall-time"], - "precision": { "repetitions": 5 } - }, - "paths": { "store": "_work/store.parquet", "results": "_work/results" }, - "compare": { "profile": "environments" } -} diff --git a/benchx/examples/archery-local/provider.sh b/benchx/examples/archery-local/provider.sh deleted file mode 100755 index a5226e0..0000000 --- a/benchx/examples/archery-local/provider.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash -# The target provider of this example (system-decomposition.md §3.4). -# -# It is the user's own build script, speaking the provider contract: one JSON -# request on stdin (schemas/target-provider-request/0.1.0), one target -# description on stdout (schemas/target-description/0.1.0). Everything else, -# above all the build's own output, goes to stderr. -# -# describe reports a target that exists and builds nothing; exit 3 if none -# prepare configures and builds it, with CMake, as Arrow's contributors do -# (an incremental no-op when it is already built), then reports it -# -# Configurations: `plain`, and `hardened` (-DDEMO_HARDENED=ON), the same -# commit built two ways, like the -DARROW_HARDENING=ON story. -# -# `how_built` is assembled from the very commands this script runs, so the -# text cannot drift from what happened. benchx never runs any of it. -set -euo pipefail -HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -WORK=${BENCHX_EXAMPLE_WORK:-$HERE/_work} -export GIT_AUTHOR_NAME=demo GIT_AUTHOR_EMAIL=demo@example.org -export GIT_COMMITTER_NAME=demo GIT_COMMITTER_EMAIL=demo@example.org - -request=$(cat) -field() { python3 -c 'import json,sys; print(json.load(sys.stdin).get(sys.argv[1]) or "")' "$1" <<<"$request"; } -operation=$(field operation) -configuration=$(field configuration) -case "$configuration" in - plain) extra=() ;; - hardened) extra=(-DDEMO_HARDENED=ON) ;; - *) echo "provider: unknown configuration '$configuration' (plain or hardened)" >&2; exit 2 ;; -esac -SRC=$WORK/src -BUILD=$WORK/build-$configuration - -describe() { - [[ -f $BUILD/.benchx-target.json ]] || { echo "provider: $configuration is not built; use prepare" >&2; exit 3; } - cat "$BUILD/.benchx-target.json" -} - -if [[ $operation == describe ]]; then describe; exit 0; fi - -HOW="" -step() { # step : record it in how_built, show it, run it (output to stderr) - local line="$*" - HOW+="$line"$'\n' - printf '\033[2m provider$ %s\033[0m\n' "$line" >&2 - "$@" >&2 -} - -if [[ ! -d $SRC/.git ]]; then - mkdir -p "$WORK" - cp -R "$HERE/../demo-suite" "$SRC" - git -C "$SRC" init -q - git -C "$SRC" remote add origin https://example.org/archery-local/demo-suite.git - git -C "$SRC" add -A - git -C "$SRC" commit -qm "demo-suite" -fi -mkdir -p "$WORK/deps/lib" # stand-in for the dependencies activate.sh points at - -step cmake -S "$SRC" -B "$BUILD" -DCMAKE_BUILD_TYPE=Release -DFETCHCONTENT_BASE_DIR="$WORK/_deps" ${extra[@]+"${extra[@]}"} -step cmake --build "$BUILD" -j - -BUILD=$BUILD SRC=$SRC HOW=$HOW CONFIGURATION=$configuration HERE=$HERE python3 - >"$BUILD/.benchx-target.json" <<'PY' -import json, os -print(json.dumps({ - "schema_version": "benchx/target-description/0.1.0", - "target": {"kind": "build", "path": os.environ["BUILD"], - "source": {"uri": "https://example.org/archery-local/demo-suite.git", "type": "git"}, - "source_dir": os.environ["SRC"]}, - "how_built": os.environ["HOW"].rstrip("\n"), - "activation": "source " + os.path.join(os.environ["HERE"], "activate.sh"), - "shell": "bash", - "provider": {"name": "provider.sh", "version": "1"}, -}, indent=1)) -PY -describe # the sidecar just written, which stays with the build directory diff --git a/benchx/examples/archery-local/run.sh b/benchx/examples/archery-local/run.sh deleted file mode 100644 index 7095da7..0000000 --- a/benchx/examples/archery-local/run.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash -# End to end, in the shape of the Arrow local story ("Measure what a build -# option costs"): one commit built two ways, plain and hardened, compared with -# noise-aware verdicts, where the comparison describes itself. -# -# You play the user. The target provider (provider.sh, named in -# benchx.conf.json) is your own build script: it builds with CMake and -# tells benchx what it built, how, and what environment you need. benchx builds -# nothing, and runs none of that text: you activate the environment yourself -# (activate.sh) and benchx records the text beside the environment it inherited. -# -# Every command is printed before it runs, as you would type it. Run it in an -# activated environment that has benchx installed (from benchx/: -# `pip install -e .`, or `uv sync --extra test`). Needs cmake and a C++ -# compiler on PATH, and network access the first time, to fetch Google -# Benchmark unless installed. -# -# ROUNDS=5 alternation rounds (default 5) -# -# benchx.conf.json is this directory's project file, so the local store is kept -# inside it too (_work/store.parquet). Delete _work/ to start over. -set -euo pipefail -cd "$(dirname "$0")" -HERE=$PWD -ROUNDS=${ROUNDS:-5} -STAMP=$(date -u +%Y%m%dT%H%M%SZ) -RUN="hardening-$STAMP" - -for tool in cmake c++ bx python; do - command -v "$tool" >/dev/null || { echo "run.sh: $tool not found on PATH (see README.md)" >&2; exit 1; } -done -bx --help >/dev/null 2>&1 || { echo "run.sh: benchx is not installed in the active environment (see README.md)" >&2; exit 1; } - -step() { printf '\n\033[1m== %s\033[0m\n' "$*"; } -say() { printf '\033[36m$ %s\033[0m\n' "$*" >&2; } -bx() { say bx "$@"; command bx "$@"; } - -step "user: activate the environment (benchx will not do this for you)" -say source activate.sh -source activate.sh -echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" - -step "ask the provider about a target that is not built yet: describe builds nothing" -bx target describe @plain || echo "(exit $?: nothing to describe yet, as expected)" - -step "prepare both configurations: the provider builds (its build output is the dimmed lines)" -bx target prepare @plain > /dev/null -bx target prepare @hardened - -step "describe now works, and the description lives in the build directory too" -bx target describe @hardened | head -3 -say head -3 _work/build-hardened/.benchx-target.json -head -3 _work/build-hardened/.benchx-target.json -echo "(the same document: a build directory carries its own description)" - -step "compare: the workbench asks the provider to prepare each side (a no-op), then alternates them" -bx compare @plain @hardened --label build=plain,hardened --rounds "$ROUNDS" --run-key "$RUN" --out _work/results - -step "the comparison names what was built and how; before, it was two opaque directory paths" -echo "# the target of work order 0001 (round 1, the hardened side), as the runner received it:" -python -c ' -import json, sys -print(json.dumps(json.load(open(sys.argv[1]))["target"], indent=1))' _work/results/orders/order-0001.json - -step "and every result carries it, next to what the runner saw (declared, observed: nothing judges)" -say bx head --json -n 1 '| …' -command bx head --json -n 1 | python -c ' -import json, sys -doc = json.loads(sys.stdin.readline()) -info = doc["provenance"]["info"] -print("provenance.info.how_built :", info["how_built"].replace("\n", "\n" + " " * 29)) -print("provenance.info.activation :", info["activation"]) -print("provenance.info.shell :", info["shell"]) -print("provenance.info.target_provider:", info["target_provider"]) -system = doc["coordinates"]["environment"]["system"] -print("environment.system.process.environment:", system.get("process", {}).get("environment")) -print("coordinates.subject.configuration:", doc["coordinates"]["subject"].get("configuration"))' - -step "forget to activate: the declaration still says what you should have run, the observation says what you did" -say env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build --rounds 3 ... -env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build \ - --label build=plain,hardened --rounds 3 --run-key "$RUN-unactivated" --out _work/results-unactivated - -printf '\nartifacts: %s/_work (results, orders, build directories, the local store)\n' "$HERE" diff --git a/benchx/examples/demo-suite/CMakeLists.txt b/benchx/examples/demo-suite/CMakeLists.txt deleted file mode 100644 index 503ff46..0000000 --- a/benchx/examples/demo-suite/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -cmake_minimum_required(VERSION 3.20) -project(demo CXX) - -option(DEMO_HARDENED "Simulated hardening checks, the one option the environments demo changes" OFF) -set(CMAKE_CXX_STANDARD 17) - -find_package(benchmark QUIET) -if(NOT benchmark_FOUND) - include(FetchContent) - set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) - set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "" FORCE) - FetchContent_Declare(benchmark - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.1 - GIT_SHALLOW TRUE) - FetchContent_MakeAvailable(benchmark) -endif() - -add_executable(demo-bench bench.cpp) -target_link_libraries(demo-bench benchmark::benchmark_main) -if(DEMO_HARDENED) - target_compile_definitions(demo-bench PRIVATE DEMO_HARDENED) -endif() diff --git a/benchx/examples/demo-suite/bench.cpp b/benchx/examples/demo-suite/bench.cpp deleted file mode 100644 index 9acb208..0000000 --- a/benchx/examples/demo-suite/bench.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Two benchmarks for the benchx demo: a quiet one and a noisy one. -// -// kWork is what the revisions demo changes between commits; DEMO_HARDENED is -// what the environments demo changes between two builds of one commit. Both -// add a few percent of work, so the expected verdicts are known: the quiet -// benchmark flags the change, the noisy one does not. -#include - -#include -#include - -constexpr int kWork = 100; -#ifdef DEMO_HARDENED -constexpr int kExtra = 3; // simulated cost of hardening checks, in percent -#else -constexpr int kExtra = 0; -#endif - -static double Sum(const std::vector& v, long rounds) { - double s = 0; - for (long r = 0; r < rounds; ++r) { - for (double x : v) s += x; - benchmark::DoNotOptimize(s); - } - return s; -} - -static void BM_Quiet(benchmark::State& state) { - std::vector v(state.range(0), 1.0); - const long rounds = kWork * (100 + kExtra); - for (auto _ : state) benchmark::DoNotOptimize(Sum(v, rounds)); -} -BENCHMARK(BM_Quiet)->Arg(1024); - -// Each repetition draws its own load level, so repetitions disagree by -// around ten percent, as they do for a benchmark at the mercy of its machine. -static void BM_Noisy(benchmark::State& state) { - std::vector v(state.range(0), 1.0); - std::random_device device; - std::uniform_real_distribution level(1.0, 1.25); - const long rounds = static_cast(kWork * (100 + kExtra) * level(device)); - for (auto _ : state) benchmark::DoNotOptimize(Sum(v, rounds)); -} -BENCHMARK(BM_Noisy)->Arg(1024); diff --git a/benchx/examples/demo/.gitignore b/benchx/examples/demo/.gitignore deleted file mode 100644 index 13eecef..0000000 --- a/benchx/examples/demo/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -store.parquet -results/ diff --git a/benchx/examples/demo/benchx.conf.json b/benchx/examples/demo/benchx.conf.json deleted file mode 100644 index fae92f0..0000000 --- a/benchx/examples/demo/benchx.conf.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "schema_version": "benchx/config/0.1.0", - "adapter": { - "name": "google-benchmark", - "suite": "demo-bench", - "precision": { "repetitions": 5 } - }, - "paths": { "store": "store.parquet", "results": "results" }, - "compare": { "profile": "revisions" } -} diff --git a/benchx/examples/matmul-params/.gitignore b/benchx/examples/matmul-params/.gitignore deleted file mode 100644 index f1e3ac4..0000000 --- a/benchx/examples/matmul-params/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -out/ -results/ diff --git a/benchx/examples/matmul-params/README.md b/benchx/examples/matmul-params/README.md deleted file mode 100644 index 2fe3e15..0000000 --- a/benchx/examples/matmul-params/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Example: one benchmark, several parameter combinations - -`bench_matmul.py` times `a @ b` for square float64 arrays, for N = 10, 50, 100 and -for numpy and torch: one function, six cases. `bx bench` runs each combination as its own -case, under [pyperf](https://pyperf.readthedocs.io). - -## Run it - -From `benchx/`, in an environment of your choice: - -```sh -pip install -e . pyperf numpy torch -cd examples/matmul-params -export BENCHX_HOME=$PWD/out/home # optional: keep the result store here, not in ~ -bx bench bench_matmul.py --out out -``` - -This uses pyperf's defaults (20 worker processes x 3 values, 0.1 s calibration) and takes -several minutes, mostly importing torch in every worker. - -## A quick version - -```sh -bx bench bench_matmul.py --processes 2 --values 2 --min-time 0.01 --out out -``` - -About 15 seconds. The timings are rougher: each case has 4 values. - -## Output - -One row per combination, named by the function and its arguments: - -``` -case median mean ± sd ... -bench_matmul [n=10, xp='numpy'] 1.627 us ... -bench_matmul [n=10, xp='torch'] 2.387 us ... -... -bench_matmul [n=100, xp='torch'] 23.88 us ... -``` - -The result files are in `out/`, and `out/home/store.parquet` is the store. `bx` reports -twice as many results as rows: each case also records `load-average`. - -## Choosing cases - -```sh -bx bench bench_matmul.py bench_matmul # a function's name: all six -bx bench bench_matmul.py "bench_matmul(100, 'torch')" # one combination -bx bench bench_matmul.py --filter "'torch'" # a regex: the three torch cases -``` - -A regex is searched in `bench_matmul(10, 'torch')`, the function's name and the argument -values (not their names). `--filter 'bench_matmul$'` is therefore refused, since `$` leaves -no room for the arguments; the refusal says so. - -## How the parameters are declared - -```python -bench_matmul.params = [[10, 50, 100], ["numpy", "torch"]] # one list per argument -bench_matmul.param_names = ["n", "xp"] -``` - -Combinations are the product of the lists, in this order, and the function is called with -one of them. A value is recorded as its `repr`, so the library is given by its name and -imported on first use, not passed as a module. - -## What is timed - -The whole body of `bench_matmul` is timed: the call to `operands(n, xp)` as well as `a @ b`. -`operands` is cached, so the arrays are made once per worker process, on its first call (a -calibration or warmup call), not in every call. What every call still pays is the cache -lookup, about 60 ns here: roughly 4% of the numpy time at N=10 and under 0.2% at N=100. -The timings include it, for numpy and torch alike. The result array of `a @ b` is also -allocated inside the timed call. diff --git a/benchx/examples/matmul-params/bench_matmul.py b/benchx/examples/matmul-params/bench_matmul.py deleted file mode 100644 index 2e71eeb..0000000 --- a/benchx/examples/matmul-params/bench_matmul.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Example: one benchmark function, six cases (3 sizes x 2 array libraries). - - bx bench bench_matmul.py # every combination - bx bench bench_matmul.py bench_matmul # the same, by name - bx bench bench_matmul.py "bench_matmul(100, 'torch')" # one combination - -`params` holds one list of values per argument of the function and `param_names` names -them; each combination is its own case. Values are recorded as their repr, so the -library is given by name and imported when first used. Both libraries use float64 -(torch would default to float32). -""" - -import functools -import importlib - - -@functools.cache -def operands(n, xp): - lib = importlib.import_module(xp) - return lib.ones((n, n), dtype=lib.float64), lib.ones((n, n), dtype=lib.float64) - - -def bench_matmul(n, xp): - a, b = operands(n, xp) - a @ b - - -bench_matmul.params = [[10, 50, 100], ["numpy", "torch"]] -bench_matmul.param_names = ["n", "xp"] diff --git a/benchx/examples/numpy-venvs/.gitignore b/benchx/examples/numpy-venvs/.gitignore deleted file mode 100644 index 388f608..0000000 --- a/benchx/examples/numpy-venvs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.venv-*/ -out/ diff --git a/benchx/examples/numpy-venvs/README.md b/benchx/examples/numpy-venvs/README.md deleted file mode 100644 index 2425bc7..0000000 --- a/benchx/examples/numpy-venvs/README.md +++ /dev/null @@ -1,102 +0,0 @@ -# numpy-venvs: UC-04 with two virtual environments - -The same code, measured under two numpy versions, by separate `bx bench` calls that alternate -round by round, then compared. This is -[UC-04](../../../docs/use-cases/UC-04-cross-env.md), "fixed branch, comparing environments": -`numpy==1.26.4` and `numpy==2.0.2` cannot be imported in one process, so each side runs in its -own environment, and a shell script drives the loop. - -You prepare the environments and enter them; benchx builds nothing and activates nothing. - -## Run it - -You need [`uv`](https://docs.astral.sh/uv/) (it fetches Python 3.12, the newest Python with numpy -1.26.4 wheels) and network access the first time. Then, from this directory: - -```console -$ ./run.sh -``` - -The versions (`NUMPY_A`, `NUMPY_B`), the Python and the number of rounds are set at the top of -`run.sh`. Everything it creates is in `.venv-a`, `.venv-b` and `out/`, all git-ignored. - -## What it does - -1. **Prepares two venvs**, `.venv-a` and `.venv-b`, each with its numpy, pyperf and benchx, and - `pyarrow<25`: benchx imports pyarrow, and pyarrow 25 refuses to import under numpy 1.x. -2. **Runs the caller's loop.** For each of the 3 rounds, one `bx bench` call per side, each in its - venv (`run.sh` sources the venv's `activate`; activating the other venv replaces it, so nothing - needs deactivating). The calls share a `--run-key`; each says which side it is - (`--label env=numpy-1.26.4`), which round, and which slot of the alternation. -3. **Compares.** `bx compare --run S --profile environments --label env --baseline numpy-1.26.4`. - The sides differ because the collector read a different numpy in each. The comparison says - what differed (`differs: ...deps.numpy`), shows what each side declared, and gives a verdict per - benchmark: - - ``` - run numpy-1793121294 profile environments baseline numpy-2.3.4 contender numpy-2.4.3 - numpy-2.3.4: how built: {"summary":{"numpy":"2.3.4"}} - numpy-2.4.3: how built: {"summary":{"numpy":"2.4.3"}} - differs: coordinates.environment.system.deps.numpy, coordinates.environment.system.deps.sha256 - - NO CHANGE DETECTED (3) - bench_matmul wall-time effect -16.28% noise 7.43% rounds=3 - ... - ``` - - (The numbers vary from run to run; `[local-only]` appears while the demo's files are - uncommitted.) -4. **Checks a declaration.** It declares numpy 2.0.2 for a run in the venv that has 1.26.4. The - collector disagrees, so every result of that run carries `collector-mismatch`. The run goes - ahead, because the collector has `on_fail: warn`, and the recorded numpy is the one found. - -## How the sides are told apart - -- `describe_env.py` is a **context collector** (`harness-adapter.md` §5.1), named in - `benchx.conf.json`. benchx runs it once per call, in the environment the call runs in. It prints - `{"summary": {"numpy": ""}}`, and that lands in each result's `environment.system.deps`, - which the `environments` comparison treats as the varying coordinate. benchx knows nothing about - venvs, conda or pixi: any command that prints facts about an environment does the same. -- `provider.py` is the **target provider**. For configuration `a` or `b` it declares, in - `how_built`, the numpy that side is meant to have. The collector has `verify: how_built`, so it - compares the declaration with what it found. The declared object has the same shape as the - collector's output, because the check compares key by key from the top. -- A check that fails refuses the run by default; `on_fail: warn` records a warning instead, which - is what the design asks for (a mismatch is recorded, not a gate). - -## UC-04's spellings, and the prototype's - -[UC-04](../../../docs/use-cases/UC-04-cross-env.md) is kept as written, and uses spellings the -prototype does not have. This is how each corresponds: - -- `benchx new-session` (`SESSION=$(...)`) → no command. The session is the run key, chosen by the - caller: `S=numpy-$(date +%s)`. -- `--session-id "$SESSION"` → `--run-key "$S"`. It lands in `provenance.run_key`. -- `benchx run run_benchmark.py` → `bx bench`. With a module and functions, - `bx bench MODULE.py [FUNCTION ...]`; with a prepared target, `bx bench @CONFIG --suite NAME`. - `bx run` takes an order file, not a script. -- `--env-label numpy-1.26.4` → `--label env=numpy-1.26.4`. The field `env_label` is - `provenance.labels.env`. -- `--round "$round"` → `--round N` and `--slot K`. UC-04 uses one number for two things. The - prototype separates the round, which pairs one attempt of each side (`procedure.round`), from - the slot, the realized position in the interleaved sequence (`procedure.slot`). UC-04's - `interleave_position` is the slot. The caller's loop keeps the slot counter. -- `benchx compare-environments --session-id S --baseline A --variant B` → - `bx compare --run S --profile environments --label env --baseline A`. The profile is UC-04's - `cross-environment-comparison`. There are exactly two sides, so the variant is the other one. -- `.venv-baseline/bin/benchx` → the `bx` of that environment, such as `.venv-a/bin/bx`. -- The output table (`benchmark_name`, `env`, `time`, `vs. baseline`) → the comparison output: - verdict groups with the effect, the noise and the rounds, and the facts that differ. -- UC-04's `session_id`, `env_label` and `interleave_position` fields → `provenance.run_key`, - `provenance.labels.env` and `procedure.slot`. The schema adds no fields for a profile - (`benchmark-result-schema.md` §5.5). - -## What is in this directory - -| File | Role | -|---|---| -| `run.sh` | the driver: venvs, the loop, the comparison, the check | -| `benchx.conf.json` | the project file: the adapter and suite, the provider, the `deps` collector | -| `provider.py` | the target provider: declares each side's numpy in `how_built` (`run.sh` exports the versions) | -| `describe_env.py` | the collector: prints the numpy this Python imports | -| `bench_suite.py` | three small numpy benchmarks | diff --git a/benchx/examples/numpy-venvs/bench_suite.py b/benchx/examples/numpy-venvs/bench_suite.py deleted file mode 100644 index 866f15a..0000000 --- a/benchx/examples/numpy-venvs/bench_suite.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Three small numpy benchmarks. pyperf calls each bench_* function many times and times the -calls, so the arrays are made once, at module level.""" - -import numpy as np - -_rng = np.random.default_rng(0) -_square = _rng.standard_normal((200, 200)) -_vector = _rng.standard_normal(100_000) - - -def bench_matmul(): - _square @ _square - - -def bench_sort(): - np.sort(_vector) - - -def bench_sum(): - _vector.sum() diff --git a/benchx/examples/numpy-venvs/benchx.conf.json b/benchx/examples/numpy-venvs/benchx.conf.json deleted file mode 100644 index 94146f8..0000000 --- a/benchx/examples/numpy-venvs/benchx.conf.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "schema_version": "benchx/config/0.1.0", - "project": "numpy-venvs", - "adapter": { - "name": "pyperf", - "suite": "bench_suite.py", - "precision": { - "repetitions": { "mode": "fixed", "levels": [{ "unit": "process", "n": 3 }, { "unit": "value", "n": 3 }] }, - "calibration": { "mode": "adaptive", "minimum_sample_seconds": 0.02 }, - "warmup": { "mode": "count", "n_warmup": 1 } - } - }, - "provider": { "command": ["./provider.py"] }, - "collectors": { - "deps": { - "command": ["python", "{root}/describe_env.py"], - "verify": "how_built", - "on_fail": "warn" - } - }, - "paths": { "store": "out/store.parquet", "results": "out" } -} diff --git a/benchx/examples/numpy-venvs/describe_env.py b/benchx/examples/numpy-venvs/describe_env.py deleted file mode 100644 index 7363bbb..0000000 --- a/benchx/examples/numpy-venvs/describe_env.py +++ /dev/null @@ -1,12 +0,0 @@ -"""The collector: print what the numpy that this Python imports says about itself. - -benchx.conf.json runs it once per call, in the environment the call runs in, so it can only -differ between the two sides if their environments do. It prints the same text for the same -environment: no paths, no timestamps. -""" - -import json - -import numpy - -print(json.dumps({"summary": {"numpy": numpy.__version__}}, sort_keys=True)) diff --git a/benchx/examples/numpy-venvs/provider.py b/benchx/examples/numpy-venvs/provider.py deleted file mode 100755 index 2fdbd5f..0000000 --- a/benchx/examples/numpy-venvs/provider.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -"""The target provider of this example (system-decomposition.md §3.4): one JSON request on stdin, -one target description on stdout. It builds nothing, so `describe` and `prepare` answer alike. - -Configuration `a` or `b` names a side. The description declares, in how_built, the numpy the -user means that side to have, in the shape describe_env.py prints. The collector in -benchx.conf.json compares the two: a side whose environment is not the one it was declared -with gets the warning collector-mismatch on every result. -""" - -import json -import os -import sys -from pathlib import Path - -HERE = Path(__file__).resolve().parent -# The canonical URI this working tree's results are filed under, as in archery-local. -URI = "https://example.org/numpy-venvs/demo.git" - - -def main(): - versions = {"a": os.environ.get("NUMPY_A"), "b": os.environ.get("NUMPY_B")} - if not all(versions.values()): - sys.exit("provider: run.sh sets NUMPY_A and NUMPY_B") - - # An absent or null configuration is refused below, like an unknown one. - configuration = json.load(sys.stdin).get("configuration") - if configuration not in versions: - print(f"provider: unknown configuration {configuration!r} (a or b)", file=sys.stderr) - sys.exit(2) - - json.dump({ - "schema_version": "benchx/target-description/0.1.0", - "target": { - "kind": "working_tree", - "path": str(HERE), - "source": {"uri": URI, "type": "git"}, - }, - "how_built": {"summary": {"numpy": versions[configuration]}}, - "provider": {"name": "provider.py", "version": "1"}, - }, sys.stdout, indent=1) - print() - - -main() diff --git a/benchx/examples/numpy-venvs/run.sh b/benchx/examples/numpy-venvs/run.sh deleted file mode 100755 index 7b67bf2..0000000 --- a/benchx/examples/numpy-venvs/run.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# UC-04 with two venvs: the same code under two numpy versions, measured by separate `bx bench` -# calls that alternate round by round, then compared. See README.md for what each step shows. -# -# Needs `uv`. Everything it creates is here: .venv-a, .venv-b and out/ (git-ignored). -# benchx builds nothing. -set -euo pipefail -cd "$(dirname "$0")" - -# provider.py reads the numpy versions from the environment. numpy 1.26.4 has no wheels beyond -# Python 3.12, so both venvs use it. Three rounds are the fewest that give a verdict. -export NUMPY_A=1.26.4 NUMPY_B=2.0.2 -PYTHON_VERSION=3.12 -ROUNDS=3 -SIDES=("a $NUMPY_A" "b $NUMPY_B") -RUN_KEY=numpy-$(date +%s) -SLOT=0 - -rm -rf out .venv-a .venv-b -mkdir -p out - -echo "== 1. the user prepares both environments; benchx builds nothing" -for SIDE_VERSION in "${SIDES[@]}"; do - read -r SIDE VERSION <<<"$SIDE_VERSION" - uv venv -q --python "$PYTHON_VERSION" ".venv-$SIDE" - # benchx imports pyarrow, and pyarrow 25 refuses to import under numpy 1.x. - uv pip install -q --python ".venv-$SIDE/bin/python" "numpy==$VERSION" "pyarrow<25" pyperf -e ../.. - echo " .venv-$SIDE: numpy $VERSION" -done - -echo "== 2. the caller's loop: alternate the sides, round by round, one slot per call" -for ROUND in $(seq 0 $((ROUNDS - 1))); do - for SIDE_VERSION in "${SIDES[@]}"; do - read -r SIDE VERSION <<<"$SIDE_VERSION" - # Activating a venv replaces the one activated before; no deactivate is needed. - source ".venv-$SIDE/bin/activate" - bx bench "@$SIDE" --run-key "$RUN_KEY" --label "env=numpy-$VERSION" \ - --round "$ROUND" --slot "$SLOT" --out "out/$RUN_KEY" > /dev/null - echo " round $ROUND, slot $SLOT: side $SIDE" - SLOT=$((SLOT + 1)) - done -done - -echo "== 3. compare: the sides differ in the numpy the collector read" -bx compare --run "$RUN_KEY" --profile environments --label env \ - --baseline "numpy-$NUMPY_A" | tee out/compare.txt -grep -q 'differs:.*deps\.numpy' out/compare.txt # the differing fact is shown -if grep -q 'failed invariants' out/compare.txt; then echo "the comparison was refused" >&2; exit 1; fi - -echo "== 4. the check: declare numpy $NUMPY_B for a run in the venv that has $NUMPY_A" -source .venv-a/bin/activate -bx bench @b --run-key "$RUN_KEY-mismatch" --label env=wrong \ - --round 0 --slot 0 --out "out/$RUN_KEY-mismatch" > /dev/null -grep -rq collector-mismatch "out/$RUN_KEY-mismatch" -echo " the results carry collector-mismatch; the run went ahead, with the numpy it found" diff --git a/benchx/examples/pyperf-bench/.gitignore b/benchx/examples/pyperf-bench/.gitignore deleted file mode 100644 index 89f9ac0..0000000 --- a/benchx/examples/pyperf-bench/.gitignore +++ /dev/null @@ -1 +0,0 @@ -out/ diff --git a/benchx/examples/pyperf-bench/README.md b/benchx/examples/pyperf-bench/README.md deleted file mode 100644 index 95db4d4..0000000 --- a/benchx/examples/pyperf-bench/README.md +++ /dev/null @@ -1,151 +0,0 @@ -# Example: benchmarking Python functions with `bx bench` - -A runnable example of a benchmark suite. `bench_suite.py` holds five small -functions; `bx bench` runs each one under [pyperf](https://pyperf.readthedocs.io), -writes benchx result files, delivers them to a local store, and prints a table. - -## Run it - -```sh -cd benchx -python3 -m venv .venv && . .venv/bin/activate # Python >= 3.11, or any environment you like -pip install -e . pyperf -examples/pyperf-bench/run.sh -``` - -The script runs `bx` and `python` from the active environment, so benchx and pyperf must -be installed there; the workers use that same `python`. Extra arguments go to -`bx bench` in step 1 and win over the config file, e.g. `./run.sh --processes 20`. -It takes about 20 seconds. Nothing outside this directory changes: the local store is -`out/store.parquet` (`paths.store` in `benchx.conf.json`), and `out/` is deleted at the start -of each run. - -## What goes in - -| Input | Where | What it does | -|---|---|---| -| `.` | first argument | The target: this directory, a working tree. pyperf reads a bare path that way. | -| `benchx.conf.json` | found by walking up from where you run `bx`; its directory is the project root | The harness (`adapter.name`), the module (`adapter.suite`, `bench_suite.py`: each function named `bench_*` or `benchmark_*` is a case), and the precision: worker processes, values per process, calibration, warmup. Here: 5 processes x 3 values, calibrate to 0.05 s per value. | -| Function names | after the target | `bx bench . bench_sorted bench_join` runs only those, any function, whatever its name. `bx bench bench_suite.py bench_sorted` still works as a shorthand: the module's git checkout is the target and its path the suite. | -| Flags | command line | `--processes`, `--values`, `--min-time`, `--loops`, `--warmups` override the file, and so does `--suite`. | -| `-e NAME=VALUE` | command line | Sets an environment variable for the workers. The workers otherwise inherit your whole environment. | - -Precision is resolved as: **flags, then the config file, then pyperf's defaults** -(20 processes x 3 values, 0.1 s calibration, 1 warmup). A level you leave out of the -file takes pyperf's default. `bx config init` writes a `benchx.conf.json` that states -the defaults, as a template to edit, and `bx config show` says where each value comes from. The resolved values are written into the work order, -so a result records the precision that applied. A bad config file is reported before -anything runs. - -The benchmark functions are timed bodies: pyperf calls each one many times, and does -the calibration (how many calls make up one value) itself. Put setup at module level. - -## The config file - -`benchx.conf.json` is the example's project file. Its `adapter.precision` is the work-order -`precision` block; `adapter.name` and `adapter.suite` say which harness and which module: - -```json -{ - "schema_version": "benchx/config/0.1.0", - "adapter": { - "name": "pyperf", - "suite": "bench_suite.py", - "precision": { - "repetitions": { - "mode": "fixed", - "levels": [{"unit": "process", "n": 5}, {"unit": "value", "n": 3}] - }, - "calibration": {"mode": "adaptive", "minimum_sample_seconds": 0.05}, - "warmup": {"mode": "count", "n_warmup": 1} - } - }, - "paths": {"store": "out/store.parquet", "results": "out"} -} -``` - -- `repetitions.levels`: `process` is the number of worker processes, `value` the values - each one records. Leave a level out and it takes pyperf's default (20 and 3). -- `calibration`: `adaptive` with `minimum_sample_seconds` lets pyperf pick how many calls - make up one value; `{"mode": "fixed", "n_iterations": N}` fixes it. -- `warmup`: `{"mode": "count", "n_warmup": N}` discards N values per process, or `{"mode": "none"}`. -- Every key is optional; pyperf's defaults (20 x 3, adaptive 0.1 s, 1 warmup) fill the rest. - Run `bx config init` in your own project to get a file that states them all. - -## What `run.sh` does - -1. `bx bench .` runs all five cases with the config file's suite and precision. -2. `bx bench . bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2` - runs one case, sets a variable for the workers, and overrides the file's process count. -3. `bx bench . bench_join --run-key example-again` runs one case a second time - under a new run key, so its series gets a second point. -4. `ls -l` and `bx head` show the local store: one Parquet file, and its latest rows. -5. `bx series` lists the series in the store. -6. `bx history` lists the points of the `bench_join` series, one per run. The script fails - if there are not two, which is what CI relies on. -7. It lists the files written. - -## What comes out - -The table, for step 1 (your numbers will differ): - -``` -case median mean ± sd min .. max n status ------------------ -------- ------------------- -------------------- -- ------- -bench_dict_build 18.29 us 18.56 us ± 680.9 ns 17.71 us .. 19.83 us 15 success -bench_join 3.585 us 3.543 us ± 272.4 ns 3.236 us .. 3.997 us 15 success -``` - -`n` is processes x values (5 x 3 = 15). Each value is the time of one call, in seconds -underneath. A `[warning]` after the status means the result carries a quality warning, -such as `env-mismatch`. - -Files, under `out/all/` and `out/env/`: - -- `__wall-time.json`: one benchx result per case (schema `measurement-result`; - with `-e` the name also carries the percent-encoded parameters, as in - `example-env_bench_env_var_%7B%22EXAMPLE_N%22%3A%22100000%22%7D_wall-time.json`): - the observations grouped by worker process, the pyperf protocol that applied, the - environment, the revision of the tree, and warnings. -- `__load-average.json`: beside each case's result, the 1-minute load - average the runner read before the case ran, a condition measurement with the same - `attempt_key` (schema §4.4). `bx series` lists its series too; `--quantity wall-time` - leaves them out. -- `workorder-.json`: the work order that was run, and `orders/order.json`, the - same order as written for you to read. Results refer to it by hash. -- `artifacts//NNNN.native.json`: pyperf's own output; `.driver.py`: the generated - script that ran the function; `.worker-env.json`: the environment a worker really had; - `.stdout` and `.stderr`. - -The local store, `out/home/store.parquet`, holds the results. `run.sh` reads it back -(your numbers and fingerprints will differ): - -``` -$ bx series --quantity wall-time -d780c9564c72 local/be8372c04606 bench_dict_build wall-time [s] est=benchx/median/v1 -e6eaa02c8309 local/be8372c04606 bench_env_var wall-time [s] est=benchx/median/v1 -4be59ef6debc local/be8372c04606 bench_join wall-time [s] est=benchx/median/v1 -... -85a6aa9c6996 local/be8372c04606 bench_env_var wall-time [s] est=benchx/median/v1 - -$ bx history 4be59ef6debc -revision order unavailable without a revision graph; listed by measurement start -d83d37b69010 2026-10-13T11:04:28.656515Z 3.32107e-06 (server) example-all:bench_join:wall-time -d83d37b69010 2026-10-13T11:04:39.045277Z 3.76987e-06 (server) example-again:bench_join:wall-time -``` - -`bench_env_var` has two series because step 2 sets `EXAMPLE_N`, which changes the -environment the case ran in. `bench_join` has one series with two points, one for each -run key. You can also `bx ingest` more result files; comparing two runs is not part of -this step. - -## Notes - -- The tree measured is the git checkout containing the module (here, this repository), as - found, uncommitted changes included. The result records the revision and whether the tree - was dirty. Because that tree is the whole repository, an uncommitted change anywhere in it, - in a doc or another example as much as in `bench_suite.py`, makes the result `dirty` and - changes its tree id; commit or stash first for a clean result. Outside git, it records a hash of the directory's contents. `--tree DIR` picks - the tree. -- pyperf prints advice when the machine is noisy; it is in `.stderr`. On a laptop, - close other programs. `python -m pyperf system tune` (needs root) quiets the CPU further. diff --git a/benchx/examples/pyperf-bench/bench_suite.py b/benchx/examples/pyperf-bench/bench_suite.py deleted file mode 100644 index cf6589d..0000000 --- a/benchx/examples/pyperf-bench/bench_suite.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Example benchmark suite: plain Python functions, one case each. - -`bx bench bench_suite.py` runs every function named bench_* or benchmark_*. -Name functions on the command line to run only those: - bx bench bench_suite.py bench_sorted bench_join - -Each function is the timed body; pyperf calls it many times. Do your setup -at module level (it runs once per worker process, outside the timing). -""" - -import os - -DATA = list(range(2000, 0, -1)) -WORDS = [f"word{i}" for i in range(500)] - - -def bench_sorted(): - sorted(DATA) - - -def bench_join(): - ",".join(WORDS) - - -def bench_dict_build(): - {w: len(w) for w in WORDS} - - -def bench_sum_squares(): - sum(i * i for i in range(1000)) - - -def bench_env_var(): - """Reads EXAMPLE_N from the environment, to show `-e NAME=VALUE` reaching the workers.""" - int(os.environ.get("EXAMPLE_N", "100")) ** 2 - - -def helper_not_a_case(): - """Not named bench_*, so only run if you name it explicitly.""" diff --git a/benchx/examples/pyperf-bench/benchx.conf.json b/benchx/examples/pyperf-bench/benchx.conf.json deleted file mode 100644 index bc59544..0000000 --- a/benchx/examples/pyperf-bench/benchx.conf.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "schema_version": "benchx/config/0.1.0", - "adapter": { - "name": "pyperf", - "suite": "bench_suite.py", - "precision": { - "repetitions": { "mode": "fixed", "levels": [{ "unit": "process", "n": 5 }, { "unit": "value", "n": 3 }] }, - "calibration": { "mode": "adaptive", "minimum_sample_seconds": 0.05 }, - "warmup": { "mode": "count", "n_warmup": 1 } - } - }, - "paths": { "store": "out/store.parquet", "results": "out" } -} diff --git a/benchx/examples/pyperf-bench/run.sh b/benchx/examples/pyperf-bench/run.sh deleted file mode 100644 index d4b48e7..0000000 --- a/benchx/examples/pyperf-bench/run.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Run the example suite with `bx bench`. See README.md for what each step does. -# -# ./run.sh uses benchx.conf.json here (5 processes x 3 values) -# ./run.sh --processes 20 extra arguments go to `bx bench` and win over the file -# -# Run it in an activated environment that has benchx installed with pyperf -# (from benchx/: `pip install -e . pyperf`, or `uv sync --extra test`). -# Results go to ./out, and the local store to ./out/store.parquet (both set in -# benchx.conf.json), so nothing outside this directory changes. -set -euo pipefail -cd "$(dirname "$0")" -bx() { printf '\033[36m$ bx %s\033[0m\n' "$*" >&2; command bx "$@"; } - -command bx --help >/dev/null 2>&1 && python -c "import pyperf" 2>/dev/null || { - echo "missing dependencies: install benchx and pyperf into the active environment (see README.md)" >&2 - exit 1; } -rm -rf out; mkdir -p out - -echo "== 1. all bench_* functions, suite and precision from benchx.conf.json" -bx bench . --run-key example-all --out out/all "$@" - -echo; echo "== 2. one function, an environment variable set for the workers, flags over the file" -bx bench . bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2 \ - --run-key example-env --out out/env - -echo; echo "== 3. the same case again under a new run key, so its series gets a second point" -bx bench . bench_join --run-key example-again --out out/again - -echo; echo "== 4. what is in the local store: one Parquet file" -ls -l out/store.parquet -bx head -n 10 - -echo; echo "== 5. its series (bench_env_var has two: the -e setting changes the environment)" -bx series --quantity wall-time > out/series.txt -cat out/series.txt - -echo; echo "== 6. the history of bench_join: one point per run" -bx series --workload bench_join --quantity wall-time > out/join-series.txt -SERIES=$(awk 'NR == 1 {print $1}' out/join-series.txt) -[ -n "$SERIES" ] || { echo "no bench_join series in the store" >&2; exit 1; } -bx history "$SERIES" > out/history.txt -cat out/history.txt -[ "$(grep -c example- out/history.txt)" -eq 2 ] || { echo "expected two history points for $SERIES" >&2; exit 1; } - -echo; echo "== 7. what was written" -find out/all out/env out/again -type f | sort | sed 's/^/ /' diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index e8c0774..2253bdc 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -75,8 +75,8 @@ environment kind: a context collector (§3) prints facts about the environment i under `environment.system.`, a part of the varying coordinate; a setup file does the same for declared facts. A collector with `verify: how_built` and `on_fail: warn` checks the facts a target's `how_built` object declares against what it found, and the comparison lists the warning beside the -facts that differ. `benchx/examples/numpy-venvs` is the worked example, with the map from UC-04's -spellings to these. +facts that differ. `examples/numpy-venvs` in benchx_examples is the worked example, with the map from +UC-04's spellings to these. A bare path is read by the harness. For Google Benchmark it is `BUILD_DIR[:SOURCE_DIR]`, a `build` target. For pyperf it is a working tree, a @@ -560,9 +560,10 @@ Allowing a local-only `revisions` comparison of a dirty tree is open Both profiles use one shape: two prepared build directories, one run key, R rounds, and `bx compare` in measure mode, which alternates the sides and -assigns `round` and `slot`. The demo script plays the user, and in that role +assigns `round` and `slot`. The demo script (`examples/arrow/demo/run.sh` in +the benchx_examples repository) plays the user, and in that role the target provider: it prepares the sources and build directories with git -and CMake, since benchx builds nothing. `benchx/examples/archery-local` +and CMake, since benchx builds nothing. `examples/arrow/archery-local` there shows the other route, where a provider script named in `benchx.conf.json` builds and describes the targets and the sides are given as `@plain` and `@hardened`. @@ -589,7 +590,7 @@ re-ingested to show idempotency, and compared again with `--results`; that document must equal the one from the local store (criterion 8). Every `bx run` delivers to the local store, so both shapes' results land in -the demo project's store (`benchx/examples/demo/store.parquet`), the ad hoc ones as thin results of project +the demo project's store (`examples/arrow/demo/store.parquet` in benchx_examples), the ad hoc ones as thin results of project `local/`. Run keys carry a timestamp, so successive demo runs add rows rather than colliding on `ingest_key`. The store and each series' history grow from run to run, and the demo ends with `bx head`.