diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2727c71..8ff9f74 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,3 +61,12 @@ jobs: if: runner.os != 'Windows' shell: bash run: uv run --extra test --python ${{ matrix.python-version }} bash examples/archery-local/run.sh + + # Makes two venvs of its own with uv (Python 3.12, the newest with numpy 1.26.4 wheels), so + # the matrix Python only runs the outer script and one cell is enough. Needs network access. + - name: Run the numpy-venvs example + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13' + shell: bash + run: >- + uv run --extra test --python ${{ matrix.python-version }} + bash examples/numpy-venvs/run.sh diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index 02f1aa1..8c94023 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -39,6 +39,7 @@ bx bench TARGET --suite NAME [CASE …] # the single-target form of bx compar [--adapter …] [--filter REGEX] [--quantity NAMES] [-e NAME=VALUE ...] [precision flags, as for bx compare] [--project P | --no-project] [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI] + [--label NAME=VALUE ... --round N --slot K] # one side of a session other calls join [--case-timeout S] [--no-ingest] bx bench MODULE.py [FUNCTION …] [--tree DIR] [...] # shorthand: the module's checkout, pyperf → result files → store → table @@ -51,13 +52,32 @@ target the same way, takes the same options, and goes through the same code: the side is resolved by `session._side` (a provider call for `@CONFIG`, else a bare path the harness reads; the adapter checks the suite before anything runs) and the order is built by `session._order`. A bench run is one order of a -comparison session with no round, no slot, and no comparison; it is run by the +comparison session with no comparison, and by default no round or slot; it is run by the same runner, delivered as `bx run` delivers (`--no-ingest`: files only), and summarized as a table of its cases. It is not `bx run`: that executes a finished order file and never resolves a target; `bx bench` turns options into an order (saved as `results/RUN_KEY/orders/order.json`, which `bx run` can replay) and then does what `bx run` does. +**A call as one side of a session (UC-04).** Environments that cannot share a process, such as two +venvs holding different numpy versions, are measured by separate `bx bench` calls that share a +`--run-key`; the caller's script is the loop (`benchmark-environments.md` §5.7). `--label NAME=VALUE` +names the call's side, `--round N` pairs its attempt with the other side's, and `--slot K` is its +position in the alternation, which the caller counts. Each call writes `orders/order-KKKK.json`. The +flags need an explicit `--run-key` (the default is a timestamp per call, so two calls would never +share a run), and `--label` and `--round` need `--slot` (the ingest key gets its `:slot-K` suffix only +when a slot is set, so two sides would otherwise share a key). `bx compare --run KEY --profile +environments --label NAME --baseline VALUE` then compares the sides, and refuses a run whose sides do +not alternate, have unequal attempt counts, or do not differ. + +What differs between the environments is recorded by the user's own commands, so benchx knows no +environment kind: a context collector (§3) prints facts about the environment it runs in, which land +under `environment.system.`, a part of the varying coordinate; a setup file does the same for +declared facts. A collector with `verify: how_built` and `on_fail: warn` checks the facts a target's +`how_built` object declares against what it found, and the comparison lists the warning beside the +facts that differ. `prototype/examples/numpy-venvs` is the worked example, with the map from UC-04's +spellings to these. + A bare path is read by the harness. For Google Benchmark it is `BUILD_DIR[:SOURCE_DIR]`, a `build` target. For pyperf it is a working tree, a `working_tree` target that is its own source, measured as found with its dirty diff --git a/prototype/README.md b/prototype/README.md index 1b58136..0b5ecab 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -27,6 +27,7 @@ bx bench TARGET --suite NAME [CASE …] # the single-target form of bx compar [--adapter …] [--filter REGEX] [--quantity …] [-e NAME=VALUE ...] [--processes N] [--values N] [--min-time S | --loops N] [--warmups N] [--repetitions N] [--project P] [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI] + [--label NAME=VALUE ... --round N --slot K] # one side of a session other calls join [--case-timeout S] [--no-ingest] bx bench MODULE.py [FUNCTION …] [--tree DIR] [...] # shorthand: Python functions under pyperf → results → store → table @@ -88,6 +89,12 @@ functions are exact case names. `bx run` is the layer underneath: it executes a order, so `bx run results/KEY/orders/order.json` replays any bench run. `examples/pyperf-bench` is a runnable walk-through. +Environments that cannot share a process, such as two venvs with different numpy versions, are +measured by separate `bx bench` calls that share a `--run-key`; `--label NAME=VALUE`, `--round N` +and `--slot K` place each call in the session, and `bx compare --run KEY --profile environments +--label NAME --baseline VALUE` compares the sides ([UC-04](../docs/use-cases/UC-04-cross-env.md)). +`examples/numpy-venvs` is the worked example, with a shell script that drives it. + `bx bench PATH --adapter asv` runs an [asv](https://asv.readthedocs.io) benchmark suite unchanged, reading asv's benchmark format as a language and timing it with pyperf; neither asv nor asv_runner is used. PATH is a directory holding `asv.conf.json` (its @@ -240,6 +247,7 @@ examples/demo-suite a two-benchmark Google Benchmark suite for the demo examples/demo the demo's project root: its benchx.conf.json, and its persistent store examples/pyperf-bench Python functions under `bx bench` (own README) examples/archery-local an end-to-end, Arrow-local-shaped example with a target provider (own README) +examples/numpy-venvs UC-04 with two venvs and two numpy versions, driven by run.sh (own README) tests/ test_prototype.py: the success criteria and the runner and store rules, on a fake Google Benchmark binary; test_comparator.py: the comparator's invariants and method on hand-built results; diff --git a/prototype/benchx/cli.py b/prototype/benchx/cli.py index 933c149..5d969fe 100644 --- a/prototype/benchx/cli.py +++ b/prototype/benchx/cli.py @@ -78,6 +78,31 @@ def _environment(items) -> dict: return env +def _session_position(args) -> tuple[dict, int | None, int | None]: + """Where `bx bench` sits in a session that other calls join through the same + `--run-key`: its `--label NAME=VALUE`s, `--round` and `--slot`. UsageError when + they could not join one.""" + labels = {} + for item in args.label or []: + name, eq, value = item.partition("=") + if not (name and eq and value): + raise UsageError(f"--label wants NAME=VALUE, not {item!r}") + if name in labels: + raise UsageError(f"--label {name} is given twice") + labels[name] = value + for flag, number in (("--round", args.round), ("--slot", args.slot)): + if number is not None and number < 0: + raise UsageError(f"{flag} wants a number from 0, not {number}") + placed = labels or args.round is not None or args.slot is not None + if placed and not args.run_key: + raise UsageError("--label, --round and --slot need an explicit --run-key: the default is a timestamp " + "per call, so two calls would never share a run") + if (labels or args.round is not None) and args.slot is None: + raise UsageError("--label and --round need --slot: without one, the sides of a run would share an " + "ingest key and their results would conflict") + return labels, args.round, args.slot + + def _bench_target(args) -> tuple[str, str | None, str | None]: """The target, the suite it forces (or None: --suite, then the file's), and the adapter it forces. With `--suite` the target is as given. Without it, the target @@ -155,6 +180,7 @@ def cmd_bench(args): raise UsageError("give case names or --filter, not both") target, forced_suite, forced_adapter = _bench_target(args) settings = _common(cfg, args, forced_adapter=forced_adapter, forced_suite=forced_suite) + settings["labels"], settings["round_"], settings["slot"] = _session_position(args) _needs_provider(cfg, target) except (UsageError, config_mod.ConfigError, bench_mod.SessionError) as e: print(f"bx bench: error: {e}", file=sys.stderr) @@ -493,6 +519,16 @@ def main(argv=None): p.add_argument("--case-timeout", type=float, help=f"seconds before one case is stopped (default: timeouts.case_s, else {orders.CASE_TIMEOUT_S:g}; " f"asv: {ASV_CASE_TIMEOUT_S}, all rounds, each also stopped at the benchmark's own timeout)") + p.add_argument("--label", action="append", metavar="NAME=VALUE", + help="name this call's side of a session, such as env=numpy-2.0 (repeatable); the calls of a " + "session share --run-key, and bx compare --run reads the label with --label NAME. " + "Needs --run-key and --slot") + p.add_argument("--round", type=int, + help="this call's round in a session: the sides' attempts are paired by it. " + "Needs --run-key and --slot") + p.add_argument("--slot", type=int, + help="this call's position in the session's alternation (0, 1, 2, ...), kept by the caller's loop. " + "Needs --run-key") p.add_argument("--no-ingest", action="store_true", help="only write result files; skip the local store") p.set_defaults(fn=cmd_bench) diff --git a/prototype/benchx/compare.py b/prototype/benchx/compare.py index 9de9e64..b9b33a5 100644 --- a/prototype/benchx/compare.py +++ b/prototype/benchx/compare.py @@ -109,7 +109,8 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou failed = doc["failed_invariants"] projected = [projection.apply(d, store_project(d)) for d in documents] if any(p.axis[0] is None for p in projected): - failed.append({"invariant": "sides", "detail": f"a result has no {varying}"}) + failed.append({"invariant": "sides", "detail": f"a result has no {varying}: give every bx bench call " + "--label NAME=VALUE, and name that label here"}) return doc # A side is a distinct axis value, named by its first part (a tree, a label). @@ -139,7 +140,9 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou a, b = (json.loads(side) for side in (base, contender)) if all(x == y for path, x, y in zip(projection.axis, a, b) if not path.startswith("/provenance/")): failed.append({"invariant": "sides-differ", - "detail": f"the sides differ only in {varying}"}) + "detail": f"the sides differ only in their label ({varying}): nothing recorded about the " + "environment differs. Describe what differs with a context collector " + "(collectors in benchx.conf.json) or a setup file for each environment"}) if "one-tree-per-side" in check: trees = {side: {d["provenance"].get("subject_tree") for d, x in zip(documents, sides) if x == side} for side in (base, contender)} @@ -157,13 +160,16 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou for d, s in zip(documents, sides): slot = d.get("procedure", {}).get("slot") if slot is None: - failed.append({"invariant": "alternating-slots", "detail": "a result has no slot"}) + failed.append({"invariant": "alternating-slots", + "detail": "a result has no slot: give every bx bench call --slot"}) break slots.setdefault(slot, set()).add(s) else: order = [slots[slot] for slot in sorted(slots)] if any(len(o) != 1 for o in order) or any(a == b for a, b in zip(order, order[1:])): - failed.append({"invariant": "alternating-slots", "detail": "sides do not alternate by procedure.slot"}) + failed.append({"invariant": "alternating-slots", + "detail": "sides do not alternate by procedure.slot: run the sides in the same " + "order every round (a, b, a, b, ...)"}) # Units: one workload variant and quantity each. Everything else the # projection matches must agree across both sides (schema §5.5). @@ -191,7 +197,8 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou continue if "equal-attempts" in check and len(by_side[base]) != len(by_side[contender]): failed.append({"invariant": "equal-attempts", **where, - "detail": f"{len(by_side[base])} against {len(by_side[contender])}"}) + "detail": f"{len(by_side[base])} against {len(by_side[contender])}: run the same " + "number of rounds on both sides"}) continue rounds = {} for s in (base, contender): @@ -206,6 +213,8 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou if base in r and contender in r] unit_doc = _unit(where, pairs, by_side, base, contender, k, min_rounds) unit_doc["unmatched_differences"] = _differences(projection, by_side[base], by_side[contender]) + if caveats := _caveats(by_side, base, contender): + unit_doc["caveats"] = caveats doc["units"].append(unit_doc) return doc @@ -247,6 +256,16 @@ def _declared(document) -> dict: return side +def _caveats(by_side, base, contender) -> list[str]: + """The warnings a side's results carry (quality.warnings), by side, for the reader.""" + found = set() + for role, side in (("baseline", base), ("contender", contender)): + for d in by_side[side]: + for w in d.get("quality", {}).get("warnings", []): + found.add(f"{role}: {w.get('code', w) if isinstance(w, dict) else w}") + return sorted(found) + + def _unit(where, pairs, by_side, base, contender, k, min_rounds): first = {s: by_side[s][0] for s in (base, contender)} quantity = first[base]["measurement"]["quantity"] @@ -310,6 +329,13 @@ def render(doc: dict) -> str: shown = side[key] if key != "observed_env" else ", ".join(f"{k}={v}" for k, v in side[key].items()) first, *rest = str(shown).splitlines() or [""] lines.append(f" {name}: {key.replace('_', ' ')}: {first}" + (f" (+{len(rest)} lines)" if rest else "")) + differs = sorted({p for u in doc["units"] for p in u.get("unmatched_differences", [])}) + if differs: + shown = ", ".join(_dotted(p) for p in differs[:6]) + lines.append(f" differs: {shown}" + (f" (+{len(differs) - 6} more)" if len(differs) > 6 else "")) + caveats = sorted({c for u in doc["units"] for c in u.get("caveats", [])}) + if caveats: + lines.append(f" caveats: {', '.join(caveats)}") for section in ("regressed", "improved", "changed", "no-change-detected", "indeterminate"): rows = [u for u in doc["units"] if u["verdict"] == section] if not rows: diff --git a/prototype/benchx/session.py b/prototype/benchx/session.py index 45cd90b..317cd42 100644 --- a/prototype/benchx/session.py +++ b/prototype/benchx/session.py @@ -177,9 +177,11 @@ def bench_target(target: str, *, suite: str, adapter: str, store=None, flags=Non environment_variables=None, filter=None, quantities=("wall-time",), project=None, run_key=None, out=None, source_uri=None, case_s=orders.CASE_TIMEOUT_S, order_s=orders.ORDER_TIMEOUT_S, progress=None, build=True, provider=None, config_path=None, results_dir=None, - collectors=None) -> dict: - """Run one target once: a comparison session of one order, with no round, - slot or comparison. Results are delivered to `store` unless it is None. + collectors=None, round_=None, slot=None, labels=None) -> dict: + """Run one target once: a comparison session of one order, with no comparison. + It has a round, a slot and labels when the caller places it in a session that + other calls join through the same run key. Results are delivered to `store` + unless it is None. `target` is `@CONFIG[=SOURCE_REF]` or a bare path, which the harness reads (see `_side`); `adapter`, `flags` and the rest are as for `compare_targets`. @@ -195,10 +197,12 @@ def bench_target(target: str, *, suite: str, adapter: str, store=None, flags=Non out = Path(out) if out else Path(results_dir or "results") / run_key.replace("/", "_") if progress: progress(f"precision: {harness.describe_precision(precision, config_path)}") - summary, outcome = _run_side(side, out / "orders" / "order.json", store, harness=harness, suite=suite, + order_file = "order.json" if slot is None else f"order-{slot:04d}.json" # one per slot, as compare_targets + summary, outcome = _run_side(side, out / "orders" / order_file, store, harness=harness, suite=suite, filter=filter, quantities=quantities, precision=precision, environment_variables=environment_variables, project=project, run_key=run_key, - source_uri=source_uri, case_s=case_s, order_s=order_s, collectors=collectors) + source_uri=source_uri, case_s=case_s, order_s=order_s, collectors=collectors, + round_=round_, slot=slot, labels=labels) return {**summary, "outcome": outcome, "precision": precision} diff --git a/prototype/benchx/snapshot.py b/prototype/benchx/snapshot.py index ac8278d..8168de9 100644 --- a/prototype/benchx/snapshot.py +++ b/prototype/benchx/snapshot.py @@ -124,13 +124,22 @@ def _ignored_in(root: Path) -> list[str]: return [p.relative_to(root).as_posix() for p in _IGNORED if p != root and p.is_relative_to(root)] +def _git_ignores(top, rel) -> bool: + """Whether the repository's own ignore rules already leave `rel` out of `git add -A`. + Naming such a path in an exclude pathspec makes `git add` fail, and the tree id + with it, so a results directory the project git-ignores must not be excluded twice.""" + return subprocess.run(["git", "check-ignore", "-q", "--", rel], cwd=top, + capture_output=True).returncode == 0 + + def git_identity(path) -> dict | None: """Revision, dirty state, and working-tree id of a checkout (schema §4.1).""" top = _run(["git", "rev-parse", "--show-toplevel"], cwd=path) head = top and _run(["git", "rev-parse", "HEAD"], cwd=top) if not head: return None - excludes = [_PYCACHE_EXCLUDE, *(f":(top,exclude){rel}" for rel in _ignored_in(top))] + excludes = [_PYCACHE_EXCLUDE, *(f":(top,exclude){rel}" for rel in _ignored_in(top) + if not _git_ignores(top, rel))] status = _run(["git", "status", "--porcelain", "--untracked-files=all", "--", ".", *excludes], cwd=top) # A scratch index that does not exist yet, so the repository's own index # is untouched (schema §4.1). diff --git a/prototype/examples/numpy-venvs/.gitignore b/prototype/examples/numpy-venvs/.gitignore new file mode 100644 index 0000000..388f608 --- /dev/null +++ b/prototype/examples/numpy-venvs/.gitignore @@ -0,0 +1,2 @@ +.venv-*/ +out/ diff --git a/prototype/examples/numpy-venvs/README.md b/prototype/examples/numpy-venvs/README.md new file mode 100644 index 0000000..2425bc7 --- /dev/null +++ b/prototype/examples/numpy-venvs/README.md @@ -0,0 +1,102 @@ +# numpy-venvs: UC-04 with two virtual environments + +The same code, measured under two numpy versions, by separate `bx bench` calls that alternate +round by round, then compared. This is +[UC-04](../../../docs/use-cases/UC-04-cross-env.md), "fixed branch, comparing environments": +`numpy==1.26.4` and `numpy==2.0.2` cannot be imported in one process, so each side runs in its +own environment, and a shell script drives the loop. + +You prepare the environments and enter them; benchx builds nothing and activates nothing. + +## Run it + +You need [`uv`](https://docs.astral.sh/uv/) (it fetches Python 3.12, the newest Python with numpy +1.26.4 wheels) and network access the first time. Then, from this directory: + +```console +$ ./run.sh +``` + +The versions (`NUMPY_A`, `NUMPY_B`), the Python and the number of rounds are set at the top of +`run.sh`. Everything it creates is in `.venv-a`, `.venv-b` and `out/`, all git-ignored. + +## What it does + +1. **Prepares two venvs**, `.venv-a` and `.venv-b`, each with its numpy, pyperf and benchx, and + `pyarrow<25`: benchx imports pyarrow, and pyarrow 25 refuses to import under numpy 1.x. +2. **Runs the caller's loop.** For each of the 3 rounds, one `bx bench` call per side, each in its + venv (`run.sh` sources the venv's `activate`; activating the other venv replaces it, so nothing + needs deactivating). The calls share a `--run-key`; each says which side it is + (`--label env=numpy-1.26.4`), which round, and which slot of the alternation. +3. **Compares.** `bx compare --run S --profile environments --label env --baseline numpy-1.26.4`. + The sides differ because the collector read a different numpy in each. The comparison says + what differed (`differs: ...deps.numpy`), shows what each side declared, and gives a verdict per + benchmark: + + ``` + run numpy-1793121294 profile environments baseline numpy-2.3.4 contender numpy-2.4.3 + numpy-2.3.4: how built: {"summary":{"numpy":"2.3.4"}} + numpy-2.4.3: how built: {"summary":{"numpy":"2.4.3"}} + differs: coordinates.environment.system.deps.numpy, coordinates.environment.system.deps.sha256 + + NO CHANGE DETECTED (3) + bench_matmul wall-time effect -16.28% noise 7.43% rounds=3 + ... + ``` + + (The numbers vary from run to run; `[local-only]` appears while the demo's files are + uncommitted.) +4. **Checks a declaration.** It declares numpy 2.0.2 for a run in the venv that has 1.26.4. The + collector disagrees, so every result of that run carries `collector-mismatch`. The run goes + ahead, because the collector has `on_fail: warn`, and the recorded numpy is the one found. + +## How the sides are told apart + +- `describe_env.py` is a **context collector** (`harness-adapter.md` §5.1), named in + `benchx.conf.json`. benchx runs it once per call, in the environment the call runs in. It prints + `{"summary": {"numpy": ""}}`, and that lands in each result's `environment.system.deps`, + which the `environments` comparison treats as the varying coordinate. benchx knows nothing about + venvs, conda or pixi: any command that prints facts about an environment does the same. +- `provider.py` is the **target provider**. For configuration `a` or `b` it declares, in + `how_built`, the numpy that side is meant to have. The collector has `verify: how_built`, so it + compares the declaration with what it found. The declared object has the same shape as the + collector's output, because the check compares key by key from the top. +- A check that fails refuses the run by default; `on_fail: warn` records a warning instead, which + is what the design asks for (a mismatch is recorded, not a gate). + +## UC-04's spellings, and the prototype's + +[UC-04](../../../docs/use-cases/UC-04-cross-env.md) is kept as written, and uses spellings the +prototype does not have. This is how each corresponds: + +- `benchx new-session` (`SESSION=$(...)`) → no command. The session is the run key, chosen by the + caller: `S=numpy-$(date +%s)`. +- `--session-id "$SESSION"` → `--run-key "$S"`. It lands in `provenance.run_key`. +- `benchx run run_benchmark.py` → `bx bench`. With a module and functions, + `bx bench MODULE.py [FUNCTION ...]`; with a prepared target, `bx bench @CONFIG --suite NAME`. + `bx run` takes an order file, not a script. +- `--env-label numpy-1.26.4` → `--label env=numpy-1.26.4`. The field `env_label` is + `provenance.labels.env`. +- `--round "$round"` → `--round N` and `--slot K`. UC-04 uses one number for two things. The + prototype separates the round, which pairs one attempt of each side (`procedure.round`), from + the slot, the realized position in the interleaved sequence (`procedure.slot`). UC-04's + `interleave_position` is the slot. The caller's loop keeps the slot counter. +- `benchx compare-environments --session-id S --baseline A --variant B` → + `bx compare --run S --profile environments --label env --baseline A`. The profile is UC-04's + `cross-environment-comparison`. There are exactly two sides, so the variant is the other one. +- `.venv-baseline/bin/benchx` → the `bx` of that environment, such as `.venv-a/bin/bx`. +- The output table (`benchmark_name`, `env`, `time`, `vs. baseline`) → the comparison output: + verdict groups with the effect, the noise and the rounds, and the facts that differ. +- UC-04's `session_id`, `env_label` and `interleave_position` fields → `provenance.run_key`, + `provenance.labels.env` and `procedure.slot`. The schema adds no fields for a profile + (`benchmark-result-schema.md` §5.5). + +## What is in this directory + +| File | Role | +|---|---| +| `run.sh` | the driver: venvs, the loop, the comparison, the check | +| `benchx.conf.json` | the project file: the adapter and suite, the provider, the `deps` collector | +| `provider.py` | the target provider: declares each side's numpy in `how_built` (`run.sh` exports the versions) | +| `describe_env.py` | the collector: prints the numpy this Python imports | +| `bench_suite.py` | three small numpy benchmarks | diff --git a/prototype/examples/numpy-venvs/bench_suite.py b/prototype/examples/numpy-venvs/bench_suite.py new file mode 100644 index 0000000..866f15a --- /dev/null +++ b/prototype/examples/numpy-venvs/bench_suite.py @@ -0,0 +1,20 @@ +"""Three small numpy benchmarks. pyperf calls each bench_* function many times and times the +calls, so the arrays are made once, at module level.""" + +import numpy as np + +_rng = np.random.default_rng(0) +_square = _rng.standard_normal((200, 200)) +_vector = _rng.standard_normal(100_000) + + +def bench_matmul(): + _square @ _square + + +def bench_sort(): + np.sort(_vector) + + +def bench_sum(): + _vector.sum() diff --git a/prototype/examples/numpy-venvs/benchx.conf.json b/prototype/examples/numpy-venvs/benchx.conf.json new file mode 100644 index 0000000..94146f8 --- /dev/null +++ b/prototype/examples/numpy-venvs/benchx.conf.json @@ -0,0 +1,22 @@ +{ + "schema_version": "benchx/config/0.1.0", + "project": "numpy-venvs", + "adapter": { + "name": "pyperf", + "suite": "bench_suite.py", + "precision": { + "repetitions": { "mode": "fixed", "levels": [{ "unit": "process", "n": 3 }, { "unit": "value", "n": 3 }] }, + "calibration": { "mode": "adaptive", "minimum_sample_seconds": 0.02 }, + "warmup": { "mode": "count", "n_warmup": 1 } + } + }, + "provider": { "command": ["./provider.py"] }, + "collectors": { + "deps": { + "command": ["python", "{root}/describe_env.py"], + "verify": "how_built", + "on_fail": "warn" + } + }, + "paths": { "store": "out/store.parquet", "results": "out" } +} diff --git a/prototype/examples/numpy-venvs/describe_env.py b/prototype/examples/numpy-venvs/describe_env.py new file mode 100644 index 0000000..7363bbb --- /dev/null +++ b/prototype/examples/numpy-venvs/describe_env.py @@ -0,0 +1,12 @@ +"""The collector: print what the numpy that this Python imports says about itself. + +benchx.conf.json runs it once per call, in the environment the call runs in, so it can only +differ between the two sides if their environments do. It prints the same text for the same +environment: no paths, no timestamps. +""" + +import json + +import numpy + +print(json.dumps({"summary": {"numpy": numpy.__version__}}, sort_keys=True)) diff --git a/prototype/examples/numpy-venvs/provider.py b/prototype/examples/numpy-venvs/provider.py new file mode 100755 index 0000000..2fdbd5f --- /dev/null +++ b/prototype/examples/numpy-venvs/provider.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""The target provider of this example (system-decomposition.md §3.4): one JSON request on stdin, +one target description on stdout. It builds nothing, so `describe` and `prepare` answer alike. + +Configuration `a` or `b` names a side. The description declares, in how_built, the numpy the +user means that side to have, in the shape describe_env.py prints. The collector in +benchx.conf.json compares the two: a side whose environment is not the one it was declared +with gets the warning collector-mismatch on every result. +""" + +import json +import os +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +# The canonical URI this working tree's results are filed under, as in archery-local. +URI = "https://example.org/numpy-venvs/demo.git" + + +def main(): + versions = {"a": os.environ.get("NUMPY_A"), "b": os.environ.get("NUMPY_B")} + if not all(versions.values()): + sys.exit("provider: run.sh sets NUMPY_A and NUMPY_B") + + # An absent or null configuration is refused below, like an unknown one. + configuration = json.load(sys.stdin).get("configuration") + if configuration not in versions: + print(f"provider: unknown configuration {configuration!r} (a or b)", file=sys.stderr) + sys.exit(2) + + json.dump({ + "schema_version": "benchx/target-description/0.1.0", + "target": { + "kind": "working_tree", + "path": str(HERE), + "source": {"uri": URI, "type": "git"}, + }, + "how_built": {"summary": {"numpy": versions[configuration]}}, + "provider": {"name": "provider.py", "version": "1"}, + }, sys.stdout, indent=1) + print() + + +main() diff --git a/prototype/examples/numpy-venvs/run.sh b/prototype/examples/numpy-venvs/run.sh new file mode 100755 index 0000000..7b67bf2 --- /dev/null +++ b/prototype/examples/numpy-venvs/run.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# UC-04 with two venvs: the same code under two numpy versions, measured by separate `bx bench` +# calls that alternate round by round, then compared. See README.md for what each step shows. +# +# Needs `uv`. Everything it creates is here: .venv-a, .venv-b and out/ (git-ignored). +# benchx builds nothing. +set -euo pipefail +cd "$(dirname "$0")" + +# provider.py reads the numpy versions from the environment. numpy 1.26.4 has no wheels beyond +# Python 3.12, so both venvs use it. Three rounds are the fewest that give a verdict. +export NUMPY_A=1.26.4 NUMPY_B=2.0.2 +PYTHON_VERSION=3.12 +ROUNDS=3 +SIDES=("a $NUMPY_A" "b $NUMPY_B") +RUN_KEY=numpy-$(date +%s) +SLOT=0 + +rm -rf out .venv-a .venv-b +mkdir -p out + +echo "== 1. the user prepares both environments; benchx builds nothing" +for SIDE_VERSION in "${SIDES[@]}"; do + read -r SIDE VERSION <<<"$SIDE_VERSION" + uv venv -q --python "$PYTHON_VERSION" ".venv-$SIDE" + # benchx imports pyarrow, and pyarrow 25 refuses to import under numpy 1.x. + uv pip install -q --python ".venv-$SIDE/bin/python" "numpy==$VERSION" "pyarrow<25" pyperf -e ../.. + echo " .venv-$SIDE: numpy $VERSION" +done + +echo "== 2. the caller's loop: alternate the sides, round by round, one slot per call" +for ROUND in $(seq 0 $((ROUNDS - 1))); do + for SIDE_VERSION in "${SIDES[@]}"; do + read -r SIDE VERSION <<<"$SIDE_VERSION" + # Activating a venv replaces the one activated before; no deactivate is needed. + source ".venv-$SIDE/bin/activate" + bx bench "@$SIDE" --run-key "$RUN_KEY" --label "env=numpy-$VERSION" \ + --round "$ROUND" --slot "$SLOT" --out "out/$RUN_KEY" > /dev/null + echo " round $ROUND, slot $SLOT: side $SIDE" + SLOT=$((SLOT + 1)) + done +done + +echo "== 3. compare: the sides differ in the numpy the collector read" +bx compare --run "$RUN_KEY" --profile environments --label env \ + --baseline "numpy-$NUMPY_A" | tee out/compare.txt +grep -q 'differs:.*deps\.numpy' out/compare.txt # the differing fact is shown +if grep -q 'failed invariants' out/compare.txt; then echo "the comparison was refused" >&2; exit 1; fi + +echo "== 4. the check: declare numpy $NUMPY_B for a run in the venv that has $NUMPY_A" +source .venv-a/bin/activate +bx bench @b --run-key "$RUN_KEY-mismatch" --label env=wrong \ + --round 0 --slot 0 --out "out/$RUN_KEY-mismatch" > /dev/null +grep -rq collector-mismatch "out/$RUN_KEY-mismatch" +echo " the results carry collector-mismatch; the run went ahead, with the numpy it found" diff --git a/prototype/tests/test_bench.py b/prototype/tests/test_bench.py index 6aa6d64..15e03c3 100644 --- a/prototype/tests/test_bench.py +++ b/prototype/tests/test_bench.py @@ -159,3 +159,45 @@ def test_source_uri_flag_overrides_the_remote(tree): "https://example.org/r.git", *FAST]) == cli.EXIT_OK order = json.loads((tree / "results" / "uri" / "orders" / "order.json").read_text()) assert order["target"]["source"]["uri"] == "https://example.org/r.git" + + +def test_bench_places_calls_in_a_session(tree): + """--label, --round and --slot make separate `bx bench` calls the sides of one session.""" + sides = [("a", 0), ("b", 0), ("a", 1), ("b", 1)] + for slot, (side, round_) in enumerate(sides): + assert cli.main(["bench", "mod.py", "bench_a", "--run-key", "S", "--label", f"env={side}", + "--round", str(round_), "--slot", str(slot), "--no-ingest", *FAST]) == cli.EXIT_OK + out = tree / "results" / "S" + assert sorted(p.name for p in (out / "orders").glob("*.json")) == [f"order-{n:04d}.json" for n in range(4)] + found = [] + for path in out.glob("*.json"): + document = json.loads(path.read_text()) + if path.name.startswith("workorder-") or document["measurement"].get("kind") == "condition": + continue + assert core.check(document, "measurement-result") == [] + found.append((document["procedure"]["slot"], document["procedure"]["round"], + document["provenance"]["labels"])) + assert sorted(found, key=lambda f: f[0]) == [(0, 0, {"env": "a"}), (1, 0, {"env": "b"}), + (2, 1, {"env": "a"}), (3, 1, {"env": "b"})] + + +@pytest.mark.parametrize("extra, message", [ + (["--label", "env=a", "--round", "0", "--slot", "0"], "explicit --run-key"), + (["--slot", "0"], "explicit --run-key"), + (["--run-key", "S", "--label", "env=a"], "need --slot"), + (["--run-key", "S", "--round", "0"], "need --slot"), + (["--run-key", "S", "--slot", "0", "--label", "env"], "NAME=VALUE"), + (["--run-key", "S", "--slot", "0", "--label", "env=a", "--label", "env=b"], "given twice"), + (["--run-key", "S", "--slot", "-1"], "from 0"), +]) +def test_session_flags_are_refused_before_anything_runs(extra, message, tree, capsys): + assert cli.main(["bench", "mod.py", "bench_a", "--no-ingest", *extra, *FAST]) == cli.EXIT_REFUSED + assert message in capsys.readouterr().err + assert not (tree / "results").exists() + + +def test_bench_without_session_flags_is_unchanged(tree): + assert cli.main(["bench", "mod.py", "bench_a", "--run-key", "plain", "--no-ingest", *FAST]) == cli.EXIT_OK + assert (tree / "results" / "plain" / "orders" / "order.json").exists() + order = json.loads((tree / "results" / "plain" / "orders" / "order.json").read_text()) + assert not {"round", "slot", "labels"} & set(order) diff --git a/prototype/tests/test_config_schema.py b/prototype/tests/test_config_schema.py index 0169bc4..a1a1c64 100644 --- a/prototype/tests/test_config_schema.py +++ b/prototype/tests/test_config_schema.py @@ -100,3 +100,17 @@ def test_a_plain_directory_tree_id_skips_ignored_paths(tmp_path): assert snapshot.directory_identity(root)["tree"] != before snapshot.ignore(root / "results", root / "store.parquet") assert snapshot.directory_identity(root)["tree"] == before + + +def test_a_results_directory_the_project_git_ignores_still_gives_a_tree_id(checkout): + """Excluding a path that .gitignore already ignores used to make `git add` fail, so the tree id was unknown.""" + (checkout / ".gitignore").write_text("results/\n") + git(checkout, "add", "-A") + git(checkout, "commit", "-qm", "ignore results") + clean = snapshot.git_identity(checkout) + write_outputs(checkout) + snapshot.ignore(checkout / "results", checkout / "store.parquet") + again = snapshot.git_identity(checkout) + assert (again["dirty"], again["tree"]) == ("clean", clean["tree"]) + snapshot.ignore(checkout / "results") # the store file is untracked and not ignored: a real difference + assert snapshot.git_identity(checkout)["dirty"] == "dirty" diff --git a/prototype/tests/test_uc04.py b/prototype/tests/test_uc04.py new file mode 100644 index 0000000..4c0fdd8 --- /dev/null +++ b/prototype/tests/test_uc04.py @@ -0,0 +1,131 @@ +"""UC-04: two environments measured by separate `bx bench` calls, then compared. + +The two environments are emulated in process: an environment variable stands for +what differs between two venvs, and a context collector (or a setup file) reads it, +as it would read a package version. The session flags are `bx bench`'s --run-key, +--label, --round and --slot; the comparison is `bx compare --run`. +""" + +import json +import sys +import textwrap + +import pytest + +pytest.importorskip("pyperf") + +from benchx import cli + +MODULE = "def bench_a():\n sum(range(100))\n" +FAST = ["--processes", "2", "--values", "2", "--loops", "20", "--no-project"] +COLLECT = textwrap.dedent(''' + import json, os + print(json.dumps({"summary": {"dep": os.environ.get("FAKE_DEP", "unset")}})) +''') + + +@pytest.fixture +def tree(tmp, monkeypatch): + path = tmp / "tree" + path.mkdir() + (path / "mod.py").write_text(MODULE) + monkeypatch.chdir(path) + return path + + +def write_config(tree, collector=None): + config = {"schema_version": "benchx/config/0.1.0", "adapter": {"name": "pyperf"}} + if collector is not None: + config["collectors"] = {"deps": {"command": [sys.executable, "-c", COLLECT], **collector}} + (tree / "benchx.conf.json").write_text(json.dumps(config)) + + +def call(side, round_, slot, *, dep=None, setup=None, monkeypatch, run_key="S"): + """One `bx bench` call, as one side of the session, in the environment `dep` or `setup` stand for.""" + if dep is not None: + monkeypatch.setenv("FAKE_DEP", dep) + if setup is not None: + monkeypatch.setenv("BENCHX_SETUP_FILE", str(setup)) + argv = ["bench", "mod.py", "bench_a", "--run-key", run_key, "--label", f"env={side}", + "--round", str(round_), "--slot", str(slot), *FAST] + assert cli.main(argv) == cli.EXIT_OK + + +def loop(sides, monkeypatch, rounds=3): + """The caller's loop: alternate the sides round by round, keeping the slot counter.""" + slot = 0 + for round_ in range(rounds): + for side, environment in sides.items(): + call(side, round_, slot, monkeypatch=monkeypatch, **environment) + slot += 1 + + +def compare(capsys, baseline="a", run_key="S"): + code = cli.main(["compare", "--run", run_key, "--profile", "environments", "--label", "env", + "--baseline", baseline]) + return code, capsys.readouterr().out + + +def test_a_collector_tells_the_sides_apart(tree, monkeypatch, capsys): + """The sides differ in what the collector reads, so the comparison goes ahead and says what differs.""" + write_config(tree, {}) + loop({"a": {"dep": "1.0"}, "b": {"dep": "2.0"}}, monkeypatch) + capsys.readouterr() + code, out = compare(capsys) + assert code == cli.EXIT_OK + assert "failed invariants" not in out + assert "differs:" in out and "coordinates.environment.system.deps" in out + assert "bench_a" in out + + +def test_a_setup_file_per_environment_tells_the_sides_apart(tree, tmp, monkeypatch, capsys): + write_config(tree) + files = {} + for side, version in (("a", "1.0"), ("b", "2.0")): + files[side] = tmp / f"setup-{side}.json" + files[side].write_text(json.dumps({"dep": version})) + loop({"a": {"setup": files["a"]}, "b": {"setup": files["b"]}}, monkeypatch) + capsys.readouterr() + code, out = compare(capsys) + assert code == cli.EXIT_OK + assert "failed invariants" not in out and "coordinates.environment.system.setup" in out + + +def test_identical_sides_are_refused_with_advice(tree, monkeypatch, capsys): + write_config(tree, {}) + call("a", 0, 0, dep="1.0", monkeypatch=monkeypatch) + call("b", 0, 1, dep="1.0", monkeypatch=monkeypatch) + capsys.readouterr() + _, out = compare(capsys) + assert "sides-differ" in out and "collector" in out and "setup file" in out + + +def test_sides_that_do_not_alternate_are_refused_with_advice(tree, monkeypatch, capsys): + write_config(tree, {}) + for slot, (side, round_, dep) in enumerate([("a", 0, "1.0"), ("a", 1, "1.0"), ("b", 0, "2.0"), ("b", 1, "2.0")]): + call(side, round_, slot, dep=dep, monkeypatch=monkeypatch) + capsys.readouterr() + _, out = compare(capsys) + assert "alternating-slots" in out and "same order every round" in out + + +def test_a_label_missing_on_a_call_is_refused_with_advice(tree, monkeypatch, capsys): + write_config(tree, {}) + call("a", 0, 0, dep="1.0", monkeypatch=monkeypatch) + capsys.readouterr() + cli.main(["compare", "--run", "S", "--profile", "environments", "--label", "env_name", "--baseline", "a"]) + out = capsys.readouterr() + assert "provenance.labels.env_name" in out.out + out.err + assert "--label NAME=VALUE" in out.out + out.err + + +def test_a_failed_check_is_shown_in_the_comparison_and_changes_nothing_else(tree, monkeypatch, capsys): + """expect, with on_fail: warn, flags the side that disagrees; the run goes ahead, the verdict is unchanged.""" + write_config(tree, {"expect": {"summary": {"dep": "1.0"}}, "on_fail": "warn"}) + loop({"a": {"dep": "1.0"}, "b": {"dep": "2.0"}}, monkeypatch) + capsys.readouterr() + code, out = compare(capsys) + assert code == cli.EXIT_OK + assert "caveats: contender: collector-mismatch" in out + assert "baseline: collector-mismatch" not in out + assert "failed invariants" not in out and "bench_a" in out