diff --git a/docs/design/prototype-design.md b/docs/design/prototype-design.md index d40dfed..6ae9b60 100644 --- a/docs/design/prototype-design.md +++ b/docs/design/prototype-design.md @@ -25,24 +25,25 @@ bx compare --run KEY --profile revisions|environments --baseline VALUE # read mode: compare results already measured bx target describe|prepare TARGET # a target's description (a provider's answer, or a build directory's sidecar) bx compare BASELINE CONTENDER --profile revisions|environments - [--adapter pyperf|google-benchmark] # default pyperf - --suite NAME [--filter REGEX] [--quantity wall-time] + [--adapter pyperf|google-benchmark] # default: adapter.name + --suite NAME [--filter REGEX] [--quantity NAMES] # default: adapter.suite, adapter.quantities --rounds R [-e NAME=VALUE ...] [--repetitions N] # google-benchmark [--processes N] [--values N] [--loops N] [--warmups N] # pyperf [--min-time S] - [--project P] [--label NAME=BASE,CONTENDER] + [--project P | --no-project] [--label NAME=BASE,CONTENDER] [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI] [--k 3] [--min-rounds 3] [--json] # measure mode: run both sides, then compare bx bench TARGET --suite NAME [CASE …] # the single-target form of bx compare: one order, a table - [--adapter …] [--filter REGEX] [--quantity …] [-e NAME=VALUE ...] + [--adapter …] [--filter REGEX] [--quantity NAMES] [-e NAME=VALUE ...] [precision flags, as for bx compare] - [--project P] [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI] + [--project P | --no-project] [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI] [--case-timeout S] [--no-ingest] bx bench MODULE.py [FUNCTION …] [--tree DIR] [...] # shorthand: the module's checkout, pyperf → result files → store → table -bx config init [--dir DIR] # write .benchx/config.json stating pyperf's defaults +bx config init [--dir DIR] [--adapter …] # write a benchx.conf.json stating a harness's defaults +bx config show # what benchx.conf.json and the defaults give, and where each value comes from ``` `bx bench` is the single-target form of `bx compare` (measure mode). It names a @@ -64,9 +65,8 @@ state and tree id recorded; `--suite` is the module's path in it, and a case is a function in it (`bench_*` and `benchmark_*` by default). A case named on the command line is an exact name; `--filter` is a regex; not both. `-e` fills `environment_variables`. Precision is resolved before the order is written, -flags over the harness's defaults and, for pyperf, over `.benchx/config.json` -(found by walking up from the current directory), so the order always states -what applied; a flag the chosen harness does not take is refused, as in +flags over `adapter.precision` in `benchx.conf.json` over the harness's defaults, so the +order always states what applied; a flag the chosen harness does not take is refused, as in `bx compare`. The pyperf adapter takes `python` from the `PATH` the workers start with and refuses the order if pyperf is not importable there. Both commands default to pyperf. @@ -76,6 +76,26 @@ is the git checkout containing the module (or `--tree`), the suite is the module's path in it, and the functions are exact case names. Without `--suite`, a TARGET ending in `.py` selects it. +**The project file.** Both commands read `benchx.conf.json` in the project root, found by +walking up from the working directory; its directory is the root, and with no file the +root is the working directory (`schemas/config/0.1.0`). It holds `project`, `source.uri`, +`provider.command`, an `adapter` block (`name`, `suite`, `quantities`, `precision`), +`environment_variables`, `timeouts`, `paths.store` and `paths.results` (default +`store.parquet` and `results/` in the root, and relative to the root), and `compare` +(`profile`, `rounds`, `k`, `min_rounds`). A command-line argument wins; if it is missing the +value is read from the file; if that is missing too, a built-in default applies, where one +exists. A required value with none of the three (`adapter.name`, `adapter.suite`, the +target, and for `bx compare` the profile and the rounds) is a hard failure that names the +flag and the key. `--adapter` naming a harness other than the file's is refused, since the +file's suite and precision belong to its own. `--quantity` takes a comma-separated list +and one run collects all of them. `--no-project` makes the run ad hoc, filed in the store +as `local/`, whatever the file's `project` says. The provider is needed only by an +`@CONFIG` target. The store and the results directory are left out of the identity of a +measured tree they sit in, so a run's own output does not mark it dirty. The earlier +`.benchx/provider.json`, `.benchx/config.json` and the `~/.benchx` home directory are gone and not read (only `$BENCHX_HOME` survives, as a store override); +`.benchx/setup.json` (declared environment facts, §3) is a separate file. +`schemas/config/0.1.0/examples` has recommended settings for each harness. + `bx compare` has two modes, chosen by its arguments. Two positional targets (a path read by the harness, or `@CONFIG[=SOURCE_REF]`, a call to the project's target provider) mean measure mode; `--run` means read mode; giving @@ -99,9 +119,11 @@ so the workbench, a CI script, or a notebook can do the same without a subprocess. Measure mode is a call into the session function, which calls the runner for each order and then the comparator. -The local store has one place, `~/.benchx/store.parquet`, and no command -takes another; `$BENCHX_HOME` relocates benchx's home as a whole, for tests -and CI. `bx run` delivers what it measures there: it writes the result files, +The local store has one place per project, `store.parquet` in the project root +unless `paths.store` in `benchx.conf.json` says otherwise; no command takes another. +`$BENCHX_HOME`, when set, relocates the store to `$BENCHX_HOME/store.parquet` and wins over +`paths.store` (for tests, CI and keeping a demo out of your real store). +`bx run` delivers what it measures there: it writes the result files, then ingests exactly those files. The `runner` module itself stays store-unaware (`runner.md` §2 principle 6); the command composes it with ingest, which is the delivery step of `runner.md` §4 and what the workbench @@ -312,7 +334,7 @@ prototype records the tree as it is at run time and does not detect staleness ## 4. Store (`bx ingest` + queries) -The local store is one Parquet file, always `~/.benchx/store.parquet` (§1); +The local store is one Parquet file, the project's `paths.store` (§1); the one other store is the temporary file `bx compare --results` compares through. There are no side files: no JSON, no separate entity tables. Each row is one result, the @@ -496,7 +518,7 @@ assigns `round` and `slot`. The demo script plays the user, and in that role the target provider: it prepares the sources and build directories with git and CMake, since benchx builds nothing. `prototype/examples/archery-local` shows the other route, where a provider script named in -`.benchx/provider.json` builds and describes the targets and the sides are +`benchx.conf.json` builds and describes the targets and the sides are given as `@plain` and `@hardened`. - **Revisions:** two git worktrees of one source at the base and head @@ -521,7 +543,7 @@ re-ingested to show idempotency, and compared again with `--results`; that document must equal the one from the local store (criterion 8). Every `bx run` delivers to the local store, so both shapes' results land in -`~/.benchx/store.parquet`, the ad hoc ones as thin results of project +the demo project's store (`prototype/examples/demo/store.parquet`), the ad hoc ones as thin results of project `local/`. Run keys carry a timestamp, so successive demo runs add rows rather than colliding on `ingest_key`. The store and each series' history grow from run to run, and the demo ends with `bx head`. @@ -534,7 +556,7 @@ history grow from run to run, and the demo ends with `bx head`. `environment_variables` and defers the rest of the environment policy, and it refuses an order that names one rather than ignoring it (§3). - The target-provider hook is minimal: one provider command per project - (`.benchx/provider.json`), `describe` and `prepare` only, `build` targets + (`provider.command` in `benchx.conf.json`), `describe` and `prepare` only, `build` targets only, and a source ref other than `WORKSPACE` is the provider's business. How cached builds are named and expired stays open (`system-decomposition.md` §6 question 5). The demo prepares its builds by diff --git a/docs/design/prototype-scope.md b/docs/design/prototype-scope.md index f7f321b..4e5e517 100644 --- a/docs/design/prototype-scope.md +++ b/docs/design/prototype-scope.md @@ -44,7 +44,7 @@ one run: | Adapter | Google Benchmark, both halves: drive filter and repetitions, translate output; pyperf, both halves, for Python functions in a working tree, through `bx bench` and `bx compare` (the default adapter of both) | all other harnesses; pyperf's `tags`, `date`, per-run load and frequency, and `pyperf check` | — | | Store | one Parquet file; ingest per schema §5.4 (validated and durable, auto-create, unit-conflict rejection, idempotency, attempt integrity); one default identity policy; queries: list series, series history, get result | continuity events, quarantine, artifact storage, identity-policy authoring | — | | Comparator | run mode with the `revisions` and `environments` profiles; eligibility guard; comparison document | history mode; `single-run`, `variants`, and `cross-machine` profiles; batch requests; provisional noise rules | — | -| Workbench session loop | `bx compare` in measure mode: writes one work order per side and round, alternates them, delivers the results, then compares; takes each side as a build directory or as `@CONFIG`, a call to the project's target provider (`describe` or `prepare`), whose description may be a build or a working tree; either harness, with the same `-e` variables on both sides; a bare path is a build directory for Google Benchmark and a working tree for pyperf; `bx bench` is the same path for one order; `bx target` asks for a description directly; copies the description's declared text (`how_built`, `activation`, `shell`, `provider`) into the order | provider-built revisions (a source ref other than `WORKSPACE` is the provider's business), an `artifact` target, a declared `build` | workbench packaging, front ends such as `spin bench` | +| Workbench session loop | the project file `benchx.conf.json` (one file, in the project root: provider, project, harness and suite, quantities, precision, environment variables, timeouts, paths, comparator policy; flags over file over defaults); `bx bench` (one order) and `bx compare` in measure mode: writes one work order per side and round, alternates them, delivers the results, then compares; takes each side as a build directory or as `@CONFIG`, a call to the project's target provider (`describe` or `prepare`), whose description may be a build or a working tree; either harness, with the same `-e` variables on both sides; a bare path is a build directory for Google Benchmark and a working tree for pyperf; `bx bench` is the same path for one order; `bx target` asks for a description directly; copies the description's declared text (`how_built`, `activation`, `shell`, `provider`) into the order | provider-built revisions (a source ref other than `WORKSPACE` is the provider's business), an `artifact` target, a declared `build` | workbench packaging, front ends such as `spin bench` | | Documents | work order (`schemas/work-order/0.1.0`), result (schema §5.2), comparison document | — | — | **Out entirely:** scheduler, change detector, alerting, views beyond query @@ -84,7 +84,7 @@ proto-workbench). - One command, `bx compare`, has a read mode and a measure mode (`prototype-design.md` §1). - The local store is a single Parquet file in one place, - `~/.benchx/store.parquet`, and `bx run` delivers to it. JSON is for + the project's `store.parquet` (`paths.store` in `benchx.conf.json`), and `bx run` delivers to it. JSON is for documents in transit (work orders, result files, comparison documents), never for the store. - Ingest acknowledgment = validated and durable (schema §5.4). diff --git a/docs/design/system-decomposition.md b/docs/design/system-decomposition.md index 12948ad..876deb2 100644 --- a/docs/design/system-decomposition.md +++ b/docs/design/system-decomposition.md @@ -179,8 +179,8 @@ The same document reaches the workbench by three routes: 1. **A sidecar file**, `.benchx-target.json` in a build directory, written by the user's own build script. It stays with a long-lived build directory instead of with one invocation. -2. **A provider command**, named in the project file `.benchx/provider.json` - as `{"command": [...]}`, which prints the description on stdout. +2. **A provider command**, named in the project file `benchx.conf.json` + as `"provider": {"command": [...]}`, which prints the description on stdout. 3. **The workbench's own input**, when the user prepared everything by hand and there is no provider. diff --git a/prototype/README.md b/prototype/README.md index 7f47e99..1b58136 100644 --- a/prototype/README.md +++ b/prototype/README.md @@ -31,16 +31,30 @@ bx bench TARGET --suite NAME [CASE …] # the single-target form of bx compar bx bench MODULE.py [FUNCTION …] [--tree DIR] [...] # shorthand: Python functions under pyperf → results → store → table bx bench PATH --adapter asv [CASE …] [--filter REGEX] [--tree DIR] [...] - # shorthand: an asv benchmark suite, timed by pyperf -bx config init [--dir DIR] # write .benchx/config.json stating pyperf's defaults + # shorthand: an asv benchmark suite, timed by pyperf +bx config init [--dir DIR] [--adapter …] # write a benchx.conf.json stating a harness's defaults +bx config show # what benchx.conf.json and the defaults give, and where each value comes from ``` +**One project file.** `benchx.conf.json` in the project root (found by walking up from the +working directory; its directory is the root) says how the project is measured: the +`provider`, the `project` name and `source`, an `adapter` block (`name`, `suite`, +`quantities`, `precision`), `environment_variables`, `timeouts`, `paths` for the store +and the results (by default `store.parquet` and `results/` in the root), and the +comparator's `compare` policy. Every value is the command line's, else the file's, else a +built-in default; a required one (`adapter.name`, `adapter.suite`, the target, and for +`bx compare` the profile and `--rounds`) that has none is a hard failure that names the +flag and the key. `--no-project` makes one run ad hoc whatever the file's `project` says. +The schema is `schemas/config/0.1.0`, whose `examples/` hold recommended settings for +pyperf and for Google Benchmark. The store and the results directory are left out of a +measured tree's identity when they sit inside it. `.benchx/setup.json` (declared +environment facts) is a separate file, written by whoever prepares an environment. + Each target is `BUILD_DIR[:SOURCE_DIR]` or `@CONFIG[=SOURCE_REF]`. Two targets select measure mode and `--run` selects read mode; mixing them is an error. -`@CONFIG` asks the project's **target provider**, the command named in -`.benchx/provider.json` (`{"command": [...]}`, found by walking up from the -current directory). The provider speaks the contract of +`@CONFIG` asks the project's **target provider**, the command named by `provider.command` +in `benchx.conf.json` (it runs with the project root as its working directory). The provider speaks the contract of `system-decomposition.md` §3.4: a JSON request on stdin, a target description (`schemas/target-description/0.1.0`) on stdout, build output on stderr, and a nonzero exit for failure. `prepare`, the default, may build; `describe` @@ -62,10 +76,10 @@ working tree for [pyperf](https://pyperf.readthedocs.io), the default as dirty). Cases are the harness's defaults (pyperf: each `bench_*` or `benchmark_*` function), the exact names given, or a `--filter` regex. A flag the chosen harness does not take is refused. `-e` sets environment variables for the workers and lands in the -workload's parameters. For pyperf, precision resolves as flags, then -`.benchx/config.json` (found by walking up from the current directory; -`bx config init` writes a template), then pyperf's defaults, and the resolved values are -written into the order. The workers run the `python` first on `PATH`, which must have +workload's parameters. Precision resolves as flags, then `adapter.precision` in +`benchx.conf.json` (`bx config init` writes a template), then the harness's defaults, and the +resolved values are written into the order. `--quantity` takes a comma-separated list, +and one run collects them all. The workers run the `python` first on `PATH`, which must have pyperf. `bx bench MODULE.py [FUNCTION …]` is the shorthand for the quickest case, timing a few @@ -126,8 +140,8 @@ every result is pyperf. Results also record the installed project's version `.benchx/setup.json`. It changes nothing on the machine and builds nothing, and it refuses an order that names a deferred field (`environment_policy`, a declared `build`, `components`, `schedule`) instead of ignoring it. -- **The store** is one Parquet file, always `~/.benchx/store.parquet` - (`BENCHX_HOME=...` relocates benchx's home, as the tests do). It reads documents strictly, validates them, and applies the +- **The store** is one Parquet file, by default `store.parquet` in the project root + (`paths.store` in `benchx.conf.json` moves it). It reads documents strictly, validates them, and applies the schema §5.4 contract: idempotency and the rejection codes `malformed`, `idempotency-conflict`, `unit-conflict`, `identity-violation`, and `attempt-conflict`. Each result is one row: the columns #30 derives from the @@ -180,14 +194,14 @@ part, then: persistent one, lists series and history, and ends with `bx head`. Every `bx run` and every measure-mode `bx compare` delivers to the local store, so each demo run adds its -results, ad hoc and tracked, to `~/.benchx/store.parquet`. Run keys carry a +results, ad hoc and tracked, to `examples/demo/store.parquet` (the demo's project root, set +in its `benchx.conf.json`). Run keys carry a timestamp, so runs accumulate and each series' history grows. Look at it any time: ```console -$ .venv/bin/python -m benchx.cli head -n 20 -$ rm ~/.benchx/store.parquet # start over -$ BENCHX_HOME=/tmp/benchx ./demo.sh # or keep the demo out of your real store +$ (cd examples/demo && ../../.venv/bin/python -m benchx.cli head -n 20) +$ rm examples/demo/store.parquet # start over ``` Both changes add two to three percent of work. The quiet benchmark flags them @@ -211,7 +225,7 @@ benchx/ adapters/asv.py asv's benchmark format, timed by pyperf (bx bench --adapter asv) adapters/asv_runtime.py asv's discovery and setup semantics, standalone, copied beside each driver bench.py bx bench: the MODULE.py and asv shorthands, and the results table - config.py .benchx/config.json: the pyperf precision defaults and their template + config.py benchx.conf.json: finding it, validating it, picking a value, the template runner.py one work order in, results out session.py the session loop: alternate two targets over rounds, then compare target.py target descriptions: the provider call, the sidecar, the declared text @@ -223,6 +237,7 @@ benchx/ cli.py bx (compare has a read mode and a measure mode) ../schemas/ the result and work-order schemas, read in place ($BENCHX_SCHEMAS overrides) examples/demo-suite a two-benchmark Google Benchmark suite for the demo +examples/demo the demo's project root: its benchx.conf.json, and its persistent store examples/pyperf-bench Python functions under `bx bench` (own README) examples/archery-local an end-to-end, Arrow-local-shaped example with a target provider (own README) tests/ test_prototype.py: the success criteria and the runner and store diff --git a/prototype/benchx/adapters/gbench.py b/prototype/benchx/adapters/gbench.py index 61efdef..f760209 100644 --- a/prototype/benchx/adapters/gbench.py +++ b/prototype/benchx/adapters/gbench.py @@ -31,12 +31,19 @@ # Google Benchmark's own defaults, recorded when the order leaves them out, so # no applied setting goes unrecorded (work-order §3 "every applied setting is # recorded"). precision.repetitions has no default: it is required. +DEFAULT_REPETITIONS = 5 # the order always states a repetition count DEFAULTS = { "calibration": {"mode": "adaptive", "minimum_sample_seconds": 0.5}, "warmup": {"mode": "none"}, } +def default_precision() -> dict: + """The precision a Google Benchmark order gets with nothing configured.""" + return {"repetitions": DEFAULT_REPETITIONS, "calibration": dict(DEFAULTS["calibration"]), + "warmup": dict(DEFAULTS["warmup"])} + + def protocol(precision: dict) -> tuple[dict, list[str]]: """The full protocol that will apply, and the flags that apply it. diff --git a/prototype/benchx/cli.py b/prototype/benchx/cli.py index 3736f98..466ff76 100644 --- a/prototype/benchx/cli.py +++ b/prototype/benchx/cli.py @@ -1,11 +1,14 @@ """bx: a thin command line over the benchx package (prototype-design.md §1). -The local store is always ~/.benchx/store.parquet ($BENCHX_HOME relocates -benchx's home). `bx run` writes result files and ingests them there. +Every command reads the project's `benchx.conf.json` (benchx.config), found from the +working directory upward. The local store and the result files are in the +project root unless that file says otherwise, or `$BENCHX_HOME` relocates the store. A command-line argument wins over +the file, and the file over a built-in default. """ import argparse import json +import os import sys import tempfile from pathlib import Path @@ -14,7 +17,7 @@ from . import bench as bench_mod from . import compare as compare_mod from . import config as config_mod -from . import core, harnesses, orders, runner, session +from . import core, harnesses, orders, runner, session, snapshot from . import target as target_mod from .core.validation import Kind from .store import Store @@ -31,11 +34,26 @@ def _refused(error: runner.Refused) -> int: return EXIT_REFUSED +def _load(command): + """The project's configuration, or None (after saying why) when it is unusable. + The store and the results directory are left out of every tree identity.""" + try: + cfg = config_mod.load() + except config_mod.ConfigError as e: + print(f"bx {command}: error: {e}", file=sys.stderr) + return None + snapshot.ignore(cfg.store_path, cfg.results_dir) + return cfg + + def cmd_run(args): """Run one order, write its result files, and deliver them to the local store. The runner itself stays store-unaware (runner.md §2 principle 6); this command composes it with ingest, as the workbench does.""" - out = Path(args.out) if args.out else Path("results") + if (cfg := _load("run")) is None: + return EXIT_REFUSED + out = Path(args.out) if args.out else cfg.results_dir + snapshot.ignore(cfg.store_path, cfg.results_dir, out) try: summary = runner.run(args.order, out) except runner.Refused as e: @@ -44,7 +62,7 @@ def cmd_run(args): f" (order {summary['order'][:19]})") if args.no_ingest: return EXIT_OK - store = Store() + store = Store(cfg.store_path) print(f"{store.path}: ", end="") return _report(store.ingest_paths(summary["files"])) @@ -60,52 +78,92 @@ def _environment(items) -> dict: return env -def _bench_target(args) -> tuple[str, str, str | None]: - """The target, suite and adapter `bx bench` runs. With `--suite` they are as given. - Without it, the target is a Python module (the shorthand `bx bench MODULE.py - [FUNCTION ...]`): its checkout is the target, its path in it the suite, and the - harness pyperf. With `--adapter asv` it is an asv suite instead (`bx bench PATH - --adapter asv`): PATH holds asv.conf.json or is the benchmark directory.""" - if args.suite: +def _bench_target(args) -> tuple[str, str | None, str | None]: + """The target, the suite it forces (or None: --suite, then the file's), and the + adapter it forces. With `--suite` the target is as given. Without it, the target + is a Python module (the shorthand `bx bench MODULE.py [FUNCTION ...]`): its + checkout is the target, its path in it the suite, and the harness pyperf. With + `--adapter asv` it is an asv suite instead (`bx bench PATH --adapter asv`): PATH + holds asv.conf.json or is the benchmark directory.""" + if not args.suite and args.adapter == "asv": + return (*bench_mod.asv_target(args.target, args.tree), "asv") + if args.suite or not args.target.endswith(".py"): if args.tree: raise UsageError("--tree belongs to the shorthands; give the tree as the TARGET with --suite") - return args.target, args.suite, args.adapter - if args.adapter == "asv": - return (*bench_mod.asv_target(args.target, args.tree), "asv") - if not args.target.endswith(".py"): - raise UsageError("give --suite, a Python file to benchmark, or an asv suite with --adapter asv") + return args.target, None, None if args.adapter not in (None, "pyperf"): raise UsageError(f"a Python file is benchmarked with pyperf, not {args.adapter}") target, suite = bench_mod.module_target(args.target, args.tree) return target, suite, "pyperf" +def _common(cfg, args, *, forced_adapter=None, forced_suite=None) -> dict: + """The settings `bx bench` and `bx compare` share, each from the command line, + else the file, else its default (config.Config.pick); ConfigError when a + required one has none. The file's adapter block belongs to the adapter it + names, so naming another is refused.""" + block = cfg.adapter + given = forced_adapter or args.adapter + if given and block.get("name") and given != block["name"]: + raise config_mod.ConfigError(f"adapter {given!r}, but {cfg.where} sets adapter.name to {block['name']!r}, " + "and its suite and precision belong to that harness") + quantities = config_mod.parse_quantities(args.quantity) if args.quantity else block.get("quantities", ["wall-time"]) + if args.project and args.no_project: + raise UsageError("give --project or --no-project, not both") + timeouts = cfg.section("timeouts") + adapter = cfg.pick(given, block.get("name"), flag="--adapter", key="adapter.name") + return { + "adapter": adapter, + "suite": cfg.pick(forced_suite or args.suite, block.get("suite"), flag="--suite", key="adapter.suite"), + "flags": {name: getattr(args, name) for name in harnesses.FLAG_NAMES}, + "file_precision": block.get("precision"), + "quantities": quantities, + "filter": args.filter, + "environment_variables": {**cfg.section("environment_variables"), **_environment(args.env)}, + "project": None if args.no_project else cfg.pick(args.project, cfg.data.get("project"), None, + flag="--project", key="project"), + "run_key": args.run_key, + "out": args.out, + "source_uri": args.source_uri or cfg.section("source").get("uri"), + "build": not args.no_build, + "provider": cfg.provider, + "results_dir": cfg.results_dir, + # An asv case is every round of its benchmark, each bounded by the benchmark's own timeout. + "case_s": cfg.pick(getattr(args, "case_timeout", None), timeouts.get("case_s"), + ASV_CASE_TIMEOUT_S if adapter == "asv" else orders.CASE_TIMEOUT_S, + flag="--case-timeout", key="timeouts.case_s"), + "order_s": timeouts.get("order_s", orders.ORDER_TIMEOUT_S), + "config_path": cfg.path, + } + + +def _needs_provider(cfg, *targets): + for spec in targets: + if spec.startswith("@"): + cfg.require_provider(spec) + + def cmd_bench(args): """Run one target once, as one order of a comparison session: resolve the target, build the order from the harness defaults, config file, flags and environment variables, run it, deliver, print a table.""" + if (cfg := _load("bench")) is None: + return EXIT_REFUSED try: - env = _environment(args.env) if args.functions and args.filter: raise UsageError("give case names or --filter, not both") - target, suite, adapter = _bench_target(args) - except UsageError as e: + target, forced_suite, forced_adapter = _bench_target(args) + settings = _common(cfg, args, forced_adapter=forced_adapter, forced_suite=forced_suite) + _needs_provider(cfg, target) + except (UsageError, config_mod.ConfigError, bench_mod.SessionError) as e: print(f"bx bench: error: {e}", file=sys.stderr) return EXIT_REFUSED - except bench_mod.SessionError as e: - print(f"bx bench: error: {e}", file=sys.stderr) - return EXIT_REFUSED - store = None if args.no_ingest else Store() - # An asv case is every round of its benchmark, each bounded by the benchmark's own timeout. - case_s = (args.case_timeout if args.case_timeout is not None - else ASV_CASE_TIMEOUT_S if adapter == "asv" else orders.CASE_TIMEOUT_S) + settings["filter"] = harnesses.exact_filter(args.functions) or settings["filter"] + snapshot.ignore(cfg.store_path, cfg.results_dir, *([args.out] if args.out else [])) + store = None if args.no_ingest else Store(cfg.store_path) try: - summary = session.bench_target( - target, suite=suite, store=store, adapter=adapter, flags={name: getattr(args, name) for name in harnesses.FLAG_NAMES}, - environment_variables=env, filter=harnesses.exact_filter(args.functions) or args.filter, - quantity=args.quantity or "wall-time", project=args.project, run_key=args.run_key, out=args.out, - source_uri=args.source_uri, case_s=case_s, build=not args.no_build, - progress=lambda message: print(message, file=sys.stderr)) + summary = session.bench_target(target, store=store, progress=lambda message: print(message, file=sys.stderr), + **settings) except (session.SessionError, config_mod.ConfigError) as e: print(f"bx bench: error: {e}", file=sys.stderr) return EXIT_REFUSED @@ -120,15 +178,52 @@ def cmd_bench(args): def cmd_config(args): - try: - path = config_mod.write_template(args.dir) - except config_mod.ConfigError as e: - print(f"bx config: error: {e}", file=sys.stderr) + if args.action == "init": + try: + path = config_mod.write_template(args.dir, args.adapter) + except config_mod.ConfigError as e: + print(f"bx config: error: {e}", file=sys.stderr) + return EXIT_REFUSED + print(f"wrote {path}") + return EXIT_OK + if (cfg := _load("config")) is None: return EXIT_REFUSED - print(f"wrote {path}") + print(f"file {cfg.where}\nroot {cfg.root}") + for key, value, source in _resolved(cfg): + print(f"{key:<26} {value} ({source})") return EXIT_OK +def _resolved(cfg): + """What the file and the built-in defaults give, before any command-line argument: + (key, value, 'file' | 'default' | 'unset').""" + def row(key, value, default=None): + if value is not None: + return key, json.dumps(value), "file" + return key, json.dumps(default) if default is not None else "-", "default" if default is not None else "unset" + + block, timeouts, compare = cfg.adapter, cfg.section("timeouts"), cfg.section("compare") + yield row("project", cfg.data.get("project")) + yield row("source.uri", cfg.section("source").get("uri")) + yield row("provider.command", cfg.section("provider").get("command")) + yield row("adapter.name", block.get("name")) + yield row("adapter.suite", block.get("suite")) + yield row("adapter.quantities", block.get("quantities"), ["wall-time"]) + if block.get("name"): + resolved = harnesses.get(block["name"]).precision({}, block.get("precision")) + yield "adapter.precision", json.dumps(resolved), "file + default" if block.get("precision") else "default" + yield row("environment_variables", cfg.data.get("environment_variables"), {}) + yield row("timeouts.case_s", timeouts.get("case_s"), orders.CASE_TIMEOUT_S) + yield row("timeouts.order_s", timeouts.get("order_s"), orders.ORDER_TIMEOUT_S) + store_from = "$BENCHX_HOME" if os.environ.get(config_mod.HOME_ENV) else "file" if "store" in cfg.section("paths") else "default" + yield "paths.store", str(cfg.store_path), store_from + yield "paths.results", str(cfg.results_dir), "file" if "results" in cfg.section("paths") else "default" + yield row("compare.profile", compare.get("profile")) + yield row("compare.rounds", compare.get("rounds")) + yield row("compare.k", compare.get("k"), 3.0) + yield row("compare.min_rounds", compare.get("min_rounds"), 3) + + def _report(outcome): for r in outcome["rejected"]: print(f" rejected {r['document']}: {r['code']} — {r['message']}", file=sys.stderr) @@ -141,11 +236,15 @@ def _report(outcome): def cmd_ingest(args): - return _report(Store().ingest_paths(args.paths)) + if (cfg := _load("ingest")) is None: + return EXIT_REFUSED + return _report(Store(cfg.store_path).ingest_paths(args.paths)) def cmd_series(args): - for s in Store().series(args.workload, args.quantity): + if (cfg := _load("series")) is None: + return EXIT_REFUSED + for s in Store(cfg.store_path).series(args.workload, args.quantity): estimator = json.loads(s["estimator"])["method_version"] print(f"{s['fingerprint'][:12]} {s['project']} {s['workload']} {s['quantity']} [{s['unit']}]" f" est={estimator}") @@ -153,7 +252,9 @@ def cmd_series(args): def cmd_history(args): - points = Store().history(args.series) + if (cfg := _load("history")) is None: + return EXIT_REFUSED + points = Store(cfg.store_path).history(args.series) print("revision order unavailable without a revision graph; listed by measurement start") for p in points: changed = f" changed: {', '.join(p['changed'])}" if p["changed"] else "" @@ -162,7 +263,9 @@ def cmd_history(args): def cmd_head(args): - store = Store() + if (cfg := _load("head")) is None: + return EXIT_REFUSED + store = Store(cfg.store_path) total, rows = store.head(args.n, args.project, args.workload, measurands=args.table) if args.json: for row in rows: @@ -190,9 +293,12 @@ def cmd_head(args): def cmd_target(args): """Ask for a target's description: `describe` only reports, `prepare` may build (the provider's own tools do that, never benchx).""" + if (cfg := _load("target")) is None: + return EXIT_REFUSED try: - description = target_mod.resolve(args.target, operation=args.operation) - except target_mod.TargetError as e: + _needs_provider(cfg, args.target) + description = target_mod.resolve(args.target, operation=args.operation, provider=cfg.provider) + except (target_mod.TargetError, config_mod.ConfigError) as e: print(f"cannot {args.operation} {args.target}: {e}", file=sys.stderr) return EXIT_REFUSED if description is None: @@ -207,7 +313,7 @@ class UsageError(Exception): # Flags that belong to one mode of `bx compare` (prototype-design.md §1). -_MEASURE_ONLY = ("adapter", "suite", "filter", "quantity", "rounds", "env", *harnesses.FLAG_NAMES, "project", +_MEASURE_ONLY = ("adapter", "suite", "filter", "quantity", "rounds", "env", *harnesses.FLAG_NAMES, "project", "no_project", "run_key", "out", "source_uri", "no_build") _READ_ONLY = ("baseline", "results") @@ -226,9 +332,6 @@ def _mode(args) -> str: raise UsageError("measure mode takes exactly two targets: BASELINE CONTENDER") if wrong := _flags(args, _READ_ONLY): raise UsageError(f"{', '.join(wrong)} belongs to read mode (--run)") - for need in ("suite", "rounds"): - if getattr(args, need) is None: - raise UsageError(f"measure mode needs --{need}") return "measure" if wrong := _flags(args, _MEASURE_ONLY): raise UsageError(f"{', '.join(wrong)} belongs to measure mode (two targets)") @@ -238,55 +341,69 @@ def _mode(args) -> str: def cmd_compare(args): + if (cfg := _load("compare")) is None: + return EXIT_REFUSED try: mode = _mode(args) - except UsageError as e: + policy = _policy(cfg, args) + except (UsageError, config_mod.ConfigError) as e: print(f"bx compare: error: {e}", file=sys.stderr) return EXIT_REFUSED - return _measure(args) if mode == "measure" else _read(args) + return _measure(cfg, args, policy) if mode == "measure" else _read(cfg, args, policy) + + +def _policy(cfg, args) -> dict: + """The comparison's own settings: the command line, else `compare` in the file, else the default.""" + compare = cfg.section("compare") + return { + "profile": cfg.pick(args.profile, compare.get("profile"), flag="--profile", key="compare.profile"), + "k": cfg.pick(args.k, compare.get("k"), 3.0, flag="--k", key="compare.k"), + "min_rounds": cfg.pick(args.min_rounds, compare.get("min_rounds"), 3, flag="--min-rounds", + key="compare.min_rounds"), + } -def _read(args): +def _read(cfg, args, policy): if args.results: with tempfile.TemporaryDirectory() as tmp: # the ad hoc path: a throwaway store store = Store(Path(tmp) / "store.parquet") outcome = store.ingest_paths(args.results) if outcome["rejected"]: return _report(outcome) - return _compare(store, args) - return _compare(Store(), args) + return _compare(store, args, policy) + return _compare(Store(cfg.store_path), args, policy) -def _measure(args): +def _measure(cfg, args, policy): """The workbench's session loop: run both sides, deliver, compare.""" - store = Store() + store = Store(cfg.store_path) print(f"{store.path}", file=sys.stderr) # stderr, so --json output stays clean try: + settings = _common(cfg, args) + _needs_provider(cfg, *args.targets) + snapshot.ignore(cfg.store_path, cfg.results_dir, *([args.out] if args.out else [])) label = session.parse_label(args.label) if args.label else None doc = session.compare_targets( - *args.targets, profile=args.profile, suite=args.suite, rounds=args.rounds, store=store, - adapter=args.adapter, flags={name: getattr(args, name) for name in harnesses.FLAG_NAMES}, - environment_variables=_environment(args.env), filter=args.filter, - quantity=args.quantity or "wall-time", project=args.project, label=label, run_key=args.run_key, - out=args.out, source_uri=args.source_uri, k=args.k, min_rounds=args.min_rounds, - build=not args.no_build, progress=lambda message: print(message, file=sys.stderr)) + *args.targets, rounds=cfg.pick(args.rounds, cfg.section("compare").get("rounds"), flag="--rounds", + key="compare.rounds"), + store=store, label=label, progress=lambda message: print(message, file=sys.stderr), **policy, **settings) except runner.Refused as e: return _refused(e) - except UsageError as e: + except (UsageError, config_mod.ConfigError) as e: print(f"bx compare: error: {e}", file=sys.stderr) return EXIT_REFUSED - except (session.SessionError, compare_mod.CompareError, config_mod.ConfigError) as e: + except (session.SessionError, compare_mod.CompareError) as e: print(f"cannot compare: {e}", file=sys.stderr) return EXIT_REFUSED print(json.dumps(doc, indent=1) if args.json else compare_mod.render(doc)) return EXIT_OK -def _compare(store, args): +def _compare(store, args, policy): try: - doc = compare_mod.compare(store.documents(args.run), run_key=args.run, profile=args.profile, - baseline=args.baseline, label=args.label, k=args.k, - min_rounds=args.min_rounds) + doc = compare_mod.compare(store.documents(args.run), run_key=args.run, profile=policy["profile"], + baseline=args.baseline, label=args.label, k=policy["k"], + min_rounds=policy["min_rounds"]) except compare_mod.CompareError as e: print(f"cannot compare: {e}", file=sys.stderr) return EXIT_REFUSED @@ -309,12 +426,15 @@ def _add_target_options(parser, command): """The options `bx bench` and `bx compare` (measure mode) share: the harness, its suite and cases, its precision flags, and where the results go.""" parser.add_argument("--adapter", choices=sorted(harnesses.HARNESSES), - help=f"the harness (default {session.HARNESS.name})") + help="the harness (default: adapter.name; required)") parser.add_argument("--suite", help="pyperf: the module's path in the working tree; " - "google-benchmark: the binary in the build directory") + "google-benchmark: the binary in the build directory " + "(default: adapter.suite; required)") parser.add_argument("--filter", help="a case filter regex (pyperf: function names; " "google-benchmark: --benchmark_filter)") - parser.add_argument("--quantity", help="wall-time (default); google-benchmark also cpu-time") + parser.add_argument("--quantity", metavar="NAMES", + help="comma-separated, collected in one run (default: adapter.quantities, else wall-time); " + "google-benchmark also reports cpu-time") parser.add_argument("-e", "--env", action="append", metavar="NAME=VALUE", help="set an environment variable for the benchmark (repeatable); " "each becomes a workload parameter") @@ -325,12 +445,15 @@ def _add_target_options(parser, command): parser.add_argument("--min-time", type=float, help="adaptive calibration: minimum seconds per sample") parser.add_argument("--loops", type=int, help="pyperf: fixed calibration, inner loops per value") parser.add_argument("--warmups", type=int, help="pyperf: warmup values per process (0: none)") - parser.add_argument("--project", help="name a project, so the results join its series; omit for an ad hoc run") + parser.add_argument("--project", help="name a project, so the results join its series (default: project)") + parser.add_argument("--no-project", action="store_true", + help="an ad hoc run, whatever the file's project says: filed as local/") parser.add_argument("--run-key", help=f"default: {command}-") - parser.add_argument("--out", help="directory for orders and result files (default ./results/RUN_KEY)") + parser.add_argument("--out", help="directory for orders and result files (default: RESULTS/RUN_KEY, " + "RESULTS being paths.results, else ./results in the project root)") parser.add_argument("--no-build", action="store_true", help="ask a provider to describe each @CONFIG target instead of preparing (building) it") - parser.add_argument("--source-uri", help="the source's URI (default: the checkout's origin remote)") + parser.add_argument("--source-uri", help="the source's URI (default: source.uri, else the checkout's origin remote)") def main(argv=None): @@ -365,14 +488,17 @@ def main(argv=None): _add_target_options(p, "bench") p.add_argument("--tree", help="the shorthand's tree (default: the module's or suite's git checkout or directory)") p.add_argument("--case-timeout", type=float, - help=f"seconds before one case is stopped (default {orders.CASE_TIMEOUT_S}; asv: " - f"{ASV_CASE_TIMEOUT_S}, all rounds, each also stopped at the benchmark's own timeout)") + help=f"seconds before one case is stopped (default: timeouts.case_s, else {orders.CASE_TIMEOUT_S:g}; " + f"asv: {ASV_CASE_TIMEOUT_S}, all rounds, each also stopped at the benchmark's own timeout)") p.add_argument("--no-ingest", action="store_true", help="only write result files; skip the local store") p.set_defaults(fn=cmd_bench) - p = sub.add_parser("config", help="manage .benchx/config.json") - p.add_argument("action", choices=["init"], help="init: write a template stating pyperf's defaults") - p.add_argument("--dir", default=".", help="where to write .benchx/config.json (default .)") + p = sub.add_parser("config", help="write or show benchx.conf.json") + p.add_argument("action", choices=["init", "show"], + help="init: write a benchx.conf.json stating a harness's defaults; show: what the file and the defaults give") + p.add_argument("--dir", default=".", help="init: where to write benchx.conf.json (default .)") + p.add_argument("--adapter", choices=sorted(harnesses.HARNESSES), default="pyperf", + help="init: the harness (default %(default)s)") p.set_defaults(fn=cmd_config) p = sub.add_parser("ingest", help="sweep result files and work orders into the local store") @@ -400,7 +526,7 @@ def main(argv=None): p = sub.add_parser("target", help="ask for a target's description (schemas/target-description/0.1.0)", description="TARGET is @CONFIG[=SOURCE_REF], asking the provider named in " - ".benchx/provider.json, or a build directory holding .benchx-target.json. " + "benchx.conf.json, or a build directory holding .benchx-target.json. " "describe only reports a target that exists; prepare may build it.") p.add_argument("operation", choices=["describe", "prepare"]) p.add_argument("target") @@ -413,9 +539,9 @@ def main(argv=None): "BASELINE CONTENDER --suite NAME --rounds R, each target BUILD_DIR[:SOURCE_DIR] " "or @CONFIG[=SOURCE_REF] (the project's target provider).") p.add_argument("targets", nargs="*", metavar="TARGET", help="measure mode: BASELINE CONTENDER") - p.add_argument("--profile", required=True, choices=["revisions", "environments"]) - p.add_argument("--k", type=float, default=3.0) - p.add_argument("--min-rounds", type=int, default=3) + p.add_argument("--profile", choices=["revisions", "environments"], help="default: compare.profile") + p.add_argument("--k", type=float, help="noise multiplier (default: compare.k, else 3)") + p.add_argument("--min-rounds", type=int, help="fewest paired rounds for a verdict (default: compare.min_rounds, else 3)") p.add_argument("--json", action="store_true") p.add_argument("--label", help="read mode: the label that names the sides (environments); " "measure mode: NAME=BASE,CONTENDER") @@ -425,7 +551,8 @@ def main(argv=None): read.add_argument("--results", nargs="+", help="compare these result directories through a throwaway store, not the local one") measure = p.add_argument_group("measure mode") - measure.add_argument("--rounds", type=int, help="alternation rounds; each runs both sides once") + measure.add_argument("--rounds", type=int, + help="alternation rounds; each runs both sides once (default: compare.rounds)") _add_target_options(measure, "compare") p.set_defaults(fn=cmd_compare) diff --git a/prototype/benchx/config.py b/prototype/benchx/config.py index 4b0d9a4..5b7a79c 100644 --- a/prototype/benchx/config.py +++ b/prototype/benchx/config.py @@ -1,66 +1,142 @@ -"""The precision config file, `.benchx/config.json` (PYPERF_PLAN.md step 4). +"""The project configuration file, `benchx.conf.json`, in the project root. - {"adapters": {"pyperf": {"precision": {...}}}} + schemas/config/0.1.0 -The precision block is spelled as in the work order (schemas/work-order -`precision`) and validated against it. It may be partial: a repetition level, -or calibration or warmup, that it leaves out takes the adapter's default. +One file says how a project is measured: the harness and its suite, quantities +and precision; the target provider; the project's name and source; environment +variables and timeouts; where the store and the result files go; and the +comparator's policy. The directory holding it is the project root, found by +walking up from the working directory. With no file the root is the working +directory and every value comes from the command line or a built-in default. -Precedence: flags, then the file, then the adapter defaults. `resolve` returns -the complete precision, which goes into the order, so the order and its hash -hold the values that applied. +Relative paths in the file, and the defaults for the store and the results, are +relative to the root, never to the working directory. + +Precedence, for every value: the command line, then this file, then a built-in +default; the one exception is the store, which `$BENCHX_HOME` relocates over the file. A value that is required and has none of the three is a hard failure +that names the flag and the key (`Config.pick`). """ import copy import json +import os +from dataclasses import dataclass from pathlib import Path from . import adapters, core from .core import loader -from . import target as target_mod -CONFIG_FILE = Path(".benchx") / "config.json" +CONFIG_FILE = "benchx.conf.json" +# Files of the format this one replaced. They are not read; finding one is an error, so +# no setting silently stops applying. +LEGACY_FILES = (Path(".benchx") / "provider.json", Path(".benchx") / "config.json") +HOME_ENV = "BENCHX_HOME" +DEFAULT_STORE = "store.parquet" +DEFAULT_RESULTS = "results" +REQUIRED = object() class ConfigError(Exception): - """The config file is unusable; the message names the file and the problem.""" - - -def template(adapter_name="pyperf") -> dict: - """A config file that states the adapter's defaults, for the user to edit.""" - return {"adapters": {adapter_name: {"precision": adapters.get(adapter_name).default_precision()}}} - - -def find(start="."): - return target_mod.find_upward(CONFIG_FILE, start) - - -def load(adapter_name, start="."): - """The precision block for one adapter from the nearest config file, as - `(block, path)`; `({}, None)` when there is no file or no block.""" - path = find(start) + """The configuration is unusable, or a required value is missing; the message says why.""" + + +def _find_upward(relative, start) -> Path | None: + start = Path(start).resolve() + for directory in (start, *start.parents): + path = directory / relative + if path.is_file(): + return path + return None + + +@dataclass(frozen=True) +class Config: + root: Path + path: Path | None # None: no file was found, and root is the working directory + data: dict + + @property + def where(self) -> str: + return str(self.path) if self.path else f"{CONFIG_FILE} (none found from {self.root} upward)" + + def section(self, name) -> dict: + return self.data.get(name, {}) + + @property + def adapter(self) -> dict: + return self.section("adapter") + + def pick(self, flag_value, file_value, default=REQUIRED, *, flag, key): + """The command-line value, else the file's, else `default`; ConfigError when + none is given and the value is required.""" + for value in (flag_value, file_value): + if value is not None: + return value + if default is REQUIRED: + raise ConfigError(f"{flag} is required: give it, or set {key} in {self.where}") + return default + + def _path(self, value) -> Path: + path = Path(value).expanduser() + return path if path.is_absolute() else self.root / path + + @property + def store_path(self) -> Path: + """The store: `$BENCHX_HOME/store.parquet` when that is set (it wins over the + file, as it relocates the store for tests and CI), else `paths.store`.""" + if home := os.environ.get(HOME_ENV): + return Path(home).expanduser() / DEFAULT_STORE + return self._path(self.section("paths").get("store", DEFAULT_STORE)) + + @property + def results_dir(self) -> Path: + return self._path(self.section("paths").get("results", DEFAULT_RESULTS)) + + @property + def provider(self) -> tuple[list[str], Path] | None: + """The provider command, and the directory it runs in (the root).""" + command = self.section("provider").get("command") + return (list(command), self.root) if command else None + + def require_provider(self, spec: str) -> tuple[list[str], Path]: + found = self.provider + if found is None: + raise ConfigError(f'{spec} needs "provider" in {self.where}') + return found + + +def load(start=".") -> Config: + """The nearest `benchx.conf.json` walking up from `start`, validated; or an empty + configuration rooted at `start` when there is none.""" + start = Path(start).resolve() + path = _find_upward(CONFIG_FILE, start) if path is None: - return {}, None - document, errors = loader.read(path) - if document is None: + for legacy in LEGACY_FILES: + if old := _find_upward(legacy, start): + raise ConfigError(f"{old} is the old configuration format, which is no longer read: " + f"move its settings into {CONFIG_FILE} (schemas/config/0.1.0)") + return Config(root=start, path=None, data={}) + data, errors = loader.read(path) + if data is None: + raise ConfigError(f"{path}: " + "; ".join(str(error) for error in errors)) + if errors := core.check(data, "config"): raise ConfigError(f"{path}: " + "; ".join(str(error) for error in errors)) - if set(document) - {"adapters"}: - raise ConfigError(f'{path}: only "adapters" is allowed at the top level') - adapter_blocks = document.get("adapters", {}) - if not isinstance(adapter_blocks, dict): - raise ConfigError(f'{path}: "adapters" must be an object') - for name in adapter_blocks: - if adapters.get(name) is None: - raise ConfigError(f"{path}: no adapter named {name!r}") - block = adapter_blocks.get(adapter_name, {}) - if not isinstance(block, dict) or set(block) - {"precision"}: - raise ConfigError(f'{path}: adapters.{adapter_name} may hold only "precision"') + _check_adapter(path, data.get("adapter")) + return Config(root=path.parent, path=path, data=data) + + +def _check_adapter(path, block): + if block is None: + return + adapter = adapters.get(block["name"]) + if adapter is None: + raise ConfigError(f"{path}: adapter.name: no adapter named {block['name']!r} ({', '.join(sorted(adapters.ADAPTERS))})") + unknown = sorted(set(block.get("quantities", [])) - set(adapter.QUANTITIES)) + if unknown: + raise ConfigError(f"{path}: adapter.quantities: {block['name']} does not report {', '.join(unknown)}") precision = block.get("precision", {}) - errors = core.check_precision({"repetitions": 1, **precision} if isinstance(precision, dict) else precision) - if errors: - problems = "; ".join(str(error) for error in errors) - raise ConfigError(f"{path}: adapters.{adapter_name}.precision: {problems}") - return precision, path + if errors := core.check_precision({"repetitions": 1, **precision}): + raise ConfigError(f"{path}: adapter.precision: " + "; ".join(str(error) for error in errors)) def merge(base: dict, over: dict) -> dict: @@ -77,19 +153,27 @@ def merge(base: dict, over: dict) -> dict: return merged -def resolve(adapter_name, flags=None, start="."): - """The complete precision, and the config file it read (or None).""" +def parse_quantities(text: str) -> list[str]: + """`--quantity wall-time,cpu-time` as a list; ConfigError on an empty or repeated name.""" + names = [name.strip() for name in text.split(",")] + if not all(names): + raise ConfigError(f"--quantity wants comma-separated names, not {text!r}") + if len(set(names)) != len(names): + raise ConfigError(f"--quantity names a quantity twice: {text!r}") + return names + + +def template(adapter_name="pyperf") -> dict: + """A configuration that states the harness's defaults, for the user to edit.""" adapter = adapters.get(adapter_name) - file_block, path = load(adapter_name, start) - precision = merge(merge(adapter.default_precision(), file_block), flags or {}) - errors = core.check_precision(precision) - if errors: - raise ConfigError("precision: " + "; ".join(str(error) for error in errors)) - return precision, path + if adapter is None: + raise ConfigError(f"no adapter named {adapter_name!r} ({', '.join(sorted(adapters.ADAPTERS))})") + return {"schema_version": "benchx/config/0.1.0", + "adapter": {"name": adapter_name, "quantities": ["wall-time"], "precision": adapter.default_precision()}} def write_template(directory=".", adapter_name="pyperf") -> Path: - """Write `.benchx/config.json` under `directory`; never overwrites.""" + """Write `benchx.conf.json` under `directory`; never overwrites.""" path = Path(directory) / CONFIG_FILE if path.exists(): raise ConfigError(f"{path} already exists") diff --git a/prototype/benchx/core/rules.py b/prototype/benchx/core/rules.py index 9072991..e782350 100644 --- a/prototype/benchx/core/rules.py +++ b/prototype/benchx/core/rules.py @@ -31,6 +31,7 @@ def check(data: dict[str, Any], kind: Kind) -> Iterator[RuleViolation]: kind == "comparison-document" or kind == "target-description" or kind == "target-provider-request" + or kind == "config" ): return # nothing beyond what their schemas say else: diff --git a/prototype/benchx/core/validation.py b/prototype/benchx/core/validation.py index b22c75e..0eab3d5 100644 --- a/prototype/benchx/core/validation.py +++ b/prototype/benchx/core/validation.py @@ -12,6 +12,7 @@ "comparison-document", "target-description", "target-provider-request", + "config", ] @@ -24,6 +25,7 @@ def find_schema( or kind == "comparison-document" or kind == "target-description" or kind == "target-provider-request" + or kind == "config" ): field = "schema_version" else: diff --git a/prototype/benchx/harnesses.py b/prototype/benchx/harnesses.py index 3acb348..c293df3 100644 --- a/prototype/benchx/harnesses.py +++ b/prototype/benchx/harnesses.py @@ -10,8 +10,8 @@ FLAGS the precision flags it takes, by keyword flags_to_precision(**flags) -> the flags spelled as a partial work-order `precision`; a flag it does not take is refused - precision(flags, start) -> (the complete precision, the config file it - read or None): flags over its defaults + precision(flags, file) -> the complete precision: flags over the config + file's precision over its defaults describe_precision(precision, config_path) -> that precision in a few words suite_entry(suite, filter) -> one `suites[]` entry BARE_PATH_KIND what a bare path names for this harness: a `build` @@ -20,7 +20,7 @@ import re -from . import config +from . import adapters, config, core from .errors import SessionError @@ -50,22 +50,29 @@ def flags_to_precision(self, **flags) -> dict: def _precision_from(self, **flags) -> dict: raise NotImplementedError - def precision(self, flags: dict | None, start=".") -> tuple[dict, object]: - raise NotImplementedError + def precision(self, flags: dict | None, file: dict | None = None) -> dict: + """The complete precision; ConfigError if the result is not a valid one.""" + merged = config.merge(config.merge(self._defaults(), file or {}), flags or {}) + if errors := core.check_precision(merged): + raise config.ConfigError("precision: " + "; ".join(str(error) for error in errors)) + return merged + + def _defaults(self) -> dict: + return adapters.get(self.name).default_precision() def describe_precision(self, precision: dict, config_path=None) -> str: raise NotImplementedError class GoogleBenchmark(Harness): - """Repetitions of one level; adaptive calibration by minimum time. No config - file: Google Benchmark's defaults are left to the harness, except the - repetition count, which the order always states.""" + """Repetitions of one level; adaptive calibration by minimum time, and a + time-based warmup. Whatever the config file and the flags leave out takes + Google Benchmark's default, and the order states it.""" name = "google-benchmark" FLAGS = ("repetitions", "min_time") BARE_PATH_KIND = "build" # the binary is built; BUILD_DIR[:SOURCE_DIR] - DEFAULT_REPETITIONS = 5 + DEFAULT_REPETITIONS = adapters.gbench.DEFAULT_REPETITIONS def _precision_from(self, repetitions=None, min_time=None) -> dict: precision = {} @@ -75,17 +82,14 @@ def _precision_from(self, repetitions=None, min_time=None) -> dict: precision["calibration"] = {"mode": "adaptive", "minimum_sample_seconds": min_time} return precision - def precision(self, flags, start="."): - return {"repetitions": self.DEFAULT_REPETITIONS, **(flags or {})}, None - def describe_precision(self, precision, config_path=None): return f"{precision['repetitions']} repetitions" class Pyperf(Harness): """Two repetition levels (processes, values), calibration by loops or minimum - time, and a warmup count. `.benchx/config.json` and pyperf's defaults fill - what the flags leave out.""" + time, and a warmup count. `benchx.conf.json` and pyperf's defaults fill what the + flags leave out.""" name = "pyperf" FLAGS = ("processes", "values", "min_time", "loops", "warmups") @@ -106,9 +110,6 @@ def _precision_from(self, processes=None, values=None, min_time=None, loops=None precision["warmup"] = {"mode": "count", "n_warmup": warmups} if warmups else {"mode": "none"} return precision - def precision(self, flags, start="."): - return config.resolve(self.name, flags, start=start) - def describe_precision(self, precision, config_path=None): reps = {level["unit"]: level["n"] for level in precision["repetitions"]["levels"]} return f"{reps['process']} processes x {reps['value']} values (config: {config_path or 'none, pyperf defaults'})" diff --git a/prototype/benchx/orders.py b/prototype/benchx/orders.py index b3972f0..a8f32a1 100644 --- a/prototype/benchx/orders.py +++ b/prototype/benchx/orders.py @@ -25,7 +25,7 @@ def default_run_key(prefix: str) -> str: return f"{prefix}-{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}" -def make_order(*, suite: dict, target: dict, precision: dict, run_key: str, quantity: str = "wall-time", +def make_order(*, suite: dict, target: dict, precision: dict, run_key: str, quantities=("wall-time",), case_s: float = CASE_TIMEOUT_S, order_s: float = ORDER_TIMEOUT_S, project=None, labels=None, environment_variables=None, round_=None, slot=None, benchmark=None) -> dict: """One resolved work order. Optional fields that are None or empty are left out. @@ -38,14 +38,14 @@ def make_order(*, suite: dict, target: dict, precision: dict, run_key: str, quan "state": "resolved", "suites": [suite], "target": target, - "quantities": [quantity], + "quantities": list(quantities), "precision": precision, "timeouts": {"case_s": case_s, "order_s": order_s}, "run_key": run_key, "requester": dict(REQUESTER), # Placeholder: the runner does not yet cross-check plan against the # case list it discovers (runner-schema.md §3.2). - "plan": [{"id": "p0", "case": "*", "quantity": quantity}], + "plan": [{"id": f"p{i}", "case": "*", "quantity": quantity} for i, quantity in enumerate(quantities)], } if round_ is not None: order["round"] = round_ diff --git a/prototype/benchx/session.py b/prototype/benchx/session.py index a52188b..a123549 100644 --- a/prototype/benchx/session.py +++ b/prototype/benchx/session.py @@ -23,7 +23,6 @@ from .identity import normalize_remote PROFILES = ("revisions", "environments") -HARNESS = harnesses.PYPERF # the default; `adapter` picks another def parse_target(text: str) -> tuple[str, str | None]: @@ -46,7 +45,7 @@ def parse_label(text: str) -> tuple[str, str, str]: KINDS = ("build", "working_tree") -def _side(spec: str, suite: str, operation: str, harness=None, environment_variables=None, +def _side(spec: str, suite: str, operation: str, harness, environment_variables=None, provider=None, timeout=None) -> dict: """Locate and identify one side before anything runs. @@ -58,9 +57,8 @@ def _side(spec: str, suite: str, operation: str, harness=None, environment_varia usable, as the runner will, so a side it would refuse is refused before any round runs. """ - harness = harness or HARNESS try: - description = target_mod.resolve(spec, operation=operation) + description = target_mod.resolve(spec, operation=operation, provider=provider) except target_mod.TargetError as e: raise SessionError(str(e)) from None source = None @@ -117,7 +115,7 @@ def _benchmark(harness, path: Path, suite: str, subject_source: str) -> dict | N **({"revision": found["revision"]} if "revision" in found else {})} -def _order(side: dict, *, harness, suite, filter, quantity, precision, environment_variables, project, run_key, +def _order(side: dict, *, harness, suite, filter, quantities, precision, environment_variables, project, run_key, source_uri, round_=None, slot=None, labels=None, case_s=orders.CASE_TIMEOUT_S, order_s=orders.ORDER_TIMEOUT_S) -> dict: """One side's order; for one round and slot of a comparison, or neither (`bx bench`).""" @@ -125,7 +123,7 @@ def _order(side: dict, *, harness, suite, filter, quantity, precision, environme "source": {"uri": source_uri, "type": "git"}, **side["text"]} try: return orders.make_order(suite=harness.suite_entry(suite, filter), target=target, precision=precision, - quantity=quantity, run_key=run_key, project=project, + quantities=quantities, run_key=run_key, project=project, labels={**side["labels"], **(labels or {})}, case_s=case_s, order_s=order_s, environment_variables=environment_variables, round_=round_, slot=slot, benchmark=side.get("benchmark")) @@ -133,13 +131,23 @@ def _order(side: dict, *, harness, suite, filter, quantity, precision, environme raise SessionError(f"work order{'' if round_ is None else f' for round {round_}'}: {e}") from None -def _harness(adapter, flags, cwd) -> tuple: - """The harness named by `adapter` (default pyperf), its complete precision - from `flags` over its defaults, and the config file it read (or None). - A flag the harness does not take is refused here, before any side is resolved.""" - harness = harnesses.get(adapter) if adapter else HARNESS - precision, config_path = harness.precision(harness.flags_to_precision(**(flags or {})), start=cwd) - return harness, precision, config_path +def _harness(adapter, flags, file_precision=None, quantities=None) -> tuple: + """The harness named by `adapter`, and its complete precision from `flags` over + the config file's precision over its defaults. A flag the harness does not + take, a precision or a quantity it cannot apply, is refused here, before any + side is resolved.""" + if not adapter: + raise SessionError("no adapter given: pass --adapter, or set adapter.name in benchx.conf.json") + harness = harnesses.get(adapter) + precision = harness.precision(harness.flags_to_precision(**(flags or {})), file_precision) + try: + adapters.get(harness.name).protocol(precision) + except Unsupported as e: + raise SessionError("; ".join(str(problem) for problem in e.problems)) from None + unknown = sorted(set(quantities or ()) - set(adapters.get(harness.name).QUANTITIES)) + if unknown: + raise SessionError(f"{harness.name} does not report {', '.join(unknown)}") + return harness, precision def _source_uri(sides, source_uri) -> str: @@ -164,38 +172,39 @@ def _run_side(side: dict, path: Path, store=None, **order) -> tuple[dict, dict | return summary, store.ingest_paths(summary["files"]) if store else None -def bench_target(target: str, *, suite: str, store=None, adapter=None, flags=None, environment_variables=None, - filter=None, quantity="wall-time", project=None, run_key=None, out=None, source_uri=None, - case_s=orders.CASE_TIMEOUT_S, order_s=orders.ORDER_TIMEOUT_S, progress=None, build=True, - cwd=".") -> dict: +def bench_target(target: str, *, suite: str, adapter: str, store=None, flags=None, file_precision=None, + environment_variables=None, filter=None, quantities=("wall-time",), project=None, run_key=None, + out=None, source_uri=None, case_s=orders.CASE_TIMEOUT_S, order_s=orders.ORDER_TIMEOUT_S, + progress=None, build=True, provider=None, config_path=None, results_dir=None) -> dict: """Run one target once: a comparison session of one order, with no round, slot or comparison. Results are delivered to `store` unless it is None. `target` is `@CONFIG[=SOURCE_REF]` or a bare path, which the harness reads (see `_side`); `adapter`, `flags` and the rest are as for `compare_targets`. - Returns the runner's summary, with the store's `outcome`, the `precision` - and the `config` file (or None) it came from. Raises SessionError when the - run cannot start, and runner.Refused when the runner refuses the order. + Returns the runner's summary, with the store's `outcome` and the `precision`. + Raises SessionError when the run cannot start, and runner.Refused when the + runner refuses the order. """ - harness, precision, config_path = _harness(adapter, flags, cwd) - side = _side(target, suite, "prepare" if build else "describe", harness, environment_variables, + harness, precision = _harness(adapter, flags, file_precision, quantities) + side = _side(target, suite, "prepare" if build else "describe", harness, environment_variables, provider, timeout=order_s) source_uri = _source_uri([side], source_uri) run_key = run_key or orders.default_run_key("bench") - out = Path(out) if out else Path("results") / run_key.replace("/", "_") + out = Path(out) if out else Path(results_dir or "results") / run_key.replace("/", "_") if progress: progress(f"precision: {harness.describe_precision(precision, config_path)}") summary, outcome = _run_side(side, out / "orders" / "order.json", store, harness=harness, suite=suite, - filter=filter, quantity=quantity, precision=precision, + filter=filter, quantities=quantities, precision=precision, environment_variables=environment_variables, project=project, run_key=run_key, source_uri=source_uri, case_s=case_s, order_s=order_s) - return {**summary, "outcome": outcome, "precision": precision, "config": config_path} + return {**summary, "outcome": outcome, "precision": precision} def compare_targets(baseline: str, contender: str, *, profile: str, suite: str, rounds: int, store, - adapter=None, flags=None, environment_variables=None, filter=None, quantity="wall-time", - project=None, label=None, run_key=None, out=None, source_uri=None, k=3.0, min_rounds=3, - progress=None, build=True, cwd=".") -> dict: + adapter, flags=None, file_precision=None, environment_variables=None, filter=None, + quantities=("wall-time",), project=None, label=None, run_key=None, out=None, source_uri=None, + k=3.0, min_rounds=3, progress=None, build=True, provider=None, results_dir=None, + case_s=orders.CASE_TIMEOUT_S, order_s=orders.ORDER_TIMEOUT_S, config_path=None) -> dict: """Alternate baseline and contender over `rounds` rounds, deliver every result to `store`, and return the run-mode comparison document. @@ -204,9 +213,10 @@ def compare_targets(baseline: str, contender: str, *, profile: str, suite: str, it is only asked to `describe`. For `environments`, `label` is (name, baseline value, contender value). - `adapter` names the harness (default pyperf); `flags` are its - precision flags by keyword (`harnesses.Harness.FLAGS`), over its defaults - and, for pyperf, the config file found from `cwd`. `environment_variables` + `adapter` names the harness; `flags` are its precision flags by keyword + (`harnesses.Harness.FLAGS`), over `file_precision` (the config file's) and + its defaults. `provider` is the (command, directory) a `@CONFIG` target + calls. `quantities` are collected in every order. `environment_variables` are set for both sides alike, and so become workload parameters of both. Raises SessionError when the session cannot start, and runner.Refused when the runner refuses an order. @@ -219,10 +229,10 @@ def compare_targets(baseline: str, contender: str, *, profile: str, suite: str, raise SessionError("the environments profile needs --label NAME=BASE,CONTENDER") if profile == "revisions" and label is not None: raise SessionError("--label names the sides of an environments comparison; revisions are named by their trees") - harness, precision, _ = _harness(adapter, flags, cwd) + harness, precision = _harness(adapter, flags, file_precision, quantities) operation = "prepare" if build else "describe" - sides = [_side(spec, suite, operation, harness, environment_variables, timeout=orders.ORDER_TIMEOUT_S) - for spec in (baseline, contender)] + sides = [_side(spec, suite, operation, harness, environment_variables, provider, + timeout=orders.ORDER_TIMEOUT_S) for spec in (baseline, contender)] if profile == "revisions": # UC-03 §10: refuse before any measurement, not after R rounds. for name, side in zip(("baseline", "contender"), sides): @@ -234,17 +244,17 @@ def compare_targets(baseline: str, contender: str, *, profile: str, suite: str, raise SessionError("the sides' suites come from different sources; give --suite as one absolute path") run_key = run_key or orders.default_run_key("compare") - out = Path(out) if out else Path("results") / run_key.replace("/", "_") + out = Path(out) if out else Path(results_dir or "results") / run_key.replace("/", "_") labels = [{label[0]: label[1]}, {label[0]: label[2]}] if label else [None, None] slot = 0 for round_ in range(rounds): for name, side, side_labels in zip(("baseline", "contender"), sides, labels): summary, outcome = _run_side(side, out / "orders" / f"order-{slot:04d}.json", store, - harness=harness, suite=suite, filter=filter, quantity=quantity, + harness=harness, suite=suite, filter=filter, quantities=quantities, precision=precision, environment_variables=environment_variables, project=project, run_key=run_key, source_uri=source_uri, - round_=round_, slot=slot, labels=side_labels) + round_=round_, slot=slot, labels=side_labels, case_s=case_s, order_s=order_s) if outcome["rejected"]: codes = sorted({r["code"] for r in outcome["rejected"]}) raise SessionError(f"the store rejected results of round {round_} ({name}): {', '.join(codes)}") diff --git a/prototype/benchx/snapshot.py b/prototype/benchx/snapshot.py index f75395c..ac8278d 100644 --- a/prototype/benchx/snapshot.py +++ b/prototype/benchx/snapshot.py @@ -107,20 +107,38 @@ def merge(base: dict, facts: dict) -> dict: _PYCACHE_EXCLUDE = ":(exclude,glob)**/__pycache__/**" +_IGNORED: tuple[Path, ...] = () + + +def ignore(*paths) -> None: + """Paths benchx itself writes (the store, the result files), left out of every + tree identity from now on, so a run's own output never marks the tree dirty + or changes its tree id. Replaces the previous set; no arguments clears it.""" + global _IGNORED + _IGNORED = tuple(Path(p).resolve() for p in paths) + + +def _ignored_in(root: Path) -> list[str]: + """The ignored paths inside `root`, relative to it.""" + root = Path(root).resolve() + return [p.relative_to(root).as_posix() for p in _IGNORED if p != root and p.is_relative_to(root)] + + def git_identity(path) -> dict | None: """Revision, dirty state, and working-tree id of a checkout (schema §4.1).""" top = _run(["git", "rev-parse", "--show-toplevel"], cwd=path) head = top and _run(["git", "rev-parse", "HEAD"], cwd=top) if not head: return None - status = _run(["git", "status", "--porcelain", "--untracked-files=all", "--", ".", _PYCACHE_EXCLUDE], cwd=top) + excludes = [_PYCACHE_EXCLUDE, *(f":(top,exclude){rel}" for rel in _ignored_in(top))] + status = _run(["git", "status", "--porcelain", "--untracked-files=all", "--", ".", *excludes], cwd=top) # A scratch index that does not exist yet, so the repository's own index # is untouched (schema §4.1). with tempfile.TemporaryDirectory() as tmp: index = os.path.join(tmp, "index") env = {**os.environ, "GIT_INDEX_FILE": index} try: - subprocess.run(["git", "add", "-A", "--", ".", _PYCACHE_EXCLUDE], cwd=top, env=env, check=True, capture_output=True) + subprocess.run(["git", "add", "-A", "--", ".", *excludes], cwd=top, env=env, check=True, capture_output=True) tree = subprocess.run(["git", "write-tree"], cwd=top, env=env, check=True, capture_output=True, text=True).stdout.strip() except (OSError, subprocess.CalledProcessError): @@ -192,8 +210,10 @@ def directory_identity(path) -> dict | None: if not root.is_dir(): return None digest = hashlib.sha1() + ignored = tuple(_IGNORED) for file in sorted(p for p in root.rglob("*") if p.is_file() - and not _SKIPPED_DIRS.intersection(p.relative_to(root).parts)): + and not _SKIPPED_DIRS.intersection(p.relative_to(root).parts) + and not any(p.resolve().is_relative_to(i) for i in ignored)): try: content = file.read_bytes() except OSError: diff --git a/prototype/benchx/store.py b/prototype/benchx/store.py index f098c6a..d72ebdf 100644 --- a/prototype/benchx/store.py +++ b/prototype/benchx/store.py @@ -47,19 +47,12 @@ def store_project(doc: dict) -> str: return f"local/{ident.get('runner') or ident.get('hostname') or 'unknown'}" -def home() -> Path: - """benchx's home: ~/.benchx, or $BENCHX_HOME to relocate it (tests, CI).""" - return Path(os.environ.get("BENCHX_HOME") or Path.home() / ".benchx") - - -def default_path() -> Path: - """The local store's one place (prototype-design.md §4).""" - return home() / "store.parquet" - - class Store: - def __init__(self, path=None): - self.path = Path(path) if path is not None else default_path() + """The local store, one Parquet file (prototype-design.md §4). Its place is the + project's: `Config.store_path`, by default `store.parquet` in the project root.""" + + def __init__(self, path): + self.path = Path(path) # -- reading ------------------------------------------------------------- diff --git a/prototype/benchx/target.py b/prototype/benchx/target.py index a1525fd..9cc8eeb 100644 --- a/prototype/benchx/target.py +++ b/prototype/benchx/target.py @@ -5,8 +5,8 @@ three ways, and `resolve` handles each: - `@CONFIG[=SOURCE_REF]`: the project's provider command, named in - `.benchx/provider.json`, asked to `prepare` (it may build) or `describe` - (it only reports). + `benchx.conf.json`, asked to `prepare` (it may build) or `describe` (it only + reports). - `BUILD_DIR[:SOURCE_DIR]` with a sidecar `.benchx-target.json` in the build directory, written by whatever built it. - `BUILD_DIR[:SOURCE_DIR]` alone: no description; the workbench identifies the @@ -24,7 +24,6 @@ from .core import loader SIDECAR = ".benchx-target.json" -PROVIDER_FILE = Path(".benchx") / "provider.json" TEXT_FIELDS = ("how_built", "activation", "shell", "provider") @@ -48,34 +47,6 @@ def text(description: dict | None) -> dict: return out -def find_upward(relative, start="."): - """The nearest file `relative` (e.g. `.benchx/config.json`) walking up from - `start`, or None.""" - start = Path(start).resolve() - for directory in (start, *start.parents): - path = directory / relative - if path.is_file(): - return path - return None - - -def find_provider(start="."): - """The provider command from the nearest `.benchx/provider.json`, walking - up from `start`. Returns (command, directory it is run in), or None.""" - start = Path(start).resolve() - for directory in (start, *start.parents): - path = directory / PROVIDER_FILE - if path.is_file(): - config, errors = loader.read(path) - if config is None: - raise TargetError(f"{path}: " + "; ".join(str(error) for error in errors)) - command = config.get("command") - if not (isinstance(command, list) and command and all(isinstance(c, str) for c in command)): - raise TargetError(f'{path} must be {{"command": ["program", "arg", ...]}}') - return command, directory - return None - - def call_provider(command, cwd, operation, source_ref, configuration) -> dict: """One request to the provider: JSON on stdin, the description on stdout. @@ -123,20 +94,20 @@ def sidecar(build_dir) -> dict | None: return description -def resolve(spec: str, *, operation="prepare", cwd=".") -> dict | None: +def resolve(spec: str, *, operation="prepare", provider=None) -> dict | None: """The description for one target spec, or None when there is none. - `@CONFIG[=SOURCE_REF]` asks the provider (`prepare` by default); a build - directory yields its sidecar. SOURCE_REF defaults to WORKSPACE. + `@CONFIG[=SOURCE_REF]` asks the provider, a (command, directory) pair + (`Config.provider`), to `prepare` by default; a build directory yields its + sidecar. SOURCE_REF defaults to WORKSPACE. """ if spec.startswith("@"): configuration, _, source_ref = spec[1:].partition("=") if not configuration: raise TargetError(f"empty configuration name in target {spec!r}") - found = find_provider(cwd) - if found is None: - raise TargetError(f"{spec} needs a provider: no {PROVIDER_FILE} in {Path(cwd).resolve()} or above") - command, directory = found + if provider is None: + raise TargetError(f"{spec} needs a provider") + command, directory = provider return call_provider(command, directory, operation, source_ref or "WORKSPACE", configuration) build = spec.partition(":")[0] return sidecar(build) if build else None diff --git a/prototype/demo.sh b/prototype/demo.sh index bcbb398..ed46383 100644 --- a/prototype/demo.sh +++ b/prototype/demo.sh @@ -15,16 +15,18 @@ # Google Benchmark unless it is installed. ROUNDS=5 by default. # # `bx compare` (measure mode) and `bx run` deliver every result to the one local store, -# ~/.benchx/store.parquet (BENCHX_HOME=... relocates it), so the store and +# examples/demo/store.parquet (paths.store in examples/demo/benchx.conf.json, the demo's +# project file, from whose directory every `bx` command runs), so the store and # each series' history grow from run to run. `bx head` shows the latest rows -# at any time. +# at any time. Delete the file to start over. set -euo pipefail cd "$(dirname "$0")" PROTOTYPE=$PWD PYTHON=${PYTHON:-$PROTOTYPE/.venv/bin/python} ROUNDS=${ROUNDS:-5} WORK=$(mktemp -d) -STORE="${BENCHX_HOME:-$HOME/.benchx}/store.parquet" +DEMO_ROOT=$PROTOTYPE/examples/demo +STORE=$DEMO_ROOT/store.parquet STAMP=$(date -u +%Y%m%dT%H%M%SZ) # run keys are unique per demo run, so runs accumulate RUN_REV="rev-adhoc-$STAMP" RUN_ENV="env-tracked-$STAMP" @@ -47,7 +49,7 @@ say() { # to stderr, so a command's own redirected output never contains it } show() { say "$@"; "$@"; } # print, then run quiet() { say "$@"; "$@" >/dev/null; } # print, then run without its output -bx() { say bx "$@"; "$PYTHON" -m benchx.cli "$@"; } +bx() { say bx "$@"; (cd "$DEMO_ROOT" && "$PYTHON" -m benchx.cli "$@"); } into() { # into : run with stdout to a file, printed with its redirect SAY_SUFFIX="> $1"; local out=$1; shift "$@" > "$out" @@ -82,18 +84,18 @@ build "$WORK/wt-head" "$WORK/build-hardened" -DDEMO_HARDENED=ON step "revisions, ad hoc: no project, measured and compared in one command" echo "# measure mode: the sides alternate over $ROUNDS rounds, one work order each" -bx compare "$WORK/build-base:$WORK/wt-base" "$WORK/build-head:$WORK/wt-head" --profile revisions \ - --adapter google-benchmark --suite demo-bench --rounds "$ROUNDS" --run-key "$RUN_REV" --out "$WORK/results-rev" +bx compare "$WORK/build-base:$WORK/wt-base" "$WORK/build-head:$WORK/wt-head" \ + --rounds "$ROUNDS" --run-key "$RUN_REV" --out "$WORK/results-rev" # adapter, suite and profile: benchx.conf.json echo "# the first work order it wrote:" show cat "$WORK/results-rev/orders/order-0000.json" echo echo "# read mode: the same run key, compared again through a throwaway store" -bx compare --run "$RUN_REV" --profile revisions --baseline "${BASE:0:10}" --results "$WORK/results-rev" +bx compare --run "$RUN_REV" --baseline "${BASE:0:10}" --results "$WORK/results-rev" echo "# bx compare also delivered these to the local store, as thin results of project local/" step "environments, tracked: project demo, into the local store" bx compare "$WORK/build-head:$WORK/wt-head" "$WORK/build-hardened:$WORK/wt-head" --profile environments \ - --label build=plain,hardened --adapter google-benchmark --suite demo-bench --rounds "$ROUNDS" --project demo \ + --label build=plain,hardened --rounds "$ROUNDS" --project demo \ --run-key "$RUN_ENV" --out "$WORK/results-env" step "re-ingest is a visible no-op" diff --git a/prototype/examples/archery-local/.benchx/provider.json b/prototype/examples/archery-local/.benchx/provider.json deleted file mode 100644 index c45fe02..0000000 --- a/prototype/examples/archery-local/.benchx/provider.json +++ /dev/null @@ -1 +0,0 @@ -{"command": ["./provider.sh"]} diff --git a/prototype/examples/archery-local/README.md b/prototype/examples/archery-local/README.md index 7f67981..dfbbe50 100644 --- a/prototype/examples/archery-local/README.md +++ b/prototype/examples/archery-local/README.md @@ -42,8 +42,8 @@ script runs `bx` and `python` from the active environment. More rounds: `ROUNDS=10 ./run.sh`. Everything it creates is under `_work/` in this directory, including the local -store (`_work/benchx-home`), so `~/.benchx` is untouched. `rm -rf _work` starts -over. +store (`_work/store.parquet`, set by `paths.store` in `benchx.conf.json`), so nothing +outside this directory changes. `rm -rf _work` starts over. ## What you will see @@ -100,18 +100,16 @@ this directory, with the environment still active: ```console $ source activate.sh -$ export BENCHX_HOME=$PWD/_work/benchx-home $ bx target describe @hardened # what exists, nothing built $ bx target prepare @hardened # incremental build, then the description $ bx target describe _work/build-hardened # a bare directory: its own sidecar # the story's own knobs: scope by filter, precision by repetitions -$ bx compare @plain @hardened --profile environments --label build=plain,hardened \ - --adapter google-benchmark --suite demo-bench --filter BM_Quiet --rounds 10 --repetitions 5 \ - --project archery-local --run-key mine +$ bx compare @plain @hardened --label build=plain,hardened --filter BM_Quiet --rounds 10 \ + --repetitions 5 --run-key mine # the adapter, suite, project and profile come from benchx.conf.json -$ bx compare --run mine --profile environments --label build --baseline plain # read mode +$ bx compare --run mine --label build --baseline plain # read mode $ bx head -n 4 # the latest stored results $ bx head --json -n 1 | python3 -m json.tool | less # provenance.info, environment.system ``` @@ -127,7 +125,7 @@ runner simply omits it from the results. |---|---| | `run.sh` | the end-to-end script | | `provider.sh` | the target provider: your build script, speaking the `describe`/`prepare` contract | -| `.benchx/provider.json` | tells the workbench which command is the provider | +| `benchx.conf.json` | the project file: the provider command, the harness and its suite, the project name, the comparison profile, and where the store and results go | | `activate.sh` | the environment you activate before calling `bx` | | `_work/` | everything generated: source copy, build directories, results, store (git-ignored) | diff --git a/prototype/examples/archery-local/benchx.conf.json b/prototype/examples/archery-local/benchx.conf.json new file mode 100644 index 0000000..9029306 --- /dev/null +++ b/prototype/examples/archery-local/benchx.conf.json @@ -0,0 +1,13 @@ +{ + "schema_version": "benchx/config/0.1.0", + "project": "archery-local", + "provider": { "command": ["./provider.sh"] }, + "adapter": { + "name": "google-benchmark", + "suite": "demo-bench", + "quantities": ["wall-time"], + "precision": { "repetitions": 5 } + }, + "paths": { "store": "_work/store.parquet", "results": "_work/results" }, + "compare": { "profile": "environments" } +} diff --git a/prototype/examples/archery-local/run.sh b/prototype/examples/archery-local/run.sh index 8582786..6389898 100644 --- a/prototype/examples/archery-local/run.sh +++ b/prototype/examples/archery-local/run.sh @@ -4,7 +4,7 @@ # noise-aware verdicts, where the comparison describes itself. # # You play the user. The target provider (provider.sh, named in -# .benchx/provider.json) is your own build script: it builds with CMake and +# benchx.conf.json) is your own build script: it builds with CMake and # tells benchx what it built, how, and what environment you need. benchx builds # nothing, and runs none of that text: you activate the environment yourself # (activate.sh) and benchx records the text beside the environment it inherited. @@ -17,13 +17,12 @@ # # ROUNDS=5 alternation rounds (default 5) # -# The local store is kept inside this directory (_work/benchx-home), not in -# ~/.benchx. Delete _work/ to start over. +# benchx.conf.json is this directory's project file, so the local store is kept +# inside it too (_work/store.parquet). Delete _work/ to start over. set -euo pipefail cd "$(dirname "$0")" HERE=$PWD ROUNDS=${ROUNDS:-5} -export BENCHX_HOME=${BENCHX_HOME:-$HERE/_work/benchx-home} STAMP=$(date -u +%Y%m%dT%H%M%SZ) RUN="hardening-$STAMP" @@ -55,8 +54,7 @@ head -3 _work/build-hardened/.benchx-target.json echo "(the same document: a build directory carries its own description)" step "compare: the workbench asks the provider to prepare each side (a no-op), then alternates them" -bx compare @plain @hardened --profile environments --label build=plain,hardened \ - --adapter google-benchmark --suite demo-bench --rounds "$ROUNDS" --project archery-local --run-key "$RUN" --out _work/results +bx compare @plain @hardened --label build=plain,hardened --rounds "$ROUNDS" --run-key "$RUN" --out _work/results step "the comparison names what was built and how; before, it was two opaque directory paths" echo "# the target of work order 0001 (round 1, the hardened side), as the runner received it:" @@ -80,8 +78,7 @@ print("coordinates.subject.configuration:", doc["coordinates"]["subject"].get("c step "forget to activate: the declaration still says what you should have run, the observation says what you did" say env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build --rounds 3 ... -env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build --profile environments \ - --label build=plain,hardened --adapter google-benchmark --suite demo-bench --rounds 3 --project archery-local \ - --run-key "$RUN-unactivated" --out _work/results-unactivated +env -u LD_LIBRARY_PATH bx compare @plain @hardened --no-build \ + --label build=plain,hardened --rounds 3 --run-key "$RUN-unactivated" --out _work/results-unactivated printf '\nartifacts: %s/_work (results, orders, build directories, the local store)\n' "$HERE" diff --git a/prototype/examples/demo/.gitignore b/prototype/examples/demo/.gitignore new file mode 100644 index 0000000..13eecef --- /dev/null +++ b/prototype/examples/demo/.gitignore @@ -0,0 +1,2 @@ +store.parquet +results/ diff --git a/prototype/examples/demo/benchx.conf.json b/prototype/examples/demo/benchx.conf.json new file mode 100644 index 0000000..fae92f0 --- /dev/null +++ b/prototype/examples/demo/benchx.conf.json @@ -0,0 +1,10 @@ +{ + "schema_version": "benchx/config/0.1.0", + "adapter": { + "name": "google-benchmark", + "suite": "demo-bench", + "precision": { "repetitions": 5 } + }, + "paths": { "store": "store.parquet", "results": "results" }, + "compare": { "profile": "revisions" } +} diff --git a/prototype/examples/pyperf-bench/.benchx/config.json b/prototype/examples/pyperf-bench/.benchx/config.json deleted file mode 100644 index e2e9672..0000000 --- a/prototype/examples/pyperf-bench/.benchx/config.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "adapters": { - "pyperf": { - "precision": { - "repetitions": { - "mode": "fixed", - "levels": [{"unit": "process", "n": 5}, {"unit": "value", "n": 3}] - }, - "calibration": {"mode": "adaptive", "minimum_sample_seconds": 0.05}, - "warmup": {"mode": "count", "n_warmup": 1} - } - } - } -} diff --git a/prototype/examples/pyperf-bench/README.md b/prototype/examples/pyperf-bench/README.md index 1f2c432..af112cf 100644 --- a/prototype/examples/pyperf-bench/README.md +++ b/prototype/examples/pyperf-bench/README.md @@ -17,23 +17,23 @@ The script runs `bx` and `python` from the active environment, so benchx and pyp be installed there; the workers use that same `python`. Extra arguments go to `bx bench` in step 1 and win over the config file, e.g. `./run.sh --processes 20`. It takes about 20 seconds. Nothing outside this directory changes: the local store is -`out/home/store.parquet` (the script sets `BENCHX_HOME`), and `out/` is deleted at -the start of each run. +`out/store.parquet` (`paths.store` in `benchx.conf.json`), and `out/` is deleted at the start +of each run. ## What goes in | Input | Where | What it does | |---|---|---| -| `bench_suite.py` | first argument | The module. Each function named `bench_*` or `benchmark_*` is a case. | -| Function names | after the module | `bx bench bench_suite.py bench_sorted bench_join` runs only those, any function, whatever its name. | -| `.benchx/config.json` | found by walking up from where you run `bx` | Precision: worker processes, values per process, calibration, warmup. Here: 5 processes x 3 values, calibrate to 0.05 s per value. | -| Flags | command line | `--processes`, `--values`, `--min-time`, `--loops`, `--warmups` override the file. | +| `.` | first argument | The target: this directory, a working tree. pyperf reads a bare path that way. | +| `benchx.conf.json` | found by walking up from where you run `bx`; its directory is the project root | The harness (`adapter.name`), the module (`adapter.suite`, `bench_suite.py`: each function named `bench_*` or `benchmark_*` is a case), and the precision: worker processes, values per process, calibration, warmup. Here: 5 processes x 3 values, calibrate to 0.05 s per value. | +| Function names | after the target | `bx bench . bench_sorted bench_join` runs only those, any function, whatever its name. `bx bench bench_suite.py bench_sorted` still works as a shorthand: the module's git checkout is the target and its path the suite. | +| Flags | command line | `--processes`, `--values`, `--min-time`, `--loops`, `--warmups` override the file, and so does `--suite`. | | `-e NAME=VALUE` | command line | Sets an environment variable for the workers. The workers otherwise inherit your whole environment. | Precision is resolved as: **flags, then the config file, then pyperf's defaults** (20 processes x 3 values, 0.1 s calibration, 1 warmup). A level you leave out of the -file takes pyperf's default. `bx config init` writes a `.benchx/config.json` that states -the defaults, as a template to edit. The resolved values are written into the work order, +file takes pyperf's default. `bx config init` writes a `benchx.conf.json` that states +the defaults, as a template to edit, and `bx config show` says where each value comes from. The resolved values are written into the work order, so a result records the precision that applied. A bad config file is reported before anything runs. @@ -42,23 +42,25 @@ the calibration (how many calls make up one value) itself. Put setup at module l ## The config file -`.benchx/config.json` (a hidden directory: use `ls -a`) is the example's precision config. -It is the work-order `precision` block under the adapter's name: +`benchx.conf.json` is the example's project file. Its `adapter.precision` is the work-order +`precision` block; `adapter.name` and `adapter.suite` say which harness and which module: ```json { - "adapters": { - "pyperf": { - "precision": { - "repetitions": { - "mode": "fixed", - "levels": [{"unit": "process", "n": 5}, {"unit": "value", "n": 3}] - }, - "calibration": {"mode": "adaptive", "minimum_sample_seconds": 0.05}, - "warmup": {"mode": "count", "n_warmup": 1} - } + "schema_version": "benchx/config/0.1.0", + "adapter": { + "name": "pyperf", + "suite": "bench_suite.py", + "precision": { + "repetitions": { + "mode": "fixed", + "levels": [{"unit": "process", "n": 5}, {"unit": "value", "n": 3}] + }, + "calibration": {"mode": "adaptive", "minimum_sample_seconds": 0.05}, + "warmup": {"mode": "count", "n_warmup": 1} } - } + }, + "paths": {"store": "out/store.parquet", "results": "out"} } ``` @@ -72,10 +74,10 @@ It is the work-order `precision` block under the adapter's name: ## What `run.sh` does -1. `bx bench bench_suite.py` runs all five cases with the config file's precision. -2. `bx bench bench_suite.py bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2` +1. `bx bench .` runs all five cases with the config file's suite and precision. +2. `bx bench . bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2` runs one case, sets a variable for the workers, and overrides the file's process count. -3. `bx bench bench_suite.py bench_join --run-key example-again` runs one case a second time +3. `bx bench . bench_join --run-key example-again` runs one case a second time under a new run key, so its series gets a second point. 4. `ls -l` and `bx head` show the local store: one Parquet file, and its latest rows. 5. `bx series` lists the series in the store. diff --git a/prototype/examples/pyperf-bench/benchx.conf.json b/prototype/examples/pyperf-bench/benchx.conf.json new file mode 100644 index 0000000..bc59544 --- /dev/null +++ b/prototype/examples/pyperf-bench/benchx.conf.json @@ -0,0 +1,13 @@ +{ + "schema_version": "benchx/config/0.1.0", + "adapter": { + "name": "pyperf", + "suite": "bench_suite.py", + "precision": { + "repetitions": { "mode": "fixed", "levels": [{ "unit": "process", "n": 5 }, { "unit": "value", "n": 3 }] }, + "calibration": { "mode": "adaptive", "minimum_sample_seconds": 0.05 }, + "warmup": { "mode": "count", "n_warmup": 1 } + } + }, + "paths": { "store": "out/store.parquet", "results": "out" } +} diff --git a/prototype/examples/pyperf-bench/run.sh b/prototype/examples/pyperf-bench/run.sh index 75c3a04..21a2fb9 100644 --- a/prototype/examples/pyperf-bench/run.sh +++ b/prototype/examples/pyperf-bench/run.sh @@ -1,16 +1,15 @@ #!/usr/bin/env bash # Run the example suite with `bx bench`. See README.md for what each step does. # -# ./run.sh uses .benchx/config.json here (5 processes x 3 values) +# ./run.sh uses benchx.conf.json here (5 processes x 3 values) # ./run.sh --processes 20 extra arguments go to `bx bench` and win over the file # # Run it in an activated environment that has benchx installed with pyperf # (from prototype/: `pip install -e . pyperf`, or `uv sync --extra test`). -# Results go to ./out, and the local store to ./out/home (BENCHX_HOME), so -# nothing outside this directory changes. +# Results go to ./out, and the local store to ./out/store.parquet (both set in +# benchx.conf.json), so nothing outside this directory changes. set -euo pipefail cd "$(dirname "$0")" -export BENCHX_HOME="$PWD/out/home" bx() { printf '\033[36m$ bx %s\033[0m\n' "$*" >&2; command bx "$@"; } command bx --help >/dev/null 2>&1 && python -c "import pyperf" 2>/dev/null || { @@ -18,18 +17,18 @@ command bx --help >/dev/null 2>&1 && python -c "import pyperf" 2>/dev/null || { exit 1; } rm -rf out; mkdir -p out -echo "== 1. all bench_* functions, precision from .benchx/config.json" -bx bench bench_suite.py --run-key example-all --out out/all "$@" +echo "== 1. all bench_* functions, suite and precision from benchx.conf.json" +bx bench . --run-key example-all --out out/all "$@" echo; echo "== 2. one function, an environment variable set for the workers, flags over the file" -bx bench bench_suite.py bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2 \ +bx bench . bench_env_var -e EXAMPLE_N=100000 --processes 3 --values 2 \ --run-key example-env --out out/env echo; echo "== 3. the same case again under a new run key, so its series gets a second point" -bx bench bench_suite.py bench_join --run-key example-again --out out/again +bx bench . bench_join --run-key example-again --out out/again echo; echo "== 4. what is in the local store: one Parquet file" -ls -l "$BENCHX_HOME/store.parquet" +ls -l out/store.parquet bx head -n 10 echo; echo "== 5. its series (bench_env_var has two: the -e setting changes the environment)" diff --git a/prototype/tests/conftest.py b/prototype/tests/conftest.py index dc87026..32001ca 100644 --- a/prototype/tests/conftest.py +++ b/prototype/tests/conftest.py @@ -14,7 +14,7 @@ import pytest -from benchx import runner +from benchx import runner, snapshot from benchx.core.catalog import SCHEMAS EXAMPLES = SCHEMAS / "measurement-result" / "0.1.0" / "examples" @@ -120,9 +120,15 @@ def run_rounds(tmp: Path, sides: list[dict], rounds: int, run_key: str, out: Pat @pytest.fixture(autouse=True) def _isolated(monkeypatch, tmp_path): - """No test reads a setup file or touches the real ~/.benchx.""" + """No test reads a setup file, and each runs in its own project root: with no + benchx.conf.json the working directory is the root, so the store and the results + land in the test's directory.""" monkeypatch.delenv("BENCHX_SETUP_FILE", raising=False) - monkeypatch.setenv("BENCHX_HOME", str(tmp_path / "benchx-home")) + monkeypatch.delenv("BENCHX_HOME", raising=False) + monkeypatch.chdir(tmp_path) + snapshot.ignore() + yield + snapshot.ignore() @pytest.fixture diff --git a/prototype/tests/test_asv.py b/prototype/tests/test_asv.py index 84d2fc5..ddf8887 100644 --- a/prototype/tests/test_asv.py +++ b/prototype/tests/test_asv.py @@ -154,7 +154,7 @@ def test_asv_command_runs_every_time_benchmark(suite, capsys): assert (broken["measurement"]["status"], broken["measurement"]["reason"]) == ("error", "harness.error") assert "boom" in broken["provenance"]["info"]["message"] assert not any("peakmem" in name for name, _ in found) - assert {d["coordinates"]["workload"]["name"] for d in Store().documents("all")} >= {"bm.time_plain"} + assert {d["coordinates"]["workload"]["name"] for d in Store(suite / "store.parquet").documents("all")} >= {"bm.time_plain"} def test_filter_and_benchmark_defaults(suite, capsys): diff --git a/prototype/tests/test_bench.py b/prototype/tests/test_bench.py index 2de0866..6aa6d64 100644 --- a/prototype/tests/test_bench.py +++ b/prototype/tests/test_bench.py @@ -9,7 +9,7 @@ pytest.importorskip("pyperf") -from benchx import cli, core, harnesses, session +from benchx import cli, config, core, harnesses, session from benchx.errors import SessionError from benchx.store import Store from conftest import SOURCE_URI @@ -32,9 +32,9 @@ def levels(precision): def write_config(directory, precision): - path = directory / ".benchx" / "config.json" - path.parent.mkdir(exist_ok=True) - path.write_text(json.dumps({"adapters": {"pyperf": {"precision": precision}}})) + path = directory / "benchx.conf.json" + path.write_text(json.dumps({"schema_version": "benchx/config/0.1.0", + "adapter": {"name": "pyperf", "precision": precision}})) return path @@ -49,12 +49,13 @@ def tree(tmp, monkeypatch): def test_order_holds_the_resolved_precision(tree): write_config(tree, {"repetitions": {"mode": "fixed", "levels": [{"unit": "process", "n": 7}]}}) - harness, precision, config_path = session._harness("pyperf", {"values": 5}, cwd=tree) + cfg = config.load(tree) + harness, precision = session._harness("pyperf", {"values": 5}, cfg.adapter["precision"]) side = session._side(str(tree), "mod.py", "describe", harness) order = session._order(side, harness=harness, suite="mod.py", filter=harnesses.exact_filter(["bench_a"]), - quantity="wall-time", precision=precision, environment_variables=None, project=None, + quantities=["wall-time"], precision=precision, environment_variables=None, project=None, run_key="r", source_uri=SOURCE_URI) - assert config_path == tree / ".benchx" / "config.json" + assert cfg.path == tree / "benchx.conf.json" assert core.check(order, "work-order") == [] assert levels(order["precision"]) == {"process": 7, "value": 5} assert order["suites"][0] == {"adapter": "pyperf", "suite": "mod.py", "filter": "^(bench_a)$"} @@ -72,7 +73,7 @@ def test_bench_command_runs_and_ingests(tree, capsys): out = capsys.readouterr() assert "bench_a" in out.out and "success" in out.out and "bench_b" not in out.out assert "2 processes x 2 values" in out.err - results = [d for d in Store().documents("cli") if d["measurement"]["quantity"]["name"] == "wall-time"] + results = [d for d in Store(tree / "store.parquet").documents("cli") if d["measurement"]["quantity"]["name"] == "wall-time"] assert [d["coordinates"]["workload"]["name"] for d in results] == ["bench_a"] order = json.loads(next((tree / "results" / "cli").glob("workorder-*.json")).read_text()) assert levels(order["precision"]) == {"process": 2, "value": 2} diff --git a/prototype/tests/test_bench_harnesses.py b/prototype/tests/test_bench_harnesses.py index 4557b1f..ad41f72 100644 --- a/prototype/tests/test_bench_harnesses.py +++ b/prototype/tests/test_bench_harnesses.py @@ -14,6 +14,7 @@ from benchx import bench, cli, core from benchx.store import Store from conftest import EXAMPLES, QUIET, NOISY, cases, git, make_build +from test_config import write_config from test_target import PROVIDER, describe GBENCH = ["--adapter", "google-benchmark", "--suite", "demo-bench"] @@ -39,7 +40,7 @@ def test_one_google_benchmark_run_prints_a_table_and_delivers(build, tmp, capsys assert (order["suites"][0], order["target"]["kind"]) == ( {"adapter": "google-benchmark", "suite": "demo-bench"}, "build") assert "round" not in order and "slot" not in order - assert {d["coordinates"]["workload"]["name"] for d in Store().documents("g1")} == {QUIET, NOISY} + assert {d["coordinates"]["workload"]["name"] for d in Store(tmp / "store.parquet").documents("g1")} == {QUIET, NOISY} def test_head_table_prints_the_bench_columns_of_the_stored_results(build, capsys): @@ -92,7 +93,7 @@ def listed(*args): def test_the_source_comes_from_cmake_cache(build, repo, tmp): assert cli.main(["bench", str(build), *GBENCH, "--run-key", "g2", "--no-ingest"]) == cli.EXIT_OK assert order_of(tmp, "g2")["target"]["source_dir"] == str(repo["wt-head"].resolve()) - assert not Store().path.exists() # --no-ingest + assert not (tmp / "store.parquet").exists() # --no-ingest def test_cases_are_named_or_filtered(build, tmp, capsys): @@ -115,10 +116,11 @@ def test_precision_and_quantity_are_the_harnesss(build, tmp): @pytest.mark.parametrize("argv, message", [ (["BUILD", *GBENCH, "--processes", "2"], "google-benchmark takes no --processes"), (["BUILD", *GBENCH, "--loops", "5"], "google-benchmark takes no --loops"), - (["BUILD", "--suite", "mod.py", "--repetitions", "3"], "pyperf takes no --repetitions"), # the default + (["BUILD", "--suite", "mod.py", "--adapter", "pyperf", "--repetitions", "3"], "pyperf takes no --repetitions"), + (["BUILD", "--suite", "mod.py"], "--adapter is required"), (["BUILD", *GBENCH, "--filter", "Quiet", QUIET], "case names or --filter"), (["BUILD", *GBENCH, "--tree", "t"], "--tree belongs to"), - (["BUILD", "--adapter", "google-benchmark"], "give --suite, a Python file"), + (["BUILD", "--adapter", "google-benchmark"], "--suite is required"), (["m.py", "--adapter", "google-benchmark"], "benchmarked with pyperf"), ]) def test_what_does_not_fit_is_refused_before_anything_runs(argv, message, build, tmp, capsys): @@ -140,12 +142,11 @@ def test_a_provider_target_is_resolved_like_compare_does(repo, build, tmp, monke script.parent.mkdir() script.write_text(PROVIDER.format(python=sys.executable, descriptions=str(descriptions))) script.chmod(script.stat().st_mode | stat.S_IXUSR) - (tmp / "proj" / ".benchx").mkdir() - (tmp / "proj" / ".benchx" / "provider.json").write_text(json.dumps({"command": [str(script)]})) + write_config(tmp / "proj", provider={"command": [str(script)]}) monkeypatch.chdir(tmp / "proj") assert cli.main(["bench", "@plain", *GBENCH, "--run-key", "pr1"]) == cli.EXIT_OK assert QUIET in capsys.readouterr().out - [doc, *_] = Store().documents("pr1") + [doc, *_] = Store(tmp / "proj" / "store.parquet").documents("pr1") assert doc["provenance"]["info"]["target_provider"]["version"] == "prepare" assert cli.main(["bench", "@plain", *GBENCH, "--run-key", "pr2", "--no-build", "--no-ingest"]) == cli.EXIT_OK result = next(p for p in (tmp / "proj" / "results" / "pr2").glob("*.json") if not p.name.startswith("workorder-")) @@ -168,7 +169,8 @@ def test_pyperf_takes_a_working_tree_path_like_the_shorthand(tmp, monkeypatch, c git(tree, "commit", "-qm", "m") monkeypatch.chdir(tmp) fast = ["--processes", "2", "--values", "2", "--loops", "20", "--no-ingest"] - assert cli.main(["bench", str(tree), "--suite", "benchmarks/mod.py", "bench_a", "--run-key", "w1", *fast]) == cli.EXIT_OK + assert cli.main(["bench", str(tree), "--adapter", "pyperf", "--suite", "benchmarks/mod.py", "bench_a", + "--run-key", "w1", *fast]) == cli.EXIT_OK assert "bench_a" in capsys.readouterr().out long_form = order_of(tmp, "w1") assert cli.main(["bench", str(tree / "benchmarks" / "mod.py"), "bench_a", "--run-key", "w2", *fast]) == cli.EXIT_OK @@ -177,3 +179,97 @@ def test_pyperf_takes_a_working_tree_path_like_the_shorthand(tmp, monkeypatch, c order.pop("work_order_id"), order.pop("run_key") assert long_form == short_form assert long_form["target"]["kind"] == "working_tree" + + +# --- from benchx.conf.json ------------------------------------------------------------------- + +CONFIG = {"project": "demo", + "adapter": {"name": "google-benchmark", "suite": "demo-bench", "quantities": ["wall-time", "cpu-time"], + "precision": {"repetitions": 3}}, + "environment_variables": {"DEMO_THREADS": "1"}} + + +def test_a_bare_command_takes_everything_from_the_file(build, tmp, capsys): + write_config(tmp, **CONFIG) + assert cli.main(["bench", str(build), QUIET, "--run-key", "f1"]) == cli.EXIT_OK + assert QUIET in capsys.readouterr().out + order = order_of(tmp, "f1") + assert (order["project"], order["quantities"], order["precision"]["repetitions"]) == ( + "demo", ["wall-time", "cpu-time"], 3) + assert order["environment_variables"] == {"DEMO_THREADS": "1"} + assert order["suites"][0] == {"adapter": "google-benchmark", "suite": "demo-bench", "filter": "^(BM_Quiet/1024)$"} + names = {(d["coordinates"]["workload"]["name"], d["measurement"]["quantity"]["name"]) + for d in Store(tmp / "store.parquet").documents("f1") if d["measurement"]["quantity"]["name"] != "load-average"} + assert names == {(QUIET, "wall-time"), (QUIET, "cpu-time")} # one run, both quantities + assert {d["project"] for d in Store(tmp / "store.parquet").documents("f1")} == {"demo"} + + +def test_a_flag_wins_over_the_file(build, tmp): + write_config(tmp, **CONFIG) + assert cli.main(["bench", str(build), "--repetitions", "4", "--quantity", "cpu-time", "-e", "DEMO_THREADS=2", + "-e", "OTHER=x", "--project", "other", "--run-key", "f2", "--no-ingest"]) == cli.EXIT_OK + order = order_of(tmp, "f2") + assert (order["project"], order["quantities"], order["precision"]["repetitions"]) == ("other", ["cpu-time"], 4) + assert order["environment_variables"] == {"DEMO_THREADS": "2", "OTHER": "x"} + + +def test_no_project_makes_the_run_ad_hoc(build, tmp): + write_config(tmp, **CONFIG) + assert cli.main(["bench", str(build), "--no-project", "--run-key", "f3"]) == cli.EXIT_OK + assert "project" not in order_of(tmp, "f3") + docs = Store(tmp / "store.parquet").documents("f3") + assert docs and all("project" not in d for d in docs) + + +def test_a_missing_required_value_is_a_hard_failure_naming_both_places(build, tmp, capsys): + write_config(tmp, adapter={"name": "google-benchmark"}) + assert cli.main(["bench", str(build)]) == cli.EXIT_REFUSED + assert "--suite is required: give it, or set adapter.suite in" in capsys.readouterr().err + assert not (tmp / "results").exists() + + +def test_the_files_adapter_is_the_only_one_its_settings_fit(build, tmp, capsys): + write_config(tmp, **CONFIG) + assert cli.main(["bench", str(build), "--adapter", "pyperf"]) == cli.EXIT_REFUSED + assert "adapter.name to 'google-benchmark'" in capsys.readouterr().err + (tmp / "mod.py").write_text("def bench_a():\n pass\n") + assert cli.main(["bench", "mod.py"]) == cli.EXIT_REFUSED # the shorthand is pyperf's + assert "google-benchmark" in capsys.readouterr().err + + +def test_a_quantity_the_harness_does_not_report_is_refused(build, tmp, capsys): + assert cli.main(["bench", str(build), *GBENCH, "--quantity", "peak-rss"]) == cli.EXIT_REFUSED + assert "does not report peak-rss" in capsys.readouterr().err + + +def test_paths_come_from_the_file(build, tmp): + write_config(tmp, adapter=CONFIG["adapter"], paths={"store": "data/s.parquet", "results": "out"}) + (tmp / "data").mkdir() + assert cli.main(["bench", str(build), "--run-key", "f4"]) == cli.EXIT_OK + assert (tmp / "data" / "s.parquet").exists() and (tmp / "out" / "f4" / "orders" / "order.json").exists() + assert not (tmp / "store.parquet").exists() and not (tmp / "results").exists() + + +def test_a_command_run_from_a_subdirectory_uses_the_project_root(build, tmp, monkeypatch): + write_config(tmp, **CONFIG) + (tmp / "sub").mkdir() + monkeypatch.chdir(tmp / "sub") + assert cli.main(["bench", str(build), "--run-key", "f5", "--no-ingest"]) == cli.EXIT_OK + assert (tmp / "results" / "f5" / "orders" / "order.json").exists() and not (tmp / "sub" / "results").exists() + + +def test_a_provider_target_without_a_provider_is_refused(build, tmp, capsys): + write_config(tmp, adapter=CONFIG["adapter"]) + assert cli.main(["bench", "@plain"]) == cli.EXIT_REFUSED + assert '@plain needs "provider"' in capsys.readouterr().err + + +def test_compare_takes_its_policy_from_the_file(repo, tmp, capsys): + base = make_build(tmp / "b0", repo["wt-base"], cases()) + head = make_build(tmp / "b1", repo["wt-head"], cases(quiet=1.02e-3)) + write_config(tmp, adapter=CONFIG["adapter"], compare={"profile": "revisions", "rounds": 3, "k": 4.0}) + assert cli.main(["compare", f"{base}:{repo['wt-base']}", f"{head}:{repo['wt-head']}", "--run-key", "cmp", + "--json"]) == cli.EXIT_OK + doc = json.loads(capsys.readouterr().out) + assert doc["method"]["configuration"] == {"k": 4.0, "min_rounds": 3} + assert {d["procedure"]["round"] for d in Store(tmp / "store.parquet").documents("cmp")} == {0, 1, 2} diff --git a/prototype/tests/test_benchmark_source.py b/prototype/tests/test_benchmark_source.py index f07882a..a29dac3 100644 --- a/prototype/tests/test_benchmark_source.py +++ b/prototype/tests/test_benchmark_source.py @@ -116,7 +116,7 @@ def test_a_versioned_suite_is_recorded_with_its_own_revision(subject, versioned_ assert document["revision"]["key"] == git(subject, "rev-parse", "HEAD") -def test_editing_the_suite_changes_its_tree_but_not_the_series(subject, loose_suite): +def test_editing_the_suite_changes_its_tree_but_not_the_series(subject, loose_suite, tmp): bench(subject, loose_suite, "e1") before = result("e1") loose_suite.write_text(SUITE + "\n# edited\n") @@ -124,9 +124,10 @@ def test_editing_the_suite_changes_its_tree_but_not_the_series(subject, loose_su after = result("e2") assert before["provenance"]["benchmark_tree"] != after["provenance"]["benchmark_tree"] # Each run also records the load average beside the case, a condition measurement and a series of its own. - series = Store().series(quantity="wall-time") + store = Store(tmp / "store.parquet") # the project root is the test directory + series = store.series(quantity="wall-time") assert len(series) == 1 - assert len(Store().history(series[0]["fingerprint"])) == 2 # both runs are points of it + assert len(store.history(series[0]["fingerprint"])) == 2 # both runs are points of it def test_a_suite_inside_the_subject_is_recorded_as_before(subject): diff --git a/prototype/tests/test_compare_harnesses.py b/prototype/tests/test_compare_harnesses.py index e0cca09..667d493 100644 --- a/prototype/tests/test_compare_harnesses.py +++ b/prototype/tests/test_compare_harnesses.py @@ -15,13 +15,19 @@ from benchx.store import Store from conftest import SOURCE_URI, git -MEASURE = ["A", "B", "--profile", "revisions", "--suite", "s", "--rounds", "1"] +MEASURE = ["A", "B", "--profile", "revisions", "--suite", "s", "--rounds", "1", "--adapter", "pyperf"] @pytest.mark.parametrize("argv, message", [ (MEASURE + ["--adapter", "google-benchmark", "--processes", "2"], "google-benchmark takes no --processes"), (MEASURE + ["--adapter", "pyperf", "--repetitions", "3"], "pyperf takes no --repetitions"), - (MEASURE + ["--repetitions", "3"], "pyperf takes no --repetitions"), # pyperf is the default + (MEASURE + ["--repetitions", "3"], "pyperf takes no --repetitions"), + (["A", "B", "--profile", "revisions", "--suite", "s", "--rounds", "1"], "--adapter is required"), + (["A", "B", "--profile", "revisions", "--rounds", "1", "--adapter", "pyperf"], "--suite is required"), + (["A", "B", "--profile", "revisions", "--suite", "s", "--adapter", "pyperf"], "--rounds is required"), + (["A", "B", "--suite", "s", "--rounds", "1", "--adapter", "pyperf"], "--profile is required"), + (MEASURE + ["--project", "p", "--no-project"], "--project or --no-project"), + (MEASURE + ["--quantity", "wall-time,wall-time"], "twice"), (MEASURE + ["-e", "NOEQUALS"], "-e wants NAME=VALUE"), (["--run", "k", "--profile", "revisions", "--baseline", "x", "--adapter", "pyperf"], "belongs to measure mode"), (["--run", "k", "--profile", "revisions", "--baseline", "x", "-e", "A=1"], "belongs to measure mode"), @@ -42,9 +48,9 @@ def test_a_pyperf_side_order(tmp): side = {"kind": "working_tree", "path": tmp, "source": str(tmp), "identity": {}, "uri": None, "text": {"activation": "source .venv/bin/activate"}, "labels": {}} pyperf = harnesses.PYPERF - precision, _ = pyperf.precision(pyperf.flags_to_precision(processes=2, values=3, loops=10), start=tmp) + precision = pyperf.precision(pyperf.flags_to_precision(processes=2, values=3, loops=10)) order = session._order(side, harness=pyperf, suite="bench/mod.py", filter=harnesses.exact_filter(["f"]), - quantity="wall-time", precision=precision, + quantities=["wall-time"], precision=precision, environment_variables={"OPENBLAS_NUM_THREADS": "1"}, project=None, run_key="rk", source_uri=SOURCE_URI, round_=0, slot=0, labels=None) assert core.check(order, "work-order") == [] @@ -86,7 +92,7 @@ def worktrees(tmp_path): def test_pyperf_revisions_end_to_end(tmp, worktrees, monkeypatch): pytest.importorskip("pyperf") - def provider(spec, operation): # what a minimal provider for two worktrees would answer + def provider(spec, operation, provider=None): # what a minimal provider for two worktrees would answer tree = worktrees[spec.removeprefix("@")] return {"schema_version": "benchx/target-description/0.1.0", "target": {"kind": "working_tree", "path": str(tree), "source": {"uri": SOURCE_URI, "type": "git"}}, @@ -110,8 +116,8 @@ def test_pyperf_bare_paths_are_working_trees(tmp, worktrees): pytest.importorskip("pyperf") doc = session.compare_targets( str(worktrees["base"]), str(worktrees["head"]), profile="revisions", suite="mod.py", rounds=2, - store=Store(tmp / "s.parquet"), flags={"processes": 2, "values": 2, "loops": 50, "warmups": 0}, - filter=harnesses.exact_filter(["f"]), out=tmp / "out", run_key="bare") # pyperf is the default + store=Store(tmp / "s.parquet"), adapter="pyperf", flags={"processes": 2, "values": 2, "loops": 50, "warmups": 0}, + filter=harnesses.exact_filter(["f"]), out=tmp / "out", run_key="bare") assert [u["workload"]["name"] for u in doc["units"]] == ["f"] order = json.loads(Path(tmp / "out" / "orders" / "order-0001.json").read_text()) assert (order["suites"][0]["adapter"], order["target"]["kind"]) == ("pyperf", "working_tree") diff --git a/prototype/tests/test_config.py b/prototype/tests/test_config.py index daa6377..c12e38d 100644 --- a/prototype/tests/test_config.py +++ b/prototype/tests/test_config.py @@ -1,86 +1,181 @@ -"""The precision config file (PYPERF_PLAN.md step 4).""" +"""benchx.conf.json: where it is found, what it holds, and how a value is picked (benchx.config).""" import json import pytest -from benchx import config +from benchx import cli, config, harnesses def levels(precision): return {lv["unit"]: lv["n"] for lv in precision["repetitions"]["levels"]} -def write_config(directory, precision): - path = directory / ".benchx" / "config.json" - path.parent.mkdir(exist_ok=True) - path.write_text(json.dumps({"adapters": {"pyperf": {"precision": precision}}})) +def write_config(directory, **content): + path = directory / config.CONFIG_FILE + path.write_text(json.dumps({"schema_version": "benchx/config/0.1.0", **content})) return path @pytest.fixture -def tree(tmp, monkeypatch): +def tree(tmp): path = tmp / "tree" path.mkdir() - (path / "mod.py").write_text("def bench_a():\n pass\n") - monkeypatch.chdir(path) return path -def test_defaults_are_pyperfs_when_there_is_no_file(tree): - precision, path = config.resolve("pyperf", start=tree) - assert path is None - assert levels(precision) == {"process": 20, "value": 3} - assert precision["calibration"] == {"mode": "adaptive", "minimum_sample_seconds": 0.1} +# --- where it is found ------------------------------------------------------------- - -def test_file_overrides_defaults_and_flags_override_the_file(tree): - write_config(tree, {"repetitions": {"mode": "fixed", "levels": [{"unit": "process", "n": 7}]}, - "calibration": {"mode": "fixed", "n_iterations": 30}}) - precision, path = config.resolve("pyperf", start=tree) - assert path == tree / ".benchx" / "config.json" - assert levels(precision) == {"process": 7, "value": 3} # the file's level, pyperf's for the other - assert precision["calibration"] == {"mode": "fixed", "n_iterations": 30} - - flags = {"repetitions": {"mode": "fixed", "levels": [{"unit": "process", "n": 4}]}, - "calibration": {"mode": "adaptive", "minimum_sample_seconds": 0.5}} - precision, _ = config.resolve("pyperf", flags, start=tree) - assert levels(precision) == {"process": 4, "value": 3} - assert precision["calibration"] == {"mode": "adaptive", "minimum_sample_seconds": 0.5} +def test_without_a_file_the_working_directory_is_the_root(tree): + cfg = config.load(tree) + assert (cfg.path, cfg.root, cfg.data) == (None, tree.resolve(), {}) + assert cfg.store_path == tree.resolve() / "store.parquet" + assert cfg.results_dir == tree.resolve() / "results" + assert cfg.provider is None -def test_config_found_by_walking_up(tree): - write_config(tree, {"warmup": {"mode": "none"}}) +def test_the_file_is_found_walking_up_and_its_directory_is_the_root(tree): + path = write_config(tree, project="demo") deep = tree / "a" / "b" deep.mkdir(parents=True) - precision, _ = config.resolve("pyperf", start=deep) - assert precision["warmup"] == {"mode": "none"} + cfg = config.load(deep) + assert (cfg.path, cfg.root) == (path, tree.resolve()) + assert cfg.store_path == tree.resolve() / "store.parquet" # the root's, not the working directory's + + +def test_paths_are_relative_to_the_root_unless_absolute(tree, tmp): + write_config(tree, paths={"store": "data/s.parquet", "results": str(tmp / "elsewhere")}) + (tree / "sub").mkdir() + cfg = config.load(tree / "sub") + assert cfg.store_path == tree.resolve() / "data" / "s.parquet" + assert cfg.results_dir == tmp / "elsewhere" + + +def test_benchx_home_relocates_the_store_over_the_file(tree, tmp, monkeypatch): + write_config(tree, paths={"store": "data/s.parquet"}) + monkeypatch.setenv("BENCHX_HOME", str(tmp / "home")) + assert config.load(tree).store_path == tmp / "home" / "store.parquet" + monkeypatch.setenv("BENCHX_HOME", "") + assert config.load(tree).store_path == tree.resolve() / "data" / "s.parquet" + +def test_the_provider_runs_in_the_root(tree): + write_config(tree, provider={"command": ["./provider.sh"]}) + assert config.load(tree).provider == (["./provider.sh"], tree.resolve()) + + +# --- what is refused --------------------------------------------------------------- @pytest.mark.parametrize("content, message", [ - ({"adapters": {"pyperf": {"precision": {"calibration": {"mode": "fixed"}}}}}, "n_iterations"), - ({"adapters": {"pyperf": {"precision": {"warmups": 3}}}}, "warmups"), - ({"adapters": {"nonesuch": {}}}, "no adapter named"), - ({"adapters": {"pyperf": {"speed": "fast"}}}, "precision"), - ({"speed": "fast"}, "top level"), + ({"adapter": {"name": "nonesuch"}}, "no adapter named"), + ({"adapter": {"name": "pyperf", "quantities": ["cpu-time"]}}, "does not report cpu-time"), + ({"adapter": {"name": "pyperf", "precision": {"calibration": {"mode": "fixed"}}}}, "n_iterations"), + ({"adapter": {"name": "pyperf", "precision": {"warmups": 3}}}, "warmups"), + ({"adapter": {"suite": "s"}}, "name"), + ({"provider": {"command": []}}, "command"), + ({"speed": "fast"}, "speed"), ]) -def test_bad_config_is_rejected(tree, content, message): - (tree / ".benchx").mkdir() - (tree / ".benchx" / "config.json").write_text(json.dumps(content)) +def test_a_bad_file_is_refused(tree, content, message): + write_config(tree, **content) with pytest.raises(config.ConfigError, match=message): - config.resolve("pyperf", start=tree) + config.load(tree) -def test_malformed_config_is_rejected(tree): - (tree / ".benchx").mkdir() - (tree / ".benchx" / "config.json").write_text("{not json") +def test_a_malformed_file_is_refused(tree): + (tree / config.CONFIG_FILE).write_text("{not json") with pytest.raises(config.ConfigError): - config.resolve("pyperf", start=tree) + config.load(tree) + + +@pytest.mark.parametrize("legacy", [".benchx/provider.json", ".benchx/config.json"]) +def test_the_old_files_are_not_read_and_not_ignored(tree, legacy): + (tree / ".benchx").mkdir() + (tree / legacy).write_text("{}") + with pytest.raises(config.ConfigError, match="old configuration format"): + config.load(tree) + write_config(tree) # once the new file exists, the old one is just a file + assert config.load(tree).path is not None + + +# --- picking a value --------------------------------------------------------------- + +def test_the_command_line_wins_then_the_file_then_the_default(tree): + cfg = config.load(tree) + assert cfg.pick("flag", "file", "default", flag="--x", key="x") == "flag" + assert cfg.pick(None, "file", "default", flag="--x", key="x") == "file" + assert cfg.pick(None, None, "default", flag="--x", key="x") == "default" + assert cfg.pick(None, None, None, flag="--x", key="x") is None # None is a default too + assert cfg.pick(0, 5, 9, flag="--x", key="x") == 0 # a falsy flag still counts as given + + +def test_a_missing_required_value_names_the_flag_and_the_key(tree): + with pytest.raises(config.ConfigError, match=r"--suite is required: give it, or set adapter\.suite in .*none found"): + config.load(tree).pick(None, None, flag="--suite", key="adapter.suite") + write_config(tree) + with pytest.raises(config.ConfigError, match=r"--suite is required: give it, or set adapter\.suite in .*benchx\.conf\.json"): + config.load(tree).pick(None, None, flag="--suite", key="adapter.suite") + + +def test_a_provider_is_needed_only_when_a_target_asks_for_it(tree): + cfg = config.load(tree) + with pytest.raises(config.ConfigError, match=r'@plain needs "provider"'): + cfg.require_provider("@plain") + + +@pytest.mark.parametrize("text, names", [("wall-time", ["wall-time"]), ("wall-time,cpu-time", ["wall-time", "cpu-time"]), + (" wall-time , cpu-time ", ["wall-time", "cpu-time"])]) +def test_quantities_are_a_comma_separated_list(text, names): + assert config.parse_quantities(text) == names + + +@pytest.mark.parametrize("text", ["", "wall-time,", ",cpu-time", "wall-time,,cpu-time", "wall-time,wall-time"]) +def test_empty_and_repeated_quantities_are_refused(text): + with pytest.raises(config.ConfigError, match="--quantity"): + config.parse_quantities(text) + + +# --- precision --------------------------------------------------------------------- + +def test_the_file_overrides_the_defaults_and_flags_override_the_file(tree): + file_precision = {"repetitions": {"mode": "fixed", "levels": [{"unit": "process", "n": 7}]}, + "calibration": {"mode": "fixed", "n_iterations": 30}} + pyperf = harnesses.PYPERF + assert levels(pyperf.precision({})) == {"process": 20, "value": 3} + precision = pyperf.precision({}, file_precision) + assert levels(precision) == {"process": 7, "value": 3} # the file's level, pyperf's for the other + assert precision["calibration"] == {"mode": "fixed", "n_iterations": 30} + + flags = pyperf.flags_to_precision(processes=4, min_time=0.5) + precision = pyperf.precision(flags, file_precision) + assert levels(precision) == {"process": 4, "value": 3} + assert precision["calibration"] == {"mode": "adaptive", "minimum_sample_seconds": 0.5} + + +def test_google_benchmark_precision_comes_from_the_same_layers(): + gb = harnesses.GOOGLE_BENCHMARK + assert gb.precision({})["repetitions"] == 5 + file_precision = {"repetitions": 10, "warmup": {"mode": "time", "seconds": 0.1}} + precision = gb.precision(gb.flags_to_precision(repetitions=3), file_precision) + assert (precision["repetitions"], precision["warmup"]) == (3, {"mode": "time", "seconds": 0.1}) + + +# --- bx config ----------------------------------------------------------------------- + +@pytest.mark.parametrize("adapter", sorted(harnesses.HARNESSES)) +def test_init_writes_a_valid_file_that_states_the_defaults(tree, adapter, capsys): + assert cli.main(["config", "init", "--dir", str(tree), "--adapter", adapter]) == cli.EXIT_OK + cfg = config.load(tree) + assert cfg.adapter["name"] == adapter + assert cfg.adapter["precision"] == harnesses.get(adapter).precision({}) + assert cli.main(["config", "init", "--dir", str(tree)]) == cli.EXIT_REFUSED + assert "already exists" in capsys.readouterr().err -def test_template_states_pyperfs_defaults_and_is_valid(tree): - path = config.write_template(tree) - precision, found = config.resolve("pyperf", start=tree) - assert found == path and levels(precision) == {"process": 20, "value": 3} - with pytest.raises(config.ConfigError, match="already exists"): - config.write_template(tree) +def test_show_says_where_each_value_comes_from(tree, monkeypatch, capsys): + write_config(tree, project="demo", adapter={"name": "pyperf", "suite": "mod.py"}) + monkeypatch.chdir(tree) + assert cli.main(["config", "show"]) == cli.EXIT_OK + out = capsys.readouterr().out + assert '"demo" (file)' in out and '"mod.py" (file)' in out + assert "compare.rounds - (unset)" in out + assert "timeouts.case_s 60 (default)" in out diff --git a/prototype/tests/test_config_schema.py b/prototype/tests/test_config_schema.py new file mode 100644 index 0000000..0169bc4 --- /dev/null +++ b/prototype/tests/test_config_schema.py @@ -0,0 +1,102 @@ +"""The config schema, its recommended examples, and the paths benchx keeps out of a tree's identity.""" + +import json +from pathlib import Path + +import pytest + +from benchx import adapters, core, snapshot +from benchx.core import catalog +from conftest import git + +EXAMPLES = sorted((catalog.SCHEMAS / "config" / "0.1.0" / "examples").glob("*.json")) + + +@pytest.mark.parametrize("path", EXAMPLES, ids=lambda p: p.stem) +def test_examples_fit_the_schema(path): + assert core.check(json.loads(path.read_text()), "config") == [] + + +@pytest.mark.parametrize("path", EXAMPLES, ids=lambda p: p.stem) +def test_recommended_settings_are_ones_the_adapter_applies(path): + adapter_block = json.loads(path.read_text())["adapter"] + adapter = adapters.get(adapter_block["name"]) + assert core.check_precision(adapter_block["precision"]) == [] + adapter.protocol(adapter_block["precision"]) # raises Unsupported for a shape it cannot apply + assert set(adapter_block["quantities"]) <= set(adapter.QUANTITIES) + + +@pytest.mark.parametrize("config, message", [ + ({"adapter": {"suite": "s"}}, "name"), + ({"adapter": {"name": "pyperf", "quantities": []}}, "quantities"), + ({"adapter": {"name": "pyperf", "quantities": ["wall-time", "wall-time"]}}, "quantities"), + ({"provider": {"command": []}}, "command"), + ({"provider": {"command": "spin"}}, "command"), + ({"compare": {"profile": "history"}}, "profile"), + ({"compare": {"rounds": 0}}, "rounds"), + ({"paths": {"store": ""}}, "store"), + ({"project": ""}, "project"), + ({"adapters": {}}, "adapters"), +]) +def test_schema_refuses(config, message): + errors = core.check({"schema_version": "benchx/config/0.1.0", **config}, "config") + assert errors and message in " ".join(str(e) for e in errors) + + +# --- paths benchx writes inside a measured tree ----------------------------------- + +@pytest.fixture(autouse=True) +def _no_ignored_paths(): + snapshot.ignore() + yield + snapshot.ignore() + + +@pytest.fixture +def checkout(tmp_path): + root = tmp_path / "wt" + root.mkdir() + git(root, "init", "-q") + (root / "bench.py").write_text("x = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "m") + return root + + +def write_outputs(root): + (root / "results").mkdir(exist_ok=True) + (root / "results" / "r.json").write_text("{}") + (root / "store.parquet").write_bytes(b"PAR1") + + +def test_a_runs_own_output_marks_the_tree_dirty_unless_ignored(checkout): + clean = snapshot.git_identity(checkout) + assert clean["dirty"] == "clean" + write_outputs(checkout) + assert snapshot.git_identity(checkout)["dirty"] == "dirty" + snapshot.ignore(checkout / "results", checkout / "store.parquet") + again = snapshot.git_identity(checkout) + assert (again["dirty"], again["tree"]) == ("clean", clean["tree"]) + + +def test_ignoring_does_not_hide_a_real_change(checkout): + snapshot.ignore(checkout / "results") + write_outputs(checkout) + (checkout / "bench.py").write_text("x = 2\n") + assert snapshot.git_identity(checkout)["dirty"] == "dirty" + + +def test_a_path_outside_the_tree_changes_nothing(checkout, tmp_path): + snapshot.ignore(tmp_path / "elsewhere" / "results") + assert snapshot.git_identity(checkout)["dirty"] == "clean" + + +def test_a_plain_directory_tree_id_skips_ignored_paths(tmp_path): + root = tmp_path / "plain" + root.mkdir() + (root / "bench.py").write_text("x = 1\n") + before = snapshot.directory_identity(root)["tree"] + write_outputs(root) + assert snapshot.directory_identity(root)["tree"] != before + snapshot.ignore(root / "results", root / "store.parquet") + assert snapshot.directory_identity(root)["tree"] == before diff --git a/prototype/tests/test_harnesses.py b/prototype/tests/test_harnesses.py index 116dffb..c1e6fd5 100644 --- a/prototype/tests/test_harnesses.py +++ b/prototype/tests/test_harnesses.py @@ -28,10 +28,11 @@ def test_suite_entry_leaves_out_an_empty_filter(harness): def test_google_benchmark_precision(): gb = harnesses.GOOGLE_BENCHMARK - assert gb.precision({})[0] == {"repetitions": 5} - flags = gb.flags_to_precision(repetitions=3, min_time=0.5) - assert gb.precision(flags) == ({"repetitions": 3, "calibration": { - "mode": "adaptive", "minimum_sample_seconds": 0.5}}, None) + assert gb.precision({}) == adapters.get("google-benchmark").default_precision() + flags = gb.flags_to_precision(repetitions=3, min_time=0.25) + precision = gb.precision(flags) + assert (precision["repetitions"], precision["calibration"]) == ( + 3, {"mode": "adaptive", "minimum_sample_seconds": 0.25}) @pytest.mark.parametrize("harness, flag", [(harnesses.GOOGLE_BENCHMARK, "processes"), @@ -50,7 +51,5 @@ def test_pyperf_calibration_is_one_or_the_other(): harnesses.PYPERF.flags_to_precision(min_time=0.1, loops=5) -def test_pyperf_precision_falls_back_to_its_defaults(tmp_path): - precision, path = harnesses.PYPERF.precision({}, start=tmp_path) - assert path is None - assert precision == adapters.get("pyperf").default_precision() +def test_pyperf_precision_falls_back_to_its_defaults(): + assert harnesses.PYPERF.precision({}) == adapters.get("pyperf").default_precision() diff --git a/prototype/tests/test_orders.py b/prototype/tests/test_orders.py index 28475af..ef7a6cd 100644 --- a/prototype/tests/test_orders.py +++ b/prototype/tests/test_orders.py @@ -4,7 +4,7 @@ import pytest -from benchx import core, orders, session +from benchx import core, harnesses, orders, session SUITE = {"adapter": "google-benchmark", "suite": "demo-bench"} TARGET = {"kind": "build", "path": "/b", "source_dir": "/s", @@ -26,11 +26,12 @@ def test_round_and_slot_zero_are_kept(): assert (order["round"], order["slot"]) == (0, 0) -def test_quantity_sets_quantities_and_plan(): +def test_quantities_set_quantities_and_plan(): order = orders.make_order(suite=SUITE, target=TARGET, precision={"repetitions": 1}, run_key="rk", - quantity="cpu-time") - assert order["quantities"] == ["cpu-time"] - assert order["plan"] == [{"id": "p0", "case": "*", "quantity": "cpu-time"}] + quantities=["wall-time", "cpu-time"]) + assert order["quantities"] == ["wall-time", "cpu-time"] + assert order["plan"] == [{"id": "p0", "case": "*", "quantity": "wall-time"}, + {"id": "p1", "case": "*", "quantity": "cpu-time"}] def test_each_order_gets_its_own_id(): @@ -49,7 +50,7 @@ def test_default_run_key(): def test_compare_orders_are_now_validated(): side = {"kind": "build", "path": Path("/b"), "source": "/s", "identity": {}, "uri": None, "text": {}, "labels": {}} - kwargs = dict(harness=session.HARNESS, suite="demo-bench", filter=None, quantity="wall-time", + kwargs = dict(harness=harnesses.GOOGLE_BENCHMARK, suite="demo-bench", filter=None, quantities=["wall-time"], environment_variables=None, project=None, run_key="rk", source_uri="https://example.org/demo.git", round_=0, slot=0, labels=None) assert core.check(session._order(side, precision={"repetitions": 5}, **kwargs), "work-order") == [] diff --git a/prototype/tests/test_prototype.py b/prototype/tests/test_prototype.py index 8557c05..db7324e 100644 --- a/prototype/tests/test_prototype.py +++ b/prototype/tests/test_prototype.py @@ -454,7 +454,7 @@ def test_a_file_that_cannot_be_read_is_one_rejection_not_a_crash(adhoc, tmp, cap (sweep / "garbage.json").write_bytes(b"\xff\xfe\x00not text") (sweep / "workorder-garbage.json").write_bytes(b"\xff") missing = tmp / "missing.json" - store = Store(tmp / "store.parquet") + store = Store(tmp / "direct.parquet") # not the project's store.parquet, which the CLI below ingests into outcome = store.ingest_paths([sweep, missing]) assert (outcome["ingested"], outcome["duplicate"]) == (1, 0) rejected = {r["document"]: r for r in outcome["rejected"]} @@ -507,19 +507,18 @@ def test_head(revisions_run, tmp): def test_bx_run_delivers_to_the_local_store(repo, tmp, capsys): """bx run writes result files and ingests exactly those into the one local store.""" from benchx import cli - from benchx.store import default_path build = make_build(tmp / "b", repo["wt-head"], cases()) for slot, name in enumerate(("o1", "o2")): (tmp / f"{name}.json").write_text(json.dumps(order(build, repo["wt-head"], "cli", slot=slot))) - assert default_path() == tmp / "benchx-home" / "store.parquet" + store_path = tmp / "store.parquet" # no benchx.conf.json: the working directory is the root assert cli.main(["run", str(tmp / "o1.json"), "--out", str(tmp / "out")]) == cli.EXIT_OK assert "ingested 4, duplicate 0" in capsys.readouterr().out # this run's files only, with load averages assert cli.main(["run", str(tmp / "o2.json"), "--out", str(tmp / "out"), "--no-ingest"]) == cli.EXIT_OK assert len(results(tmp / "out")) == 4 # both runs' files are written - total, rows = Store().head() + total, rows = Store(store_path).head() assert total == 4 and {load(tmp / "o1.json")["run_key"]} == {r["run_key"] for r in rows} ref = json.loads(rows[0]["document"])["provenance"]["info"]["workorder_ref"] - assert Store().order(ref) == load(tmp / "o1.json") # the order travelled with its results + assert Store(store_path).order(ref) == load(tmp / "o1.json") # the order travelled with its results def _working_tree_order(path, run_key, **target): diff --git a/prototype/tests/test_session.py b/prototype/tests/test_session.py index 395cc12..761dea3 100644 --- a/prototype/tests/test_session.py +++ b/prototype/tests/test_session.py @@ -100,8 +100,8 @@ def test_measure_mode_equals_read_mode(repo, tmp, builds, capsys): (["--profile", "revisions"], "give two targets"), (["A", "B", "--run", "k", "--profile", "revisions"], "not both"), (["A", "--profile", "revisions"], "exactly two targets"), - (["A", "B", "--profile", "revisions", "--rounds", "3"], "needs --suite"), - (["A", "B", "--profile", "revisions", "--suite", "s"], "needs --rounds"), + (["A", "B", "--profile", "revisions", "--rounds", "3", "--adapter", "pyperf"], "--suite is required"), + (["A", "B", "--profile", "revisions", "--suite", "s", "--adapter", "pyperf"], "--rounds is required"), (["A", "B", "--profile", "revisions", "--suite", "s", "--rounds", "3", "--baseline", "x"], "read mode"), (["--run", "k", "--profile", "revisions"], "needs --baseline"), (["--run", "k", "--profile", "revisions", "--baseline", "x", "--rounds", "3"], "measure mode"), diff --git a/prototype/tests/test_sides.py b/prototype/tests/test_sides.py index 0d5f23d..11aecc4 100644 --- a/prototype/tests/test_sides.py +++ b/prototype/tests/test_sides.py @@ -15,7 +15,7 @@ def description(path, kind="working_tree", **target): def provided(monkeypatch, desc): """Stand in for the provider: every @spec resolves to `desc`.""" - monkeypatch.setattr(session.target_mod, "resolve", lambda spec, operation: desc) + monkeypatch.setattr(session.target_mod, "resolve", lambda spec, operation, provider=None: desc) def executable(path): @@ -68,7 +68,7 @@ def test_a_provided_working_tree_is_its_own_source(tree, monkeypatch): def test_a_working_tree_order_is_valid(tree, monkeypatch): provided(monkeypatch, description(tree)) side = session._side("@tree", SUITE, "prepare", harness=harnesses.GOOGLE_BENCHMARK) - order = session._order(side, harness=harnesses.GOOGLE_BENCHMARK, suite=SUITE, filter=None, quantity="wall-time", + order = session._order(side, harness=harnesses.GOOGLE_BENCHMARK, suite=SUITE, filter=None, quantities=["wall-time"], precision={"repetitions": 3}, environment_variables=None, project=None, run_key="rk", source_uri=SOURCE_URI, round_=0, slot=0, labels=None) assert order["target"]["kind"] == "working_tree" diff --git a/prototype/tests/test_target.py b/prototype/tests/test_target.py index a04c624..78ba474 100644 --- a/prototype/tests/test_target.py +++ b/prototype/tests/test_target.py @@ -7,7 +7,7 @@ import pytest -from benchx import compare, core, runner, session, snapshot +from benchx import compare, config, core, runner, session, snapshot from benchx import target as target_mod from benchx.store import Store from conftest import SOURCE_URI, cases, make_build, order @@ -82,34 +82,40 @@ def provider(repo, tmp, monkeypatch): script.parent.mkdir() script.write_text(PROVIDER.format(python=sys.executable, descriptions=str(descriptions))) script.chmod(script.stat().st_mode | stat.S_IXUSR) - (tmp / "proj" / ".benchx").mkdir() - (tmp / "proj" / ".benchx" / "provider.json").write_text(json.dumps({"command": [str(script)]})) + (tmp / "proj" / "benchx.conf.json").write_text(json.dumps({"schema_version": "benchx/config/0.1.0", + "provider": {"command": [str(script)]}})) monkeypatch.chdir(tmp / "proj") return {"plain": plain, "hardened": hardened} +def provider_of(start): + return config.load(start).provider + + def test_provider_is_found_walking_up_and_answers_a_request(provider, tmp): (tmp / "proj" / "deeper").mkdir() - command, directory = target_mod.find_provider(tmp / "proj" / "deeper") + found = provider_of(tmp / "proj" / "deeper") + command, directory = found assert directory == (tmp / "proj").resolve() and len(command) == 1 - doc = target_mod.resolve("@hardened", operation="describe", cwd=tmp / "proj" / "deeper") + doc = target_mod.resolve("@hardened", operation="describe", provider=found) assert doc["provider"] == {"name": "fake-provider", "version": "describe"} - assert target_mod.resolve("@hardened", operation="prepare")["provider"]["version"] == "prepare" + assert target_mod.resolve("@hardened", operation="prepare", provider=found)["provider"]["version"] == "prepare" -def test_provider_failure_and_missing_provider_are_errors(provider, tmp, monkeypatch): +def test_provider_failure_and_missing_provider_are_errors(provider, tmp): with pytest.raises(target_mod.TargetError, match="failed to prepare @broken .*exit 7"): - target_mod.resolve("@broken") - monkeypatch.chdir(tmp) + target_mod.resolve("@broken", provider=provider_of(tmp / "proj")) with pytest.raises(target_mod.TargetError, match="needs a provider"): target_mod.resolve("@plain") + with pytest.raises(config.ConfigError, match=r'@plain needs "provider"'): + config.load(tmp).require_provider("@plain") def test_declared_text_reaches_orders_results_and_the_comparison(repo, tmp, provider): store = Store(tmp / "store.parquet") doc = session.compare_targets("@plain", "@hardened", profile="environments", adapter="google-benchmark", suite="demo-bench", rounds=4, label=("build", "plain", "hardened"), project="demo", store=store, - run_key="t1", out=tmp / "out") + run_key="t1", out=tmp / "out", provider=provider_of(".")) orders = [json.loads(p.read_text()) for p in sorted((tmp / "out" / "orders").glob("*.json"))] plain_target, hardened_target = orders[0]["target"], orders[1]["target"] assert plain_target["how_built"] == "cmake plain" @@ -133,7 +139,7 @@ def test_declared_text_reaches_orders_results_and_the_comparison(repo, tmp, prov def test_no_build_asks_the_provider_to_describe(repo, tmp, provider): session.compare_targets("@plain", "@hardened", profile="environments", adapter="google-benchmark", suite="demo-bench", rounds=3, label=("build", "plain", "hardened"), store=Store(tmp / "store.parquet"), - run_key="t2", out=tmp / "out", build=False) + run_key="t2", out=tmp / "out", build=False, provider=provider_of(".")) result = next(p for p in (tmp / "out").glob("*.json") if not p.name.startswith("workorder-")) assert json.loads(result.read_text())["provenance"]["info"]["target_provider"]["version"] == "describe" @@ -191,7 +197,7 @@ class _Stop(Exception): def _compare(tmp): - return session.compare_targets("a", "b", profile="revisions", suite="s", rounds=1, store=Store(tmp / "s.parquet"), + return session.compare_targets("a", "b", profile="revisions", suite="s", rounds=1, adapter="pyperf", store=Store(tmp / "s.parquet"), run_key="u", out=tmp / "out") diff --git a/schemas/config/0.1.0/examples/google-benchmark-recommended.json b/schemas/config/0.1.0/examples/google-benchmark-recommended.json new file mode 100644 index 0000000..95a1f8b --- /dev/null +++ b/schemas/config/0.1.0/examples/google-benchmark-recommended.json @@ -0,0 +1,14 @@ +{ + "schema_version": "benchx/config/0.1.0", + "adapter": { + "name": "google-benchmark", + "suite": "bench/example-bench", + "quantities": ["wall-time", "cpu-time"], + "precision": { + "repetitions": 10, + "calibration": { "mode": "adaptive", "minimum_sample_seconds": 0.5 }, + "warmup": { "mode": "time", "seconds": 0.1 } + } + }, + "timeouts": { "case_s": 120, "order_s": 1800 } +} diff --git a/schemas/config/0.1.0/examples/pyperf-recommended.json b/schemas/config/0.1.0/examples/pyperf-recommended.json new file mode 100644 index 0000000..40f97c2 --- /dev/null +++ b/schemas/config/0.1.0/examples/pyperf-recommended.json @@ -0,0 +1,19 @@ +{ + "schema_version": "benchx/config/0.1.0", + "adapter": { + "name": "pyperf", + "suite": "benchmarks/bench_example.py", + "quantities": ["wall-time"], + "precision": { + "repetitions": { "mode": "fixed", "levels": [{ "unit": "process", "n": 20 }, { "unit": "value", "n": 3 }] }, + "calibration": { "mode": "adaptive", "minimum_sample_seconds": 0.1 }, + "warmup": { "mode": "count", "n_warmup": 1 } + } + }, + "environment_variables": { + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1" + }, + "timeouts": { "case_s": 120, "order_s": 1800 } +} diff --git a/schemas/config/0.1.0/schema.json b/schemas/config/0.1.0/schema.json new file mode 100644 index 0000000..5dc94a5 --- /dev/null +++ b/schemas/config/0.1.0/schema.json @@ -0,0 +1,87 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:benchx:schema:config:0.1.0", + "title": "BenchX project configuration", + "description": "benchx.conf.json, in the project root: how this project is measured. Every key is optional here; the workbench takes a value from the command line first, then from this file, then from a built-in default, and refuses a command that is left without a value it requires (adapter.name, adapter.suite, a target). Relative paths are relative to the directory holding the file.", + "$comment": "adapter.precision is checked against the work-order precision schema by the loader, after it is merged over the harness defaults, because it may be partial here (a repetition level, or calibration or warmup, may be left to the harness). adapter.name must name a harness; the loader checks that, so a new harness needs no schema change.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version"], + "properties": { + "schema_version": { "const": "benchx/config/0.1.0" }, + "project": { + "description": "Names the project, so results join its series. Without it a run is ad hoc, and the store files it as local/.", + "type": "string", + "minLength": 1 + }, + "source": { + "type": "object", + "additionalProperties": false, + "properties": { + "uri": { "description": "The source's URI, when the checkouts' origin remote does not give it.", "type": "string", "minLength": 1 } + } + }, + "provider": { + "description": "The target provider, called for an @CONFIG target. It runs with the project root as its working directory.", + "type": "object", + "additionalProperties": false, + "required": ["command"], + "properties": { + "command": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } } + } + }, + "adapter": { + "description": "The harness, and what is specific to it.", + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { "description": "pyperf or google-benchmark.", "type": "string", "minLength": 1 }, + "suite": { + "description": "The benchmark program in the target: pyperf, the module's path in the working tree; google-benchmark, the binary in the build directory.", + "type": "string", + "minLength": 1 + }, + "quantities": { + "description": "What one run collects, from those the harness reports.", + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 } + }, + "precision": { "description": "A work-order precision, possibly partial.", "type": "object" } + } + }, + "environment_variables": { + "description": "Set for the benchmark's processes; each becomes a workload parameter.", + "type": "object", + "additionalProperties": { "type": "string" } + }, + "timeouts": { + "type": "object", + "additionalProperties": false, + "properties": { + "case_s": { "type": "number", "exclusiveMinimum": 0 }, + "order_s": { "type": "number", "exclusiveMinimum": 0 } + } + }, + "paths": { + "type": "object", + "additionalProperties": false, + "properties": { + "store": { "description": "The local store (default store.parquet).", "type": "string", "minLength": 1 }, + "results": { "description": "Where result files are written (default results).", "type": "string", "minLength": 1 } + } + }, + "compare": { + "type": "object", + "additionalProperties": false, + "properties": { + "profile": { "enum": ["revisions", "environments"] }, + "rounds": { "type": "integer", "minimum": 1 }, + "k": { "type": "number", "exclusiveMinimum": 0 }, + "min_rounds": { "type": "integer", "minimum": 1 } + } + } + } +}