ENH: Implement UC-04, cross-env comparisons

.github/workflows/tests.yml

@@ -61,3 +61,12 @@ jobs:
         if: runner.os != 'Windows'
         shell: bash
         run: uv run --extra test --python ${{ matrix.python-version }} bash examples/archery-local/run.sh
+
+      # Makes two venvs of its own with uv (Python 3.12, the newest with numpy 1.26.4 wheels), so
+      # the matrix Python only runs the outer script and one cell is enough. Needs network access.
+      - name: Run the numpy-venvs example
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
+        shell: bash
+        run: >-
+          uv run --extra test --python ${{ matrix.python-version }}
+          bash examples/numpy-venvs/run.sh

docs/design/prototype-design.md

@@ -39,6 +39,7 @@ bx bench TARGET --suite NAME [CASE …]    # the single-target form of bx compar
          [--adapter …] [--filter REGEX] [--quantity NAMES] [-e NAME=VALUE ...]
          [precision flags, as for bx compare]
          [--project P | --no-project] [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI]
+         [--label NAME=VALUE ... --round N --slot K]   # one side of a session other calls join
          [--case-timeout S] [--no-ingest]
 bx bench MODULE.py [FUNCTION …] [--tree DIR] [...]
                                          # shorthand: the module's checkout, pyperf → result files → store → table
@@ -51,13 +52,32 @@ target the same way, takes the same options, and goes through the same code:
 the side is resolved by `session._side` (a provider call for `@CONFIG`, else a
 bare path the harness reads; the adapter checks the suite before anything runs)
 and the order is built by `session._order`. A bench run is one order of a
-comparison session with no round, no slot, and no comparison; it is run by the
+comparison session with no comparison, and by default no round or slot; it is run by the
 same runner, delivered as `bx run` delivers (`--no-ingest`: files only), and
 summarized as a table of its cases. It is not `bx run`: that executes a finished
 order file and never resolves a target; `bx bench` turns options into an order
 (saved as `results/RUN_KEY/orders/order.json`, which `bx run` can replay) and
 then does what `bx run` does.
 
+**A call as one side of a session (UC-04).** Environments that cannot share a process, such as two
+venvs holding different numpy versions, are measured by separate `bx bench` calls that share a
+`--run-key`; the caller's script is the loop (`benchmark-environments.md` §5.7). `--label NAME=VALUE`
+names the call's side, `--round N` pairs its attempt with the other side's, and `--slot K` is its
+position in the alternation, which the caller counts. Each call writes `orders/order-KKKK.json`. The
+flags need an explicit `--run-key` (the default is a timestamp per call, so two calls would never
+share a run), and `--label` and `--round` need `--slot` (the ingest key gets its `:slot-K` suffix only
+when a slot is set, so two sides would otherwise share a key). `bx compare --run KEY --profile
+environments --label NAME --baseline VALUE` then compares the sides, and refuses a run whose sides do
+not alternate, have unequal attempt counts, or do not differ.
+
+What differs between the environments is recorded by the user's own commands, so benchx knows no
+environment kind: a context collector (§3) prints facts about the environment it runs in, which land
+under `environment.system.<name>`, a part of the varying coordinate; a setup file does the same for
+declared facts. A collector with `verify: how_built` and `on_fail: warn` checks the facts a target's
+`how_built` object declares against what it found, and the comparison lists the warning beside the
+facts that differ. `prototype/examples/numpy-venvs` is the worked example, with the map from UC-04's
+spellings to these.
+
 A bare path is read by the harness. For Google Benchmark it is
 `BUILD_DIR[:SOURCE_DIR]`, a `build` target. For pyperf it is a working tree, a
 `working_tree` target that is its own source, measured as found with its dirty

prototype/README.md

@@ -27,6 +27,7 @@ bx bench TARGET --suite NAME [CASE …]    # the single-target form of bx compar
          [--adapter …] [--filter REGEX] [--quantity …] [-e NAME=VALUE ...]
          [--processes N] [--values N] [--min-time S | --loops N] [--warmups N] [--repetitions N]
          [--project P] [--run-key KEY] [--out DIR] [--no-build] [--source-uri URI]
+         [--label NAME=VALUE ... --round N --slot K]   # one side of a session other calls join
          [--case-timeout S] [--no-ingest]
 bx bench MODULE.py [FUNCTION …] [--tree DIR] [...]
                                          # shorthand: Python functions under pyperf → results → store → table
@@ -88,6 +89,12 @@ functions are exact case names. `bx run` is the layer underneath: it executes a
 order, so `bx run results/KEY/orders/order.json` replays any bench run.
 `examples/pyperf-bench` is a runnable walk-through.
 
+Environments that cannot share a process, such as two venvs with different numpy versions, are
+measured by separate `bx bench` calls that share a `--run-key`; `--label NAME=VALUE`, `--round N`
+and `--slot K` place each call in the session, and `bx compare --run KEY --profile environments
+--label NAME --baseline VALUE` compares the sides ([UC-04](../docs/use-cases/UC-04-cross-env.md)).
+`examples/numpy-venvs` is the worked example, with a shell script that drives it.
+
 `bx bench PATH --adapter asv` runs an [asv](https://asv.readthedocs.io) benchmark suite
 unchanged, reading asv's benchmark format as a language and timing it with pyperf;
 neither asv nor asv_runner is used. PATH is a directory holding `asv.conf.json` (its
@@ -240,6 +247,7 @@ examples/demo-suite  a two-benchmark Google Benchmark suite for the demo
 examples/demo        the demo's project root: its benchx.conf.json, and its persistent store
 examples/pyperf-bench  Python functions under `bx bench` (own README)
 examples/archery-local  an end-to-end, Arrow-local-shaped example with a target provider (own README)
+examples/numpy-venvs  UC-04 with two venvs and two numpy versions, driven by run.sh (own README)
 tests/               test_prototype.py: the success criteria and the runner and store
                      rules, on a fake Google Benchmark binary; test_comparator.py: the
                      comparator's invariants and method on hand-built results;

prototype/benchx/cli.py

@@ -78,6 +78,31 @@ def _environment(items) -> dict:
     return env
 
 
+def _session_position(args) -> tuple[dict, int | None, int | None]:
+    """Where `bx bench` sits in a session that other calls join through the same
+    `--run-key`: its `--label NAME=VALUE`s, `--round` and `--slot`. UsageError when
+    they could not join one."""
+    labels = {}
+    for item in args.label or []:
+        name, eq, value = item.partition("=")
+        if not (name and eq and value):
+            raise UsageError(f"--label wants NAME=VALUE, not {item!r}")
+        if name in labels:
+            raise UsageError(f"--label {name} is given twice")
+        labels[name] = value
+    for flag, number in (("--round", args.round), ("--slot", args.slot)):
+        if number is not None and number < 0:
+            raise UsageError(f"{flag} wants a number from 0, not {number}")
+    placed = labels or args.round is not None or args.slot is not None
+    if placed and not args.run_key:
+        raise UsageError("--label, --round and --slot need an explicit --run-key: the default is a timestamp "
+                         "per call, so two calls would never share a run")
+    if (labels or args.round is not None) and args.slot is None:
+        raise UsageError("--label and --round need --slot: without one, the sides of a run would share an "
+                         "ingest key and their results would conflict")
+    return labels, args.round, args.slot
+
+
 def _bench_target(args) -> tuple[str, str | None, str | None]:
     """The target, the suite it forces (or None: --suite, then the file's), and the
     adapter it forces. With `--suite` the target is as given. Without it, the target
@@ -155,6 +180,7 @@ def cmd_bench(args):
             raise UsageError("give case names or --filter, not both")
         target, forced_suite, forced_adapter = _bench_target(args)
         settings = _common(cfg, args, forced_adapter=forced_adapter, forced_suite=forced_suite)
+        settings["labels"], settings["round_"], settings["slot"] = _session_position(args)
         _needs_provider(cfg, target)
     except (UsageError, config_mod.ConfigError, bench_mod.SessionError) as e:
         print(f"bx bench: error: {e}", file=sys.stderr)
@@ -493,6 +519,16 @@ def main(argv=None):
     p.add_argument("--case-timeout", type=float,
                    help=f"seconds before one case is stopped (default: timeouts.case_s, else {orders.CASE_TIMEOUT_S:g}; "
                         f"asv: {ASV_CASE_TIMEOUT_S}, all rounds, each also stopped at the benchmark's own timeout)")
+    p.add_argument("--label", action="append", metavar="NAME=VALUE",
+                   help="name this call's side of a session, such as env=numpy-2.0 (repeatable); the calls of a "
+                        "session share --run-key, and bx compare --run reads the label with --label NAME. "
+                        "Needs --run-key and --slot")
+    p.add_argument("--round", type=int,
+                   help="this call's round in a session: the sides' attempts are paired by it. "
+                        "Needs --run-key and --slot")
+    p.add_argument("--slot", type=int,
+                   help="this call's position in the session's alternation (0, 1, 2, ...), kept by the caller's loop. "
+                        "Needs --run-key")
     p.add_argument("--no-ingest", action="store_true", help="only write result files; skip the local store")
     p.set_defaults(fn=cmd_bench)
 

prototype/benchx/compare.py

@@ -109,7 +109,8 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou
     failed = doc["failed_invariants"]
     projected = [projection.apply(d, store_project(d)) for d in documents]
     if any(p.axis[0] is None for p in projected):
-        failed.append({"invariant": "sides", "detail": f"a result has no {varying}"})
+        failed.append({"invariant": "sides", "detail": f"a result has no {varying}: give every bx bench call "
+                                                       "--label NAME=VALUE, and name that label here"})
         return doc
 
     # A side is a distinct axis value, named by its first part (a tree, a label).
@@ -139,7 +140,9 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou
         a, b = (json.loads(side) for side in (base, contender))
         if all(x == y for path, x, y in zip(projection.axis, a, b) if not path.startswith("/provenance/")):
             failed.append({"invariant": "sides-differ",
-                           "detail": f"the sides differ only in {varying}"})
+                           "detail": f"the sides differ only in their label ({varying}): nothing recorded about the "
+                                     "environment differs. Describe what differs with a context collector "
+                                     "(collectors in benchx.conf.json) or a setup file for each environment"})
     if "one-tree-per-side" in check:
         trees = {side: {d["provenance"].get("subject_tree") for d, x in zip(documents, sides) if x == side}
                  for side in (base, contender)}
@@ -157,13 +160,16 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou
         for d, s in zip(documents, sides):
             slot = d.get("procedure", {}).get("slot")
             if slot is None:
-                failed.append({"invariant": "alternating-slots", "detail": "a result has no slot"})
+                failed.append({"invariant": "alternating-slots",
+                               "detail": "a result has no slot: give every bx bench call --slot"})
                 break
             slots.setdefault(slot, set()).add(s)
         else:
             order = [slots[slot] for slot in sorted(slots)]
             if any(len(o) != 1 for o in order) or any(a == b for a, b in zip(order, order[1:])):
-                failed.append({"invariant": "alternating-slots", "detail": "sides do not alternate by procedure.slot"})
+                failed.append({"invariant": "alternating-slots",
+                               "detail": "sides do not alternate by procedure.slot: run the sides in the same "
+                                         "order every round (a, b, a, b, ...)"})
 
     # Units: one workload variant and quantity each. Everything else the
     # projection matches must agree across both sides (schema §5.5).
@@ -191,7 +197,8 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou
             continue
         if "equal-attempts" in check and len(by_side[base]) != len(by_side[contender]):
             failed.append({"invariant": "equal-attempts", **where,
-                           "detail": f"{len(by_side[base])} against {len(by_side[contender])}"})
+                           "detail": f"{len(by_side[base])} against {len(by_side[contender])}: run the same "
+                                     "number of rounds on both sides"})
             continue
         rounds = {}
         for s in (base, contender):
@@ -206,6 +213,8 @@ def compare(documents, *, run_key, profile, baseline, label=None, k=3.0, min_rou
                  if base in r and contender in r]
         unit_doc = _unit(where, pairs, by_side, base, contender, k, min_rounds)
         unit_doc["unmatched_differences"] = _differences(projection, by_side[base], by_side[contender])
+        if caveats := _caveats(by_side, base, contender):
+            unit_doc["caveats"] = caveats
         doc["units"].append(unit_doc)
     return doc
 
@@ -247,6 +256,16 @@ def _declared(document) -> dict:
     return side
 
 
+def _caveats(by_side, base, contender) -> list[str]:
+    """The warnings a side's results carry (quality.warnings), by side, for the reader."""
+    found = set()
+    for role, side in (("baseline", base), ("contender", contender)):
+        for d in by_side[side]:
+            for w in d.get("quality", {}).get("warnings", []):
+                found.add(f"{role}: {w.get('code', w) if isinstance(w, dict) else w}")
+    return sorted(found)
+
+
 def _unit(where, pairs, by_side, base, contender, k, min_rounds):
     first = {s: by_side[s][0] for s in (base, contender)}
     quantity = first[base]["measurement"]["quantity"]
@@ -310,6 +329,13 @@ def render(doc: dict) -> str:
                 shown = side[key] if key != "observed_env" else ", ".join(f"{k}={v}" for k, v in side[key].items())
                 first, *rest = str(shown).splitlines() or [""]
                 lines.append(f"  {name}: {key.replace('_', ' ')}: {first}" + (f" (+{len(rest)} lines)" if rest else ""))
+    differs = sorted({p for u in doc["units"] for p in u.get("unmatched_differences", [])})
+    if differs:
+        shown = ", ".join(_dotted(p) for p in differs[:6])
+        lines.append(f"  differs: {shown}" + (f" (+{len(differs) - 6} more)" if len(differs) > 6 else ""))
+    caveats = sorted({c for u in doc["units"] for c in u.get("caveats", [])})
+    if caveats:
+        lines.append(f"  caveats: {', '.join(caveats)}")
     for section in ("regressed", "improved", "changed", "no-change-detected", "indeterminate"):
         rows = [u for u in doc["units"] if u["verdict"] == section]
         if not rows:

prototype/benchx/session.py

@@ -177,9 +177,11 @@ def bench_target(target: str, *, suite: str, adapter: str, store=None, flags=Non
                  environment_variables=None, filter=None, quantities=("wall-time",), project=None, run_key=None,
                  out=None, source_uri=None, case_s=orders.CASE_TIMEOUT_S, order_s=orders.ORDER_TIMEOUT_S,
                  progress=None, build=True, provider=None, config_path=None, results_dir=None,
-                 collectors=None) -> dict:
-    """Run one target once: a comparison session of one order, with no round,
-    slot or comparison. Results are delivered to `store` unless it is None.
+                 collectors=None, round_=None, slot=None, labels=None) -> dict:
+    """Run one target once: a comparison session of one order, with no comparison.
+    It has a round, a slot and labels when the caller places it in a session that
+    other calls join through the same run key. Results are delivered to `store`
+    unless it is None.
 
     `target` is `@CONFIG[=SOURCE_REF]` or a bare path, which the harness reads
     (see `_side`); `adapter`, `flags` and the rest are as for `compare_targets`.
@@ -195,10 +197,12 @@ def bench_target(target: str, *, suite: str, adapter: str, store=None, flags=Non
     out = Path(out) if out else Path(results_dir or "results") / run_key.replace("/", "_")
     if progress:
         progress(f"precision: {harness.describe_precision(precision, config_path)}")
-    summary, outcome = _run_side(side, out / "orders" / "order.json", store, harness=harness, suite=suite,
+    order_file = "order.json" if slot is None else f"order-{slot:04d}.json"  # one per slot, as compare_targets
+    summary, outcome = _run_side(side, out / "orders" / order_file, store, harness=harness, suite=suite,
                                  filter=filter, quantities=quantities, precision=precision,
                                  environment_variables=environment_variables, project=project, run_key=run_key,
-                                 source_uri=source_uri, case_s=case_s, order_s=order_s, collectors=collectors)
+                                 source_uri=source_uri, case_s=case_s, order_s=order_s, collectors=collectors,
+                                 round_=round_, slot=slot, labels=labels)
     return {**summary, "outcome": outcome, "precision": precision}
 
 

prototype/benchx/snapshot.py

@@ -124,13 +124,22 @@ def _ignored_in(root: Path) -> list[str]:
     return [p.relative_to(root).as_posix() for p in _IGNORED if p != root and p.is_relative_to(root)]
 
 
+def _git_ignores(top, rel) -> bool:
+    """Whether the repository's own ignore rules already leave `rel` out of `git add -A`.
+    Naming such a path in an exclude pathspec makes `git add` fail, and the tree id
+    with it, so a results directory the project git-ignores must not be excluded twice."""
+    return subprocess.run(["git", "check-ignore", "-q", "--", rel], cwd=top,
+                          capture_output=True).returncode == 0
+
+
 def git_identity(path) -> dict | None:
     """Revision, dirty state, and working-tree id of a checkout (schema §4.1)."""
     top = _run(["git", "rev-parse", "--show-toplevel"], cwd=path)
     head = top and _run(["git", "rev-parse", "HEAD"], cwd=top)
     if not head:
         return None
-    excludes = [_PYCACHE_EXCLUDE, *(f":(top,exclude){rel}" for rel in _ignored_in(top))]
+    excludes = [_PYCACHE_EXCLUDE, *(f":(top,exclude){rel}" for rel in _ignored_in(top)
+                                    if not _git_ignores(top, rel))]
     status = _run(["git", "status", "--porcelain", "--untracked-files=all", "--", ".", *excludes], cwd=top)
     # A scratch index that does not exist yet, so the repository's own index
     # is untouched (schema §4.1).

prototype/examples/numpy-venvs/.gitignore

@@ -0,0 +1,2 @@
+.venv-*/
+out/