diff --git a/.gitignore b/.gitignore index e79468111..9866d0f6c 100644 --- a/.gitignore +++ b/.gitignore @@ -501,3 +501,6 @@ data/candidates/ # README's measurement section, so a clone that lacks it has a dangling reference. !/docs/evidence-export.md !/docs/wheel-console.md + +# Public native scorer preparation is a supported cross-machine workflow. +!/docs/public-native-scoring.md diff --git a/data/datasets/alfworld.py b/data/datasets/alfworld.py index 943910dfd..3b647dafd 100644 --- a/data/datasets/alfworld.py +++ b/data/datasets/alfworld.py @@ -57,7 +57,8 @@ The environment is `kind: "files"`. Dataset discovery and the actor's socket client use only the standard library. The trusted relay explicitly binds the separately prepared `alfworld` execution artifact; no host Python package fallback is allowed. -The native verifier retains its independently pinned, sealed scorer runtime. +The native verifier uses public scorer source and locally prepared execution dependencies; +a historical machine archive is not required. What the agent can read, and what that used to cost -------------------------------------------------- @@ -88,16 +89,10 @@ environment the agent acts in and the environment the check replays are built from one text -- `tests/quality/test_alfworld_relay.py` pins that byte identity for all 134 games. -Two things about this are **not** in this repository yet, and both are required before -an end-to-end run: (1) the runner must stage those bytes outside the sandbox and start -`relay.py`, then pass the socket to the sandbox as `tool_socket=` so it is published at -`/run/harnessgrad-tool/tool.sock` where the CLI imports and calls it (the mount wiring -lives in `eval/containers/`, a separate task -- and this migration is the first to -require `PLAN_VERSION` 4 -> 5, because from then on what a subject sees has changed); -(2) a relay fault currently makes `alf` exit nonzero, and nothing yet maps that to an -**unmeasured** task rather than a 0 -- `Relay.receipt` is the evidence the runner needs -for that. Neither is faked here. `control.py --oracle` (the local arm) publishes the -same pair into a temp directory itself and is unaffected. +The runner stages the selected game outside the actor's namespace, starts the +managed relay and publishes only its socket and stdlib client. Relay failures +remain unmeasured rather than becoming an agent's zero. `control.py --oracle`, +`--noop` and `--platform` use this lifecycle and the public native verifier. The metric, and what is deliberately not added ---------------------------------------------- @@ -496,9 +491,8 @@ def _verifier(record: dict) -> dict: "evidence_file": EVIDENCE_FILE, "timeout_s": 300, } - from data.runtimes import NATIVE_PINS - spec["native_runtime"] = NATIVE_PINS["alfworld"].copy() - return bind_scorer_input(spec, record["task_id"], [GAME_IN_WORKDIR]) + from data.scaffolds.public_native import bind as bind_public_native + return bind_public_native("alfworld", bind_scorer_input(spec, record["task_id"], [GAME_IN_WORKDIR])) def downloads() -> list[dict]: diff --git a/data/datasets/alfworld/control.py b/data/datasets/alfworld/control.py index 4b674c048..b9b4d7c77 100644 --- a/data/datasets/alfworld/control.py +++ b/data/datasets/alfworld/control.py @@ -8,7 +8,7 @@ a reader to be able to reproduce. Both legacy `--oracle`/`--noop` and `--platform` use the normal runner, -managed trusted service and independently sealed native verifier. Dependencies +managed trusted service and public native verifier. Dependencies must be prepared explicitly; platform Python and the actor CLI remain stdlib. `--no-sandbox` selects the runner's weaker diagnostic mode for either spelling. diff --git a/data/datasets/alfworld/execution-recipe.md b/data/datasets/alfworld/execution-recipe.md index aee0463a8..6f3644d5d 100644 --- a/data/datasets/alfworld/execution-recipe.md +++ b/data/datasets/alfworld/execution-recipe.md @@ -3,7 +3,7 @@ ALFWorld trusted interactive service Discovery, task filtering, game stripping, the actor CLI and action-log parsing use the standard library. Third-party imports occur only in `gym_env.build()` in -the trusted service or the independently sealed native verifier. Task setup binds +the trusted service or the public native verifier. Task setup binds a prepared execution artifact with role `service`; a missing or changed artifact is an error, without host interpreter fallback or implicit installation. @@ -31,9 +31,9 @@ without importing those packages. `native-source-version-evidence.json` records these limited identity checks; it does not claim a complete original inventory. The exact pins in `execution-requirements.txt` are a **new service dependency resolution**, made on 2026-11-07 with uv using the public Tsinghua index for CPython 3.12/Linux. They -are not a reconstruction of the historical scorer environment. The existing -`data/runtimes/alfworld-native.json`, its hashes and the native scoring algorithm -are unchanged and continue to define verification. +are not a reconstruction of the historical scorer environment. Current verification uses the public source scorer and this locally prepared +execution recipe. `data/runtimes/alfworld-native.json` remains historical provenance; +it is not required for new evaluations. See [public native scoring](../../../docs/public-native-scoring.md). Only three source files are mounted for the service: relay.py, lib/gym_env.py and the platform tool_transport.py. The runner separately stages the selected stripped @@ -54,8 +54,7 @@ python -m pytest -q tests/quality/test_alfworld_managed_runtime.py \ ``` The legacy `control.py --oracle` / `--noop` and `--platform` spellings now all -use the normal runner with the complete managed relay declaration and original -sealed native verifier. `--budget` is carried consistently to service, actor CLI +use the normal runner with the complete managed relay declaration and public native verifier. `--budget` is carried consistently to service, actor CLI and replay; missing scores or relay failures remain unmeasured. The older helper API also starts the admitted service through the runner's relay lifecycle and scores through the native verifier; it never imports the engine in platform Python. diff --git a/data/datasets/bird_minidev.py b/data/datasets/bird_minidev.py index f923534e6..04e9fadbf 100644 --- a/data/datasets/bird_minidev.py +++ b/data/datasets/bird_minidev.py @@ -270,9 +270,8 @@ def _verify(record: dict) -> dict: {"path": "database.sqlite", "from": str(database)}, ], } - from data.runtimes import NATIVE_PINS - spec["native_runtime"] = NATIVE_PINS["bird_minidev"].copy() - return bind_scorer_input(spec, record["task_id"], ["gold.sql", "database.sqlite"]) + from data.scaffolds.public_native import bind as bind_public_native + return bind_public_native("bird_minidev", bind_scorer_input(spec, record["task_id"], ["gold.sql", "database.sqlite"])) def setup_map() -> dict: diff --git a/data/datasets/bird_minidev/execution-runtime.json b/data/datasets/bird_minidev/execution-runtime.json new file mode 100644 index 000000000..9087d8f1e --- /dev/null +++ b/data/datasets/bird_minidev/execution-runtime.json @@ -0,0 +1,6 @@ +{ + "schema": "harnessgrad.dataset-execution-runtime.v1", + "dataset": "bird_minidev", + "python": "3.12", + "requirements": "scoring-requirements.txt" +} diff --git a/data/datasets/bird_minidev/scoring-requirements.txt b/data/datasets/bird_minidev/scoring-requirements.txt new file mode 100644 index 000000000..e516f04c9 --- /dev/null +++ b/data/datasets/bird_minidev/scoring-requirements.txt @@ -0,0 +1 @@ +# BIRD EX uses Python standard-library sqlite3; no external Python packages. diff --git a/data/datasets/chessbench_puzzles.py b/data/datasets/chessbench_puzzles.py index 0331d927c..4c0cf834b 100644 --- a/data/datasets/chessbench_puzzles.py +++ b/data/datasets/chessbench_puzzles.py @@ -442,13 +442,12 @@ def setup_for(record: dict) -> dict: def verify_for(record: dict) -> dict: - from data.runtimes import NATIVE_PINS + from data.scaffolds.public_native import bind as bind_public_native spec = _SPEC.verifier(record) - # The native runner supplies -I -S from its independently pinned manifest; - # retain its fixed script-at-argv[1] route contract. + # The public native runner supplies isolated Python flags; retain its + # script-at-argv[1] route contract. spec['argv'].remove('-I') - spec['native_runtime'] = NATIVE_PINS['chessbench_puzzles'].copy() - return spec + return bind_public_native('chessbench_puzzles', spec) __all__ = ["KIND", "DATASET_ROOT", "REVISION", "load", "load_selected", "control_task", diff --git a/data/datasets/chessbench_puzzles/NATIVE_RUNTIME.md b/data/datasets/chessbench_puzzles/NATIVE_RUNTIME.md index 9cc83d543..e66a2093e 100644 --- a/data/datasets/chessbench_puzzles/NATIVE_RUNTIME.md +++ b/data/datasets/chessbench_puzzles/NATIVE_RUNTIME.md @@ -1,3 +1,9 @@ +Current evaluation uses the public-native-local-v1 profile. Prepare this dataset's +execution dependencies with `python tools/prepare_dataset.py --dataset chessbench_puzzles --runtime execution`. +No original scorer-machine archive is required. See [public native scoring](../../../docs/public-native-scoring.md). +The exact-byte archive material below is retained as historical evidence or explicit +legacy reproduction guidance; it is not the current default installation requirement. + # Independent ChessBench native scorer The managed `chess==1.11.2` provider is for loading and interactive board state. diff --git a/data/datasets/chessbench_puzzles/execution-runtime.json b/data/datasets/chessbench_puzzles/execution-runtime.json new file mode 100644 index 000000000..e92d26268 --- /dev/null +++ b/data/datasets/chessbench_puzzles/execution-runtime.json @@ -0,0 +1,6 @@ +{ + "schema": "harnessgrad.dataset-execution-runtime.v1", + "dataset": "chessbench_puzzles", + "python": "3.12", + "requirements": "requirements-load.txt" +} diff --git a/data/datasets/jericho.py b/data/datasets/jericho.py index 8fc20fbbf..9d2e8c536 100644 --- a/data/datasets/jericho.py +++ b/data/datasets/jericho.py @@ -276,9 +276,8 @@ def _verifier(record: dict) -> dict: "evidence_file": EVIDENCE_FILE, "timeout_s": VERIFY_TIMEOUT_S, } - from data.runtimes import NATIVE_PINS - spec["native_runtime"] = NATIVE_PINS["jericho"].copy() - return bind_scorer_input(spec, record["task_id"], [GAME_FILE, record["rom"]]) + from data.scaffolds.public_native import bind as bind_public_native + return bind_public_native("jericho", bind_scorer_input(spec, record["task_id"], [GAME_FILE, record["rom"]])) def downloads() -> list[dict]: diff --git a/data/datasets/jericho/NATIVE_CONTROL.md b/data/datasets/jericho/NATIVE_CONTROL.md index 4342cd9b1..e59245e45 100644 --- a/data/datasets/jericho/NATIVE_CONTROL.md +++ b/data/datasets/jericho/NATIVE_CONTROL.md @@ -1,3 +1,9 @@ +Current evaluation uses the public-native-local-v1 profile. Prepare this dataset's +execution dependencies with `python tools/prepare_dataset.py --dataset jericho --runtime execution`. +No original scorer-machine archive is required. See [public native scoring](../../../docs/public-native-scoring.md). +The exact-byte archive material below is retained as historical evidence or explicit +legacy reproduction guidance; it is not the current default installation requirement. + # Original supported-game native control `authored_story.py` generates an original, CC0-1.0 Z-machine v5 story. It contains diff --git a/data/datasets/scienceworld.py b/data/datasets/scienceworld.py index 09b8b6443..b5702aad7 100644 --- a/data/datasets/scienceworld.py +++ b/data/datasets/scienceworld.py @@ -468,9 +468,8 @@ def _verifier(record: dict, *, world_jar=None) -> dict: "evidence_file": EVIDENCE_FILE, "timeout_s": 900, } - from data.runtimes import NATIVE_PINS - spec['native_runtime'] = NATIVE_PINS['scienceworld'].copy() - return bind_scorer_input(spec, record["task_id"], [TASK_IN_WORKDIR, "scienceworld.jar"]) + from data.scaffolds.public_native import bind as bind_public_native + return bind_public_native("scienceworld", bind_scorer_input(spec, record["task_id"], [TASK_IN_WORKDIR, "scienceworld.jar"])) def downloads() -> list[dict]: diff --git a/data/datasets/scienceworld/ISOLATION_STATUS.md b/data/datasets/scienceworld/ISOLATION_STATUS.md index 9b8006b1f..d4907ecc7 100644 --- a/data/datasets/scienceworld/ISOLATION_STATUS.md +++ b/data/datasets/scienceworld/ISOLATION_STATUS.md @@ -1,3 +1,9 @@ +Current evaluation uses the public-native-local-v1 profile. Prepare this dataset's +execution dependencies with `python tools/prepare_dataset.py --dataset scienceworld --runtime execution`. +No original scorer-machine archive is required. See [public native scoring](../../../docs/public-native-scoring.md). +The exact-byte archive material below is retained as historical evidence or explicit +legacy reproduction guidance; it is not the current default installation requirement. + ScienceWorld actor dependency boundary ===================================== diff --git a/data/runtimes/__init__.py b/data/runtimes/__init__.py index e0a360320..2d092cf0b 100644 --- a/data/runtimes/__init__.py +++ b/data/runtimes/__init__.py @@ -1,4 +1,8 @@ -"""Reviewed independent admission pins; never recalculate expectations at runtime.""" +"""Historical sealed scorer pins for explicit archive reproduction. + +Current built-in scoring uses public-native-local-v1 and local execution recipes. +These immutable pins preserve old evidence; they are not new-machine prerequisites. +""" NATIVE_PINS = { "alfworld": { diff --git a/data/scaffolds/public_native.py b/data/scaffolds/public_native.py new file mode 100644 index 000000000..6d8ac0748 --- /dev/null +++ b/data/scaffolds/public_native.py @@ -0,0 +1,45 @@ +"""Bind public scorer source and dependency recipes, not a historical machine image.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path, PurePosixPath + +ROOT = Path(__file__).resolve().parents[2] +PROFILE = 'public-native-local-v1' +DATASETS = frozenset(('alfworld', 'bird_minidev', 'chessbench_puzzles', 'jericho', 'scienceworld')) + + +def identity(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def bind(dataset: str, spec: dict) -> dict: + """Keep original argv/metric/inputs; identify only the public Python scorer files.""" + if dataset not in DATASETS: + raise ValueError('unsupported public native scorer') + sources = {} + for entry in spec.get('inputs', []): + name = entry.get('path', '') + if not name.endswith('.py'): + continue + path = PurePosixPath(name) + if path.is_absolute() or '..' in path.parts or str(path) != name: + raise ValueError('public scorer code path must be normalized and relative') + if 'from' in entry: + source = Path(entry['from']).resolve(strict=True) + if not source.is_relative_to(ROOT) or not source.is_file(): + raise ValueError('public scorer code must come from the installed platform') + content = source.read_bytes() + value = {'source': source.relative_to(ROOT).as_posix()} + else: + content = entry['content'].encode('utf-8') + value = {'content': entry['content']} + value['sha256'] = hashlib.sha256(content).hexdigest() + if name in sources and sources[name] != value: + raise ValueError('conflicting public scorer code entries') + sources[name] = value + if not sources: + raise ValueError('public native scorer requires declared scoring code') + binding = {'dataset': dataset, 'profile': PROFILE, 'sources': sources} + return {**spec, 'native_runtime': {**binding, 'sha256': identity(binding)}} diff --git a/docs/dataset-runtime-interfaces.md b/docs/dataset-runtime-interfaces.md index 360623c8e..d676ac093 100644 --- a/docs/dataset-runtime-interfaces.md +++ b/docs/dataset-runtime-interfaces.md @@ -239,36 +239,25 @@ VERIFY[task_id] = { 都必须在声明目录或沙箱已有的系统运行目录内。缺失、不可见或无效的声明保持不可测。 解释器版本及依赖要求由适配器确定，通用平台不要求某个 Python 版本。 -### 5.2 封存原生评分运行时的宿主兼容性 +### 5.2 公开原生评分与本机依赖 -Bird、Jericho、ALFWorld、ScienceWorld 和 ChessBench 的 `native_runtime` 清单固定 -解释器、依赖、评分代码、沙箱映射及 launcher 的内容摘要。清单 `identity.kernel` -保留生成时记录的内核版本；准入要求 Linux 和匹配的机器架构，并检查实际能力， -不要求宿主内核版本字符串与构建时完全相同。 +ALFWorld、Bird Mini-Dev、ChessBench puzzles、Jericho 和 ScienceWorld 默认使用 +`public-native-local-v1`。评分算法和原生引擎的版本要求是公开的；新机器通过 +`tools/prepare_dataset.py --dataset --runtime execution` 准备自己的运行环境， +不需要恢复原机器的 Python、标准库、共享库、依赖或 launcher 的字节。 +完整命令、指标和边界见[公开原生评分](public-native-scoring.md)。 -每次准入先校验文件并封存到 memfd，确认写入、扩缩和继续加封均被锁定，然后使用 -已封存的 launcher 和解释器运行独立探针。探针检查私有 user/mount/PID/network/IPC/UTS -命名空间、只读运行时及任务挂载、不可见的宿主临时文件、隔离 Python 和封存的子解释器启动。 -能力缺失、探针失败或超时仍按不可测处理，不回退到宿主解释器或关闭隔离。 +评分代码在可信目录中生成只读快照，隔离 Python 启动，依赖使用本机显式准备的环境。 +回执记录公开代码身份、实际 Python/SQLite/依赖版本、当前运行时身份和宿主信息。 +本机环境的文件清单用于检测本次运行中被更改的安装，不用于要求跨机器复制旧环境。 +数据与评分规则的校验、错误与零分的区分、只读输入和 actor 隔离继续生效。 +不同本机运行时可执行同一公开评分；复用旧分数仍要求对应测量身份一致。 -宿主能力探针使用独立的 30 秒预算，覆盖完整封存运行时的沙箱挂载与探针执行， -为 ALFWorld 等较大运行时的并发准入预留启动时间。评分命令使用验证阶段自己的超时配置。 +旧 schema 1/2 封存清单仅供显式历史复现使用，后文描述其归档格式， +不构成新评测的安装前提。ChessBench 交互协议仍由其控制器评分， +离线公开评分声明不会替代交互协议本身的语义。 -评分回执的 `native_runtime.identity` 保留清单身份；新增 `native_runtime.host` 记录实际 -system、machine、kernel、准入策略及通过的能力检查，逐次 verdict 归档保留这些字段。 -内核与 launcher 的宿主加载环境仍属于平台可信边界；这些检查不声明所有内核行为等价。 - -评分缓存读取前使用同一套声明校验和运行时准入，并按每个选中的声明去重检查。 -缓存键包含已验证的 native 回执（清单摘要、用户态身份及实际宿主兼容回执）；路径定位改变而 -内容身份不变可以复用，运行时或宿主回执改变则重新测量。缓存中的最后一个 verdict 和每次 -trial 的 native 回执必须全部匹配；新测量若出现不匹配则保持不可测且不写入缓存。 -旧缓存键、缺失或未验证的 native 回执均不能作为当前原生评分的结果使用。 - -上述 native 校验仅用于实际执行原生评分器的路径。ChessBench 交互模式由进程内控制器 -评分，不使用 verifier 中为离线评分准备的 `native_runtime` 声明；交互缓存按协议区分， -复用前仍要求通过交互会话的沙箱、环境类型及可信输入校验。 - -### 5.3 可搬迁的原生运行时包 +### 5.3 历史原生运行时包（显式复现） 五份受审查清单使用 schema 2。`files[*].source` 和 `launcher.source` 只引用 `blobs/`，宿主读取位置由外部存储配置决定。包同时包含解释器、用户态依赖、 diff --git a/docs/getting-started.md b/docs/getting-started.md index 618bf8a8a..be7fdefe5 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -59,6 +59,11 @@ command when the native runtime has not been prepared. This is a warning because the run can also build the runtime from cached wheels. If a published runtime fails validation, the check exits with its specific error. +ALFWorld, BIRD Mini-Dev, ChessBench puzzles, Jericho and ScienceWorld use public +native scoring with locally prepared execution dependencies. They do not require +the original scorer-machine archive. See [public native scoring](public-native-scoring.md) +for the five recipes and preparation commands. + Data download and dependency preparation are separate operations. For a research recipe using GSM8K, set an external data root and prepare its loading environment: diff --git a/docs/linux-runtime-remaining.md b/docs/linux-runtime-remaining.md index 5758f12ce..a995e9fc9 100644 --- a/docs/linux-runtime-remaining.md +++ b/docs/linux-runtime-remaining.md @@ -1,3 +1,9 @@ +Current default for the five Python/native host scorers is described in +[public native scoring](public-native-scoring.md). They use locally prepared public +dependencies, not original-machine archives. The archive recovery receipts and +blockers below preserve historical observations; they are not current new-machine +installation requirements. + # Linux runtime acceptance remaining work The [2026-11-08 final-source Linux validation](linux-release-validation.md) records diff --git a/docs/public-native-scoring.md b/docs/public-native-scoring.md new file mode 100644 index 000000000..56b7a05d8 --- /dev/null +++ b/docs/public-native-scoring.md @@ -0,0 +1,85 @@ +# Public native scoring on a new machine + +ALFWorld, BIRD Mini-Dev, ChessBench puzzles, Jericho and ScienceWorld now use the +`public-native-local-v1` scoring profile by default. New evaluations do not require +an archive of the original machine's interpreter, standard library, shared +libraries, packages or Bubblewrap binary. + +The scoring code and native algorithm stay explicit: + +| Dataset | Scoring logic / engine | +| --- | --- | +| `alfworld` | Replay the submitted actions in a fresh ALFWorld/TextWorld environment; use native `won`. | +| `bird_minidev` | Execute prediction and gold SQL with local Python's SQLite engine; retain the documented EX set-comparison rule. | +| `chessbench_puzzles` | Public ChessBench scoring source and the declared python-chess version; retain the existing strict replay policy and evidence. | +| `jericho` | Replay commands with Jericho's native interpreter and use its score/max-score. | +| `scienceworld` | Replay actions through the public ScienceWorld engine/JVM and use the final native score. | + +The dependency versions are in each dataset's `execution-runtime.json` and named +requirements file. The current recipes support CPython 3.12; they do not require a +particular 3.12 patch release, historical executable hash or historical host libc. +ScienceWorld still needs its public, versioned JRE/JAR assets. Game data, ROMs, +databases, splits and benchmark assets remain separate preparation requirements. + +## Prepare a local scoring environment + +With an already installed, compatible Python interpreter, prepare only the dataset +you will use. This explicit command may download its declared public dependencies: + +```bash +python tools/prepare_dataset.py --dataset alfworld --runtime execution \ + --python /absolute/path/to/python3.12 --runtime-root /external/dataset-runtimes +python tools/prepare_dataset.py --dataset alfworld --runtime execution \ + --runtime-root /external/dataset-runtimes --check +``` + +Replace `alfworld` with `bird_minidev`, `chessbench_puzzles`, `jericho` or +`scienceworld`. Set `HG_DATASET_RUNTIME_ROOT=/external/dataset-runtimes` for the run. +For offline preparation, use an existing wheel cache and `UV_OFFLINE=1`. BIRD's +execution requirements are empty because its engine is standard-library SQLite. +Provider loading dependencies, where declared, are prepared separately with the +same tool's default `--runtime provider` profile. Check does not launch a task or +prove benchmark quality; it verifies the prepared local environment. + +The actor's interactive service and offline scorer use the selected dataset's +execution recipe. Scoring imports trusted code from a private read-only snapshot, +with isolated Python startup and a read-only task view. The actor cannot replace +the scoring code or inject task-local Python modules into scorer imports. + +## What is checked and recorded + +* The public scoring source and input contract identify what algorithm was run. +* The actual local interpreter, SQLite version, dependency versions and prepared + runtime identity are recorded with the verdict. +* A freshly prepared runtime on another machine is admitted without comparing its + interpreter or dependency bytes to a historical machine. A read-only local + inventory still detects changes to the selected artifact during a run. +* Missing dependencies, startup failures or engine failures remain **unmeasured**; + they are not silently converted into an agent's zero. +* Changed scoring source, input data or local runtime identity leads to a fresh + measurement. A common public metric does not authorize silently reusing an old + numerical score from a different environment. + +## Historical archives + +The old `data/runtimes/*-native.json` pins and their provenance remain available +for explicitly requested forensic reproduction. `tools/prepare_native_runtime.py`, +`tools/build_bird_native_runtime.py` and `tools/prepare_chessbench_native.py` are +historical archive tools, not prerequisites for current evaluation. They retain +their exact-byte semantics so old evidence is not relabelled as a new measurement. + +Synthetic tests cover all five default declarations and public recipes. An authored +SQLite control runs the actual BIRD scorer in locally prepared environments, +checks positive/negative rewards and actor import isolation, and proves two distinct +local installations can run the same public scorer. This is not a released-benchmark +or real-model result; native engine controls require their public dependencies and +assets on the target machine. + +The optional authored Chess controls use an already prepared public store: + +```bash +HG_PUBLIC_NATIVE_TEST_ROOT=/external/dataset-runtimes python -m pytest -q \ + tests/quality/test_public_native_scorers.py -k public_chess +``` + +They run only a fabricated starting-position trajectory and make no model calls. diff --git a/docs/wheel-adapter-resource-inventory.md b/docs/wheel-adapter-resource-inventory.md index 37d80aa23..7895e9f2e 100644 --- a/docs/wheel-adapter-resource-inventory.md +++ b/docs/wheel-adapter-resource-inventory.md @@ -76,3 +76,9 @@ benchmark or proves all external dependencies are installed. Loop version selection also ships the pinned `tools/assets/loop-0.2.0.zip` source archive. It is exported by `harnessgrad harness install loop --version 0.2.0` without Git, Node or downloads; preparation of the selected version's dependencies remains separate. + +The current public-native scoring route ships `execution-runtime.json` for +`bird_minidev` and `chessbench_puzzles`, plus BIRD's empty standard-library scoring +requirements. The other three native scorer datasets share their published service +execution recipes. Historical native archive manifests remain packaged as old +evidence, not default evaluation prerequisites. diff --git a/eval/verification/native_runtime.py b/eval/verification/native_runtime.py index c9ec200b7..f82ea1fb2 100644 --- a/eval/verification/native_runtime.py +++ b/eval/verification/native_runtime.py @@ -1,11 +1,8 @@ -"""Sealed native userland pins for explicitly supported host scorer routes. +"""Native scoring admission: public local recipes and explicit historical archives. -Hash while copying into sealed memfds; execute only those snapshots in an empty -bubblewrap root. Never hash then execute a mutable source path. Missing bytes, -manifest drift, missing namespace support, and startup failure are unmeasured. -The host kernel/CPU and namespace launcher remain platform compatibility/TCB. -The captured kernel release is build metadata; admission checks Linux capabilities -and records the actual execution host instead of requiring an identical release. +Current dataset defaults select public-native-local-v1. The retained schema 1/2 +implementation authenticates and seals old userland snapshots only when those +historical bindings are explicitly selected; it is not a new-machine prerequisite. """ from __future__ import annotations @@ -240,7 +237,15 @@ def _check_host(runtime: FrozenRuntime) -> dict: @contextmanager def admit(binding: dict): - """Authenticate the selected manifest and retain exact sealed bytes until exit.""" + """Admit a public local scorer, or an explicitly selected historical snapshot.""" + if isinstance(binding, dict) and binding.get('profile') == 'public-native-local-v1': + from eval.verification.public_native_runtime import admit as admit_public + try: + with admit_public(binding) as runtime: + yield runtime + except (OSError, ValueError, RuntimeError, KeyError, TypeError, AttributeError, subprocess.SubprocessError) as exc: + raise NativeRuntimeUnavailable(f'public native runtime unavailable: {exc}') from exc + return fds = {} launcher = None try: diff --git a/eval/verification/public_native_runtime.py b/eval/verification/public_native_runtime.py new file mode 100644 index 000000000..2f88fc40d --- /dev/null +++ b/eval/verification/public_native_runtime.py @@ -0,0 +1,104 @@ +"""Native benchmark scoring using public source and locally prepared dependencies. + +Only scoring source is authenticated against the selected repository revision. +The interpreter and installed packages belong to this machine's prepared runtime; +no bytes from an earlier machine are required. Local identities remain recorded. +""" +from __future__ import annotations + +from contextlib import contextmanager +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import platform +import tempfile + +from data.scaffolds.public_native import DATASETS, PROFILE, ROOT, identity +from eval.containers import sandbox +from harnessgrad import dataset_runtime + + +class PublicRuntime: + def __init__(self, binding, runtime, code): + self.binding, self.runtime, self.code = binding, runtime, code + self.receipt = {'dataset': binding['dataset'], 'sha256': binding['sha256'], + 'verified': True, 'mechanism': PROFILE, + 'files': len(binding['sources']), + 'identity': {'public_logic_sha256': binding['sha256'], + 'python_runtime': runtime, + 'dependency_versions': dataset_runtime.execution_descriptor(binding['dataset'])['pins']}, + 'host': {'system': platform.system(), 'machine': platform.machine(), + 'kernel': platform.release(), 'admission_policy': 'local-public-runtime-v1'}} + + def run(self, argv, *, workdir, reward_dir, timeout_s, immutable_runtimes=()): + dataset_runtime.admit_binding(self.runtime, 'verifier') + task = Path(workdir).resolve() + for value in self.runtime['runtime_paths']: + path = Path(value).resolve() + if path == task or path.is_relative_to(task) or task.is_relative_to(path): + raise ValueError('public scorer dependencies overlap task storage') + if argv[1] == '-c': + bootstrap = argv[2] + arguments = [] + else: + script = PurePosixPath(argv[1]) + if str(script) not in self.binding['sources']: + raise ValueError('entry point is outside public scorer sources') + bootstrap = ('import runpy; args=sys.argv[1:]; sys.argv=args; ' + 'runpy.run_path(args[0],run_name="__main__")') + arguments = [str(self.code / script), *argv[2:]] + # Avoid task-local module shadows and package startup hooks. All scorer + # imports resolve from the trusted code snapshot or the local prepared site. + site = Path(self.runtime['path']) / 'lib/python3.12/site-packages' + prefix = 'import sys; sys.path[:0]=' + repr([str(self.code), str(site)]) + '; ' + command = [self.runtime['python'], '-I', '-S', '-B', '-c', prefix + bootstrap, *arguments] + env = {'PATH': os.defpath, 'HOME': '/tmp', 'TMPDIR': '/tmp', 'LANG': 'C.UTF-8', + 'LC_ALL': 'C.UTF-8', 'PYTHONDONTWRITEBYTECODE': '1'} + return sandbox.run(command, platform=ROOT, work_root=Path(workdir), task_dir=Path(workdir), + repo=self.code, env=env, cwd=Path(workdir), timeout_s=timeout_s, + task_dir_writable=False, runtime_paths=tuple(Path(p) for p in self.runtime['runtime_paths']), + writable_paths=(Path(reward_dir),), network='none', + immutable_runtimes=immutable_runtimes, platform_interpreter=False) + + +@contextmanager +def admit(binding): + if (not isinstance(binding, dict) or set(binding) != {'dataset','profile','sources','sha256'} + or binding['dataset'] not in DATASETS or binding['profile'] != PROFILE + or binding['sha256'] != identity({k:v for k,v in binding.items() if k != 'sha256'})): + raise ValueError('invalid public native scoring contract') + selected = dataset_runtime.bind(dataset_runtime.resolve_execution(binding['dataset']), 'verifier') + with tempfile.TemporaryDirectory(prefix='hg-public-scorer-') as temporary: + code = Path(temporary) + for name, entry in binding['sources'].items(): + relative = PurePosixPath(name) + if not name.endswith('.py') or relative.is_absolute() or '..' in relative.parts or str(relative) != name: + raise ValueError('invalid public scorer source path') + if 'source' in entry: + source = (ROOT / entry['source']).resolve(strict=True) + if not source.is_relative_to(ROOT) or not source.is_file(): + raise ValueError('public scorer source escapes the platform') + raw = source.read_bytes() + else: + raw = entry['content'].encode('utf-8') + if hashlib.sha256(raw).hexdigest() != entry['sha256']: + raise ValueError('public scoring code changed during selection') + target = code / relative; target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(raw) + runtime = PublicRuntime(binding, selected, code) + work = code / '.probe-task'; work.mkdir() + reward = work / 'verifier'; reward.mkdir() + probe_code = 'import json; facts={"python":sys.version,"isolated":sys.flags.isolated,"no_site":sys.flags.no_site}; ' + if binding['dataset'] == 'bird_minidev': + probe_code += 'import sqlite3; facts["sqlite"]=sqlite3.sqlite_version; ' + probe_code += 'print(json.dumps(facts))' + probe = runtime.run(['python3', '-c', probe_code], + workdir=work, reward_dir=reward, timeout_s=30) + if probe.returncode: + raise ValueError('public scorer runtime startup failed: ' + probe.stderr[-1500:]) + facts = json.loads(probe.stdout) + if not facts.get('isolated') or not facts.get('no_site'): + raise ValueError('public scorer Python isolation unavailable') + runtime.receipt['identity']['execution'] = facts + yield runtime diff --git a/pyproject.toml b/pyproject.toml index 4b927bdaa..2a343751f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,7 +162,7 @@ exclude = ["data.authored*", "data.candidates*", "console.uat*"] "data.scaffolds.native_research.vendor.flashrsi.benchmarks.travelplanner" = ["upstream.json"] "data.scaffolds.native_research.vendor.flashrsi.benchmarks.widesearch" = ["upstream.json"] "data.datasets.livecodebench" = ["execution-runtime.json", "requirements.lock", "LOADER_RUNTIME.md", "cli/lcb"] -"data.datasets.chessbench_puzzles" = ["runtime.json", "requirements-load.txt", "LOADER_RUNTIME.md", "NATIVE_RUNTIME.md"] +"data.datasets.chessbench_puzzles" = ["execution-runtime.json", "runtime.json", "requirements-load.txt", "LOADER_RUNTIME.md", "NATIVE_RUNTIME.md"] "data.datasets.appworld" = ["execution-runtime.json", "requirements.lock", "PREPARATION.md", "cli/aw"] "data.runtimes" = ["chessbench_puzzles-native.json", "alfworld-native.json", "bird_minidev-native.json", "jericho-native.json", "scienceworld-native.json"] @@ -193,7 +193,7 @@ exclude = ["data.authored*", "data.candidates*", "console.uat*"] "source-manifest.json", "prepared-manifest.json", "README.md"] "data.datasets.alfworld" = ["cli/alf", "execution-runtime.json", "execution-requirements.in", "execution-requirements.txt", "execution-recipe.md", "native-source-version-evidence.json"] -"data.datasets.bird_minidev" = ["cli/bird"] +"data.datasets.bird_minidev" = ["execution-runtime.json", "scoring-requirements.txt", "cli/bird"] "data.datasets.gaia" = ["cli/ask", "PREPARATION.md"] "data.datasets.jericho" = ["cli/jer", "cli/jer-relay", "runtime.json", "execution-runtime.json", "requirements.lock", "LICENSE.upstream", "RUNTIME_PROVENANCE.json", diff --git a/tests/quality/test_chess_native_preparation.py b/tests/quality/test_chess_native_preparation.py index 231b4a1d7..62a88ff9f 100644 --- a/tests/quality/test_chess_native_preparation.py +++ b/tests/quality/test_chess_native_preparation.py @@ -81,7 +81,9 @@ def test_actual_original_native_scorer_authored_trajectories(tmp_path, monkeypat 'answer': 'e7e5 b8c6', 'puzzle_id': 'PUBLIC-AUTHORED-CHESS-CONTROL', 'private_reference': 'PRIVATE-AUTHORED-REFERENCE'} spec = dataset.verify_for({'task_id': 'chessbench_puzzles/test/0', 'gold': gold}) - assert spec['native_runtime'] == prepare.BINDING + # This opt-in control deliberately exercises the retained historical archive. + assert spec['native_runtime']['profile'] == 'public-native-local-v1' + spec['native_runtime'] = prepare.BINDING.copy() work = tmp_path / 'work' work.mkdir() (work / 'answer.txt').write_text(prediction) diff --git a/tests/quality/test_jericho_managed_adapter.py b/tests/quality/test_jericho_managed_adapter.py index 1fc2e06f6..f6088a0cc 100644 --- a/tests/quality/test_jericho_managed_adapter.py +++ b/tests/quality/test_jericho_managed_adapter.py @@ -39,8 +39,8 @@ def test_private_control_keeps_reference_out_of_task_and_actor_files(tmp_path, m snapshot = jericho.command_control('authored') assert snapshot['oracle_files'] == {'.oracle/actions.json': json.dumps(item['walkthrough'])} assert 'PRIVATE-AUTHORED-REFERENCE' not in json.dumps(snapshot['task']) - from data.runtimes import NATIVE_PINS - assert snapshot['verifier']['native_runtime'] == NATIVE_PINS['jericho'] + assert snapshot['verifier']['native_runtime']['profile'] == 'public-native-local-v1' + assert 'manifest' not in snapshot['verifier']['native_runtime'] def test_rom_content_changes_verification_context(tmp_path): diff --git a/tests/quality/test_native_runtime_hostpin.py b/tests/quality/test_native_runtime_hostpin.py index 4c708087d..eb254570f 100644 --- a/tests/quality/test_native_runtime_hostpin.py +++ b/tests/quality/test_native_runtime_hostpin.py @@ -181,14 +181,15 @@ def test_missing_payload_cannot_fall_back_to_host(tmp_path, local_runtime): assert got['native_runtime']['verified'] is False -def test_adapter_declares_independently_fixed_runtime_without_reading_dataset(tmp_path, monkeypatch): +def test_adapter_declares_public_scoring_without_reading_dataset(tmp_path, monkeypatch): import data.datasets.bird_minidev as adapter dbroot = tmp_path / 'dbs' (dbroot / 'authored').mkdir(parents=True) (dbroot / 'authored/authored.sqlite').write_bytes(b'authored input pin only') monkeypatch.setattr(adapter, 'DBS', dbroot) selected = adapter._verify({'task_id': 'authored', 'db_id': 'authored', 'gold_sql': 'select 1'}) - assert selected['native_runtime']['sha256'] == binding()['sha256'] + assert selected['native_runtime']['profile'] == 'public-native-local-v1' + assert 'manifest' not in selected['native_runtime'] assert selected['native_runtime']['dataset'] == 'bird_minidev' diff --git a/tests/quality/test_native_runtime_hp3.py b/tests/quality/test_native_runtime_hp3.py index 6229050d4..757e42b29 100644 --- a/tests/quality/test_native_runtime_hp3.py +++ b/tests/quality/test_native_runtime_hp3.py @@ -1,6 +1,7 @@ """Authored native replay controls; no benchmark inputs or model calls.""" import hashlib import json +import os from pathlib import Path import sys @@ -13,7 +14,7 @@ DATASETS = ['jericho', 'alfworld'] -def adapter_spec(dataset, tmp_path): +def public_adapter_spec(dataset, tmp_path): if dataset == 'jericho': from data.datasets import jericho rom = tmp_path / 'authored.z3' @@ -24,6 +25,13 @@ def adapter_spec(dataset, tmp_path): return alfworld._verifier({'task_id': 'authored/alfworld', 'stripped_game': '{}'}) +def adapter_spec(dataset, tmp_path): + """Explicit historical route for the sealed-archive regression controls below.""" + spec = public_adapter_spec(dataset, tmp_path) + spec['native_runtime'] = selected_binding(dataset) + return spec + + @pytest.mark.parametrize('dataset', DATASETS) def test_admission_receipt_uses_selected_dataset(dataset, tmp_path, captured_native_python_manifest): # Admission-only control derived from BIRD; never used as a dataset scorer. @@ -39,11 +47,11 @@ def test_admission_receipt_uses_selected_dataset(dataset, tmp_path, captured_nat @pytest.mark.parametrize('dataset', DATASETS) -def test_adapter_binds_independent_pin(dataset, tmp_path): - selected = adapter_spec(dataset, tmp_path)['native_runtime'] +def test_adapter_defaults_to_public_source_contract(dataset, tmp_path): + selected = public_adapter_spec(dataset, tmp_path)['native_runtime'] assert selected['dataset'] == dataset - path = ROOT / selected['manifest'] - assert selected['sha256'] == hashlib.sha256(path.read_bytes()).hexdigest() + assert selected['profile'] == 'public-native-local-v1' + assert 'manifest' not in selected and selected['sources'] def selected_binding(dataset): @@ -96,6 +104,7 @@ def authored_spec(dataset, tmp_path): @pytest.mark.parametrize('dataset', DATASETS) +@pytest.mark.skipif(not os.environ.get('HG_DATA_NATIVE_RUNTIMES'), reason='opt-in historical archive control; default public scoring needs no archive') def test_actual_bound_replay_and_input_refusal(dataset, tmp_path): work = tmp_path / 'task' work.mkdir() @@ -134,6 +143,7 @@ def test_actual_bound_replay_and_input_refusal(dataset, tmp_path): @pytest.mark.parametrize('dataset', DATASETS) +@pytest.mark.skipif(not os.environ.get('HG_DATA_NATIVE_RUNTIMES'), reason='opt-in historical archive control; default public scoring needs no archive') def test_parent_and_child_replay_only_load_admitted_bytes(dataset, tmp_path, monkeypatch): monkeypatch.setenv('PYTHONPATH', '/task') monkeypatch.setenv('LD_PRELOAD', '/task/evil.so') @@ -189,6 +199,7 @@ def audit(event,args): 'parent': parent, 'child': child}, sort_keys=True)) +@pytest.mark.skipif(not os.environ.get('HG_DATA_NATIVE_RUNTIMES'), reason='opt-in historical archive control; default public scoring needs no archive') def test_admitted_runtime_can_repeat_readline_startup(tmp_path): (tmp_path / 'verifier').mkdir() with native_runtime.admit(selected_binding('alfworld')) as runtime: diff --git a/tests/quality/test_native_runtime_hp4.py b/tests/quality/test_native_runtime_hp4.py index 84a49b1c3..c468c2715 100644 --- a/tests/quality/test_native_runtime_hp4.py +++ b/tests/quality/test_native_runtime_hp4.py @@ -2,6 +2,7 @@ import hashlib import importlib.util import json +import os from pathlib import Path import sys from types import SimpleNamespace @@ -34,9 +35,10 @@ def find_spec(name, *args, **kwargs): (jre / 'bin').mkdir(parents=True) (jre / 'bin/java').write_bytes(b'authored locator only; never executed') monkeypatch.setattr(scienceworld, 'JRE_STAGED', jre) + monkeypatch.setattr(scienceworld, '_runtime_payload', lambda: package / 'scienceworld.jar') -def adapter_spec(dataset): +def public_adapter_spec(dataset): if dataset == 'scienceworld': from data.datasets import scienceworld # Intentionally unknown authored name: no released task/variation is loaded. @@ -53,11 +55,19 @@ def binding(dataset): return NATIVE_PINS[dataset].copy() +def adapter_spec(dataset): + """Explicit historical route for the sealed-archive regression controls below.""" + spec = public_adapter_spec(dataset) + spec['native_runtime'] = binding(dataset) + return spec + + @pytest.mark.parametrize('dataset', DATASETS) -def test_adapter_binds_independent_pin(dataset, authored_package_locator): - selected = adapter_spec(dataset)['native_runtime'] +def test_adapter_defaults_to_public_source_contract(dataset, authored_package_locator): + selected = public_adapter_spec(dataset)['native_runtime'] assert selected['dataset'] == dataset - assert selected['sha256'] == hashlib.sha256((ROOT / selected['manifest']).read_bytes()).hexdigest() + assert selected['profile'] == 'public-native-local-v1' + assert 'manifest' not in selected and selected['sources'] def test_java_prefers_sealed_home_and_refuses_missing(monkeypatch, tmp_path): @@ -104,6 +114,7 @@ def test_route_manifest_payload_and_sandbox_refusal(dataset, tmp_path, authored_ pytest.fail('modified provider admitted') +@pytest.mark.skipif(not os.environ.get('HG_DATA_NATIVE_RUNTIMES'), reason='opt-in historical archive control; default public scoring needs no archive') def test_chess_actual_bound_check_shadow_zero_and_input_refusal(tmp_path): spec = adapter_spec('chessbench_puzzles') (tmp_path / 'chess.py').write_text('raise RuntimeError("actor package")') @@ -128,6 +139,7 @@ def test_chess_actual_bound_check_shadow_zero_and_input_refusal(tmp_path): 'input_mismatch': bad, 'evidence': evidence}, sort_keys=True)) +@pytest.mark.skipif(not os.environ.get('HG_DATA_NATIVE_RUNTIMES'), reason='opt-in historical archive control; default public scoring needs no archive') def test_scienceworld_actual_check_unknown_authored_task_is_unmeasured(tmp_path): spec = adapter_spec('scienceworld') # This is an intentional no-task failure control, never a measured zero. @@ -150,6 +162,7 @@ def test_scienceworld_actual_check_unknown_authored_task_is_unmeasured(tmp_path) @pytest.mark.parametrize('dataset', DATASETS) +@pytest.mark.skipif(not os.environ.get('HG_DATA_NATIVE_RUNTIMES'), reason='opt-in historical archive control; default public scoring needs no archive') def test_parent_and_child_native_maps_and_lifecycle(dataset, tmp_path, monkeypatch): monkeypatch.setenv('PYTHONPATH', '/task') monkeypatch.setenv('LD_PRELOAD', '/task/evil.so') @@ -222,4 +235,4 @@ def test_chess_declared_verify_map_also_binds_pin(monkeypatch): 'puzzle_id':'authored-map'}} monkeypatch.setattr(dataset._SPEC,'ensure',lambda:[record]) selected=dataset.__getattr__('VERIFY')['authored/chess-map']['native_runtime'] - assert selected==binding('chessbench_puzzles') + assert selected['profile']=='public-native-local-v1' and 'manifest' not in selected diff --git a/tests/quality/test_public_native_scorers.py b/tests/quality/test_public_native_scorers.py new file mode 100644 index 000000000..c6a89d8fb --- /dev/null +++ b/tests/quality/test_public_native_scorers.py @@ -0,0 +1,174 @@ +"""Public scoring uses current-machine dependencies without historical byte archives.""" +from copy import deepcopy +import json +import os +from pathlib import Path +import shutil +import sqlite3 +import stat +import sys +import tempfile + +import pytest + +from data.scaffolds import public_native +from eval.verification import native_runtime, verifier +from harnessgrad import dataset_runtime + +ROOT = Path(__file__).resolve().parents[2] + + +def bird_spec(tmp_path, monkeypatch): + from data.datasets import bird_minidev + dbs=tmp_path/'dbs';(dbs/'authored').mkdir(parents=True) + with sqlite3.connect(dbs/'authored/authored.sqlite') as db: + db.execute('create table items (n int)');db.executemany('insert into items values (?)',[(1,),(2,)]) + monkeypatch.setattr(bird_minidev,'DBS',dbs) + return bird_minidev._verify({'task_id':'authored-public','db_id':'authored','gold_sql':'select n from items'}) + + +def test_all_five_declare_public_execution_recipes(): + for name in public_native.DATASETS: + recipe=dataset_runtime.execution_descriptor(name) + assert recipe and recipe['profile']=='execution' + assert dataset_runtime.execution_descriptor('bird_minidev')['pins']=={} + assert dataset_runtime.execution_descriptor('chessbench_puzzles')['pins']=={'chess':'1.11.2'} + + +def test_default_binding_contains_logic_not_host_bytes(tmp_path,monkeypatch): + spec=bird_spec(tmp_path,monkeypatch);binding=spec['native_runtime'] + assert binding['profile']==public_native.PROFILE and 'manifest' not in binding + assert all('site-packages' not in path and 'libc.so' not in path for path in binding['sources']) + assert spec['scorer_input']['files'] # Task inputs remain bound independently. + assert binding['sha256']==public_native.identity({k:v for k,v in binding.items() if k!='sha256'}) + + +def test_missing_local_dependencies_names_public_preparation_not_archive(tmp_path,monkeypatch): + spec=bird_spec(tmp_path,monkeypatch) + def missing(name): + raise dataset_runtime.DatasetRuntimeUnavailable('prepare_dataset.py --dataset '+name+' --runtime execution') + monkeypatch.setattr(dataset_runtime,'resolve_execution',missing) + def no_archive(*args):raise AssertionError('historical archive selected') + monkeypatch.setattr(native_runtime,'_snapshot',no_archive) + got=verifier._verify(tmp_path,spec,'',sandbox=True,timeout_s=5) + assert got['score'] is None and '--runtime execution' in got['unmeasured'] + assert got['native_runtime']['verified'] is False + + +@pytest.fixture +def local_store(monkeypatch): + if sys.version_info[:2]!=(3,12) or shutil.which('uv') is None or shutil.which('bwrap') is None: + pytest.skip('native control needs local CPython 3.12, uv and bubblewrap') + with tempfile.TemporaryDirectory(prefix='hg-public-native-',dir='/var/tmp') as temporary: + root=Path(temporary) + monkeypatch.setenv('UV_OFFLINE','1');monkeypatch.setenv('UV_CACHE_DIR',str(root/'uv-cache')) + monkeypatch.setenv('HG_DATASET_RUNTIME_ROOT',str(root/'store')) + monkeypatch.setenv('HG_DATA_NATIVE_RUNTIMES',str(root/'absent-historical-archive')) + selected=dataset_runtime.prepare_execution('bird_minidev',python=sys.executable,root=root/'store',timeout_s=60) + yield root,selected + for path in root.rglob('*'): + if path.is_dir() and not path.is_symlink():path.chmod(stat.S_IMODE(path.stat().st_mode)|0o700) + + +@pytest.mark.parametrize('query,expected',[('select n from items',1.0),('select n from items where n=1',0.0)]) +def test_actual_local_sqlite_scoring_without_archive_and_without_actor_imports(tmp_path,monkeypatch,local_store,query,expected): + spec=bird_spec(tmp_path,monkeypatch);work=tmp_path/'task';work.mkdir() + (work/'answer.sql').write_text(query) + for name in ('sqlite3','random','tempfile'):(work/(name+'.py')).write_text('raise RuntimeError("actor shadow imported")') + (work/'data').mkdir();(work/'data/__init__.py').write_text('raise RuntimeError("actor package imported")') + got=verifier._verify(work,spec,'',sandbox=True,timeout_s=30,env={'kind':'files'},task_id='authored-public') + assert got.get('score')==expected,got + assert got['native_runtime']['mechanism']==public_native.PROFILE + assert got['native_runtime']['identity']['execution']['sqlite'] + assert got['evidence']['scorer_input']['verified'] + + +def test_same_logic_accepts_a_second_locally_prepared_runtime(tmp_path,monkeypatch,local_store): + root,first=local_store;spec=bird_spec(tmp_path,monkeypatch) + second=dataset_runtime.prepare_execution('bird_minidev',python=sys.executable,root=root/'other-store',timeout_s=60) + assert first['runtime_id']!=second['runtime_id'] + observed=[] + for store in (root/'store',root/'other-store'): + monkeypatch.setenv('HG_DATASET_RUNTIME_ROOT',str(store)) + with native_runtime.admit(spec['native_runtime']) as runtime:observed.append(deepcopy(runtime.receipt)) + assert observed[0]['sha256']==observed[1]['sha256'] + assert observed[0]['identity']['python_runtime']['runtime_id']!=observed[1]['identity']['python_runtime']['runtime_id'] + assert all(x['verified'] for x in observed) + + +def test_edited_scoring_contract_is_not_silently_executed(tmp_path,monkeypatch): + spec=bird_spec(tmp_path,monkeypatch);spec['native_runtime']['sha256']='0'*64 + got=verifier._verify(tmp_path,spec,'',sandbox=True,timeout_s=5) + assert got['score'] is None and 'invalid public native' in got['unmeasured'] + + +@pytest.mark.parametrize('name',sorted(public_native.DATASETS)) +def test_each_default_scorer_uses_public_source_without_archive_lookup(name,tmp_path,monkeypatch): + from harnessgrad import native_runtime_store + monkeypatch.setattr(native_runtime_store,'load_manifest',lambda *a:pytest.fail('default consulted historical archive')) + if name=='bird_minidev': + spec=bird_spec(tmp_path,monkeypatch) + elif name=='alfworld': + from data.datasets import alfworld + spec=alfworld._verifier({'task_id':'authored-public','stripped_game':'{}'}) + elif name=='jericho': + from data.datasets import jericho + rom=tmp_path/'authored.z3';rom.write_bytes(b'not executed') + spec=jericho._verifier({'task_id':'authored-public','rom':rom.name,'path':rom,'seed':0}) + elif name=='scienceworld': + from data.datasets import scienceworld + jar=tmp_path/'authored.jar';jar.write_bytes(b'not executed') + monkeypatch.setattr(scienceworld,'_runtime_payload',lambda:jar) + monkeypatch.setattr(scienceworld,'JRE_STAGED',tmp_path/'jre') + spec=scienceworld._verifier({'task_id':'authored-public','task_name':'authored','variation':0}) + else: + from data.datasets import chessbench_puzzles + spec=chessbench_puzzles.verify_for({'task_id':'authored-public','gold':{'fen':'authored','moves':[],'puzzle_id':'authored'}}) + assert spec['native_runtime']['profile']==public_native.PROFILE + assert 'manifest' not in spec['native_runtime'] + assert spec['native_runtime']['sources'] + + +def test_startup_timeout_is_unmeasured_not_a_crash(tmp_path,monkeypatch,local_store): + import subprocess + from eval.verification import public_native_runtime + spec=bird_spec(tmp_path,monkeypatch) + def timeout(*args,**kwargs):raise subprocess.TimeoutExpired('authored startup',1) + monkeypatch.setattr(public_native_runtime.PublicRuntime,'run',timeout) + got=verifier._verify(tmp_path,spec,'',sandbox=True,timeout_s=5) + assert got['score'] is None and got['outcome']=='unmeasured' + assert 'timed out' in got['unmeasured'] + + +@pytest.mark.parametrize('answer,expected',[('e7e5 b8c6',1.0),('illegal',0.0)]) +def test_public_chess_in_prepared_local_environment(tmp_path,monkeypatch,answer,expected): + root=os.environ.get('HG_PUBLIC_NATIVE_TEST_ROOT') + if not root:pytest.skip('set HG_PUBLIC_NATIVE_TEST_ROOT to a prepared public Chess execution store') + from data.datasets import chessbench_puzzles + monkeypatch.setenv('HG_DATASET_RUNTIME_ROOT',root) + monkeypatch.setenv('HG_DATA_NATIVE_RUNTIMES',str(tmp_path/'no-historical-archive')) + spec=chessbench_puzzles.verify_for({'task_id':'authored-public-chess','gold':{ + 'fen':'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1', + 'moves':['e2e4','e7e5','g1f3','b8c6','f1b5'],'puzzle_id':'authored','answer':'e7e5 b8c6'}}) + work=tmp_path/'task';work.mkdir();(work/'answer.txt').write_text(answer) + (work/'chess.py').write_text('raise RuntimeError("actor shadow")') + got=verifier._verify(work,spec,'',sandbox=True,timeout_s=30,env={'kind':'files'},task_id='authored-public-chess') + assert got.get('score')==expected,got + assert got['native_runtime']['mechanism']==public_native.PROFILE + assert got['evidence']['gold']=={'redacted':True} + + +def test_input_mutation_still_refuses_scoring(tmp_path,monkeypatch,local_store): + spec=bird_spec(tmp_path,monkeypatch);work=tmp_path/'task';work.mkdir() + (work/'answer.sql').write_text('select n from items') + next(row for row in spec['inputs'] if row['path']=='gold.sql')['content']='select n+1 from items' + got=verifier._verify(work,spec,'',sandbox=True,timeout_s=30,env={'kind':'files'}) + assert got['score'] is None and got.get('unmeasured'), got + + +def test_scoring_code_change_is_detected_after_contract_selection(tmp_path,monkeypatch,local_store): + spec=bird_spec(tmp_path,monkeypatch);binding=spec['native_runtime'] + binding['sources']['check.py']['sha256']='0'*64 + binding['sha256']=public_native.identity({k:v for k,v in binding.items() if k!='sha256'}) + got=verifier._verify(tmp_path,spec,'',sandbox=True,timeout_s=5) + assert got['score'] is None and 'scoring code changed' in got['unmeasured'] diff --git a/tests/quality/test_scienceworld_managed_service.py b/tests/quality/test_scienceworld_managed_service.py index 5e39c75bb..58f429f5b 100644 --- a/tests/quality/test_scienceworld_managed_service.py +++ b/tests/quality/test_scienceworld_managed_service.py @@ -78,8 +78,8 @@ def test_setup_and_verifier_do_not_find_platform_package(tmp_path,monkeypatch): assert spec['task']=={'task_name':'boil','variation':1,'simplifications':''} assert spec['java_sha256']==hashlib.sha256(b'authored java').hexdigest() verify=dataset._verifier(record) - from data.runtimes import NATIVE_PINS - assert verify['native_runtime']==NATIVE_PINS['scienceworld'] + assert verify['native_runtime']['profile']=='public-native-local-v1' + assert 'manifest' not in verify['native_runtime'] assert next(p for p in verify['inputs'] if p['path']=='scienceworld.jar')['from']==str(jar) diff --git a/tests/quality/test_wheel_adapter_resources.py b/tests/quality/test_wheel_adapter_resources.py index b13154334..83da408db 100644 --- a/tests/quality/test_wheel_adapter_resources.py +++ b/tests/quality/test_wheel_adapter_resources.py @@ -30,14 +30,14 @@ 'cooperbench': ['recipes.json', 'hf_mirror/python', 'vendor/LICENSE'], 'deepswe': ['SOURCE.json'], 'jobbench': ['SOURCE.json'], 'swe_bench_pro': ['SOURCE.json'], 'agentdojo': ['cli/dojo', 'execution-runtime.json', 'execution-requirements.lock', - 'source-manifest.json', 'prepared-manifest.json', 'README.md'], 'appworld': ['cli/aw'], 'bird_minidev': ['cli/bird'], + 'source-manifest.json', 'prepared-manifest.json', 'README.md'], 'appworld': ['cli/aw'], 'bird_minidev': ['cli/bird', 'execution-runtime.json', 'scoring-requirements.txt'], 'gaia': ['cli/ask', 'PREPARATION.md'], 'alfworld': ['cli/alf', 'execution-runtime.json', 'execution-requirements.in', 'execution-requirements.txt', 'execution-recipe.md', 'native-source-version-evidence.json'], 'jericho': ['cli/jer', 'cli/jer-relay', 'runtime.json', 'execution-runtime.json', 'requirements.lock', 'LICENSE.upstream', 'RUNTIME_PROVENANCE.json', 'ISOLATION_STATUS.md', 'AUTHORED_STORY_LICENSE', 'NATIVE_CONTROL.md'], - 'chessbench_puzzles': ['runtime.json', 'requirements-load.txt', 'LOADER_RUNTIME.md', + 'chessbench_puzzles': ['runtime.json', 'execution-runtime.json', 'requirements-load.txt', 'LOADER_RUNTIME.md', 'NATIVE_RUNTIME.md'], 'reasoning_gym': ['execution-runtime.json', 'requirements.lock', 'LOADER_RUNTIME.md', 'SOURCE_LICENSE_AUDIT.json', 'SOURCE_LICENSE_AUDIT.md'], diff --git a/tools/build_bird_native_runtime.py b/tools/build_bird_native_runtime.py index 10079efed..b177ef6a1 100644 --- a/tools/build_bird_native_runtime.py +++ b/tools/build_bird_native_runtime.py @@ -1,6 +1,10 @@ #!/usr/bin/env python3 """Capture supported sealed native scorer userlands, without running a dataset. +Historical archive tooling only. Current dataset evaluation uses public native +scoring; prepare local dependencies with tools/prepare_dataset.py --runtime execution. + + The manifest is a reviewed admission pin, not auto-updated during verification. Collection resolution is evidence only; execution sees only the captured files. Portable captures use /opt/hg-python for Python and its standard library. diff --git a/tools/prepare_chessbench_native.py b/tools/prepare_chessbench_native.py index 4c8d24f02..34be762a9 100644 --- a/tools/prepare_chessbench_native.py +++ b/tools/prepare_chessbench_native.py @@ -1,4 +1,9 @@ -"""Prepare/check the independently pinned Chess scorer, never the chess provider.""" +"""Prepare/check the independently pinned historical Chess scorer. + +Historical archive tooling only. Current dataset evaluation uses public native +scoring; prepare local dependencies with tools/prepare_dataset.py --runtime execution. +""" + from __future__ import annotations import argparse import hashlib diff --git a/tools/prepare_native_runtime.py b/tools/prepare_native_runtime.py index 6e0bba0df..af929c5a5 100644 --- a/tools/prepare_native_runtime.py +++ b/tools/prepare_native_runtime.py @@ -1,6 +1,10 @@ #!/usr/bin/env python3 """Import, export or verify reviewed native scorer packages without running tasks. +Historical archive tooling only. Current dataset evaluation uses public native +scoring; prepare local dependencies with tools/prepare_dataset.py --runtime execution. + + The store follows HG_DATA_NATIVE_RUNTIMES, or HG_DATA_ROOT/native-runtimes. Assembly can recover the exact pinned bytes from explicit legacy source ledgers and archives. It never captures the current interpreter as a replacement.