diff --git a/EGM.md b/EGM.md index 6a4918b..1b24963 100644 --- a/EGM.md +++ b/EGM.md @@ -73,7 +73,8 @@ Build and packaging (`CMakeLists.txt`): - [./tools/gen/EGM.md](./tools/gen/EGM.md) - Model generator (`dump_schema.py`, `generate.py`, `check_names.py`). - [./tools/fixtures/EGM.md](./tools/fixtures/EGM.md) - Golden test files written by SPINE's - own writer, plus their expected JSON and `tests/golden_members.inc`. + own writer (the `current_*` set by any tag, the frozen cases by old releases), plus their + expected JSON, `manifest.json` and `tests/golden_members.inc`. - examples: [./examples/EGM.md](./examples/EGM.md) - The `spine-dump` command-line tool. - ci: [./ci/EGM.md](./ci/EGM.md) - the GitHub Actions workflow and `github-ci.sh` (HDF5 1.14 and 1.12 from LCG views on CVMFS), `check_tag_version.sh` (release tag against @@ -112,7 +113,7 @@ Build and packaging (`CMakeLists.txt`): - Temporarily changes HDF5's library-global automatic error printing (`h5::ErrorSilencer`), restoring it afterwards. - Developer tools only: `tools/gen` rewrites the generated model; `tools/fixtures` clones SPINE - tags and rewrites `tests/data/fixtures/` and `tests/golden_members.inc`; `ci/` scripts + commits and rewrites `tests/data/fixtures/` (cases and manifest) and `tests/golden_members.inc`; `ci/` scripts create `build-ci/`, `build-asan/`, `build-tsan/` and `build-sl7/`. - CI: the GitHub Actions workflow builds and tests on GitHub's hosted runners, with read-only access to the repository and no secrets. diff --git a/tests/data/fixtures/incomplete.expected.json b/tests/data/fixtures/current_incomplete.expected.json similarity index 99% rename from tests/data/fixtures/incomplete.expected.json rename to tests/data/fixtures/current_incomplete.expected.json index e2bf66c..d8430c7 100644 --- a/tests/data/fixtures/incomplete.expected.json +++ b/tests/data/fixtures/current_incomplete.expected.json @@ -1,5 +1,5 @@ { - "case": "incomplete", + "case": "current_incomplete", "writer": {"spine_version":"1.4.0","format_version":1,"lite":false,"complete":false}, "layout": "V1", "num_events": 2, diff --git a/tests/data/fixtures/incomplete.h5 b/tests/data/fixtures/current_incomplete.h5 similarity index 100% rename from tests/data/fixtures/incomplete.h5 rename to tests/data/fixtures/current_incomplete.h5 diff --git a/tests/data/fixtures/v1.4.0_v1.expected.json b/tests/data/fixtures/current_v1.expected.json similarity index 99% rename from tests/data/fixtures/v1.4.0_v1.expected.json rename to tests/data/fixtures/current_v1.expected.json index 32da368..37ba42c 100644 --- a/tests/data/fixtures/v1.4.0_v1.expected.json +++ b/tests/data/fixtures/current_v1.expected.json @@ -1,5 +1,5 @@ { - "case": "v1.4.0_v1", + "case": "current_v1", "writer": {"spine_version":"1.4.0","format_version":1,"lite":false,"complete":true}, "layout": "V1", "num_events": 5, diff --git a/tests/data/fixtures/v1.4.0_v1.h5 b/tests/data/fixtures/current_v1.h5 similarity index 100% rename from tests/data/fixtures/v1.4.0_v1.h5 rename to tests/data/fixtures/current_v1.h5 diff --git a/tests/data/fixtures/v1.4.0_v1_lite.expected.json b/tests/data/fixtures/current_v1_lite.expected.json similarity index 99% rename from tests/data/fixtures/v1.4.0_v1_lite.expected.json rename to tests/data/fixtures/current_v1_lite.expected.json index 5cbcf1f..9fd5363 100644 --- a/tests/data/fixtures/v1.4.0_v1_lite.expected.json +++ b/tests/data/fixtures/current_v1_lite.expected.json @@ -1,5 +1,5 @@ { - "case": "v1.4.0_v1_lite", + "case": "current_v1_lite", "writer": {"spine_version":"1.4.0","format_version":1,"lite":true,"complete":true}, "layout": "V1", "num_events": 5, diff --git a/tests/data/fixtures/v1.4.0_v1_lite.h5 b/tests/data/fixtures/current_v1_lite.h5 similarity index 100% rename from tests/data/fixtures/v1.4.0_v1_lite.h5 rename to tests/data/fixtures/current_v1_lite.h5 diff --git a/tests/data/fixtures/v1.4.0_v2.expected.json b/tests/data/fixtures/current_v2.expected.json similarity index 99% rename from tests/data/fixtures/v1.4.0_v2.expected.json rename to tests/data/fixtures/current_v2.expected.json index 813f3a1..a821970 100644 --- a/tests/data/fixtures/v1.4.0_v2.expected.json +++ b/tests/data/fixtures/current_v2.expected.json @@ -1,5 +1,5 @@ { - "case": "v1.4.0_v2", + "case": "current_v2", "writer": {"spine_version":"1.4.0","format_version":2,"lite":false,"complete":true}, "layout": "V2Products", "num_events": 5, diff --git a/tests/data/fixtures/v1.4.0_v2.h5 b/tests/data/fixtures/current_v2.h5 similarity index 100% rename from tests/data/fixtures/v1.4.0_v2.h5 rename to tests/data/fixtures/current_v2.h5 diff --git a/tests/data/fixtures/v1.4.0_v2_fixed_only.expected.json b/tests/data/fixtures/current_v2_fixed_only.expected.json similarity index 99% rename from tests/data/fixtures/v1.4.0_v2_fixed_only.expected.json rename to tests/data/fixtures/current_v2_fixed_only.expected.json index ab91586..0cf08a7 100644 --- a/tests/data/fixtures/v1.4.0_v2_fixed_only.expected.json +++ b/tests/data/fixtures/current_v2_fixed_only.expected.json @@ -1,5 +1,5 @@ { - "case": "v1.4.0_v2_fixed_only", + "case": "current_v2_fixed_only", "writer": {"spine_version":"1.4.0","format_version":2,"lite":false,"complete":true}, "layout": "V2Products", "num_events": 5, diff --git a/tests/data/fixtures/v1.4.0_v2_fixed_only.h5 b/tests/data/fixtures/current_v2_fixed_only.h5 similarity index 100% rename from tests/data/fixtures/v1.4.0_v2_fixed_only.h5 rename to tests/data/fixtures/current_v2_fixed_only.h5 diff --git a/tests/data/fixtures/v1.4.0_v2_lite.expected.json b/tests/data/fixtures/current_v2_lite.expected.json similarity index 99% rename from tests/data/fixtures/v1.4.0_v2_lite.expected.json rename to tests/data/fixtures/current_v2_lite.expected.json index e1e6a54..1f0f1cd 100644 --- a/tests/data/fixtures/v1.4.0_v2_lite.expected.json +++ b/tests/data/fixtures/current_v2_lite.expected.json @@ -1,5 +1,5 @@ { - "case": "v1.4.0_v2_lite", + "case": "current_v2_lite", "writer": {"spine_version":"1.4.0","format_version":2,"lite":true,"complete":true}, "layout": "V2Products", "num_events": 5, diff --git a/tests/data/fixtures/v1.4.0_v2_lite.h5 b/tests/data/fixtures/current_v2_lite.h5 similarity index 100% rename from tests/data/fixtures/v1.4.0_v2_lite.h5 rename to tests/data/fixtures/current_v2_lite.h5 diff --git a/tests/data/fixtures/manifest.json b/tests/data/fixtures/manifest.json new file mode 100644 index 0000000..f556ad4 --- /dev/null +++ b/tests/data/fixtures/manifest.json @@ -0,0 +1,90 @@ +{ + "cases": { + "current_incomplete": { + "derived_from": null, + "expected": "current_incomplete.expected.json", + "h5": "current_incomplete.h5", + "layout": "V1", + "lite": false, + "role": "current", + "sha": "4726e068f17e433e2fec068dc8ee498fa9a0658f", + "tag": "v1.4.0" + }, + "current_v1": { + "derived_from": null, + "expected": "current_v1.expected.json", + "h5": "current_v1.h5", + "layout": "V1", + "lite": false, + "role": "current", + "sha": "4726e068f17e433e2fec068dc8ee498fa9a0658f", + "tag": "v1.4.0" + }, + "current_v1_lite": { + "derived_from": null, + "expected": "current_v1_lite.expected.json", + "h5": "current_v1_lite.h5", + "layout": "V1", + "lite": true, + "role": "current", + "sha": "4726e068f17e433e2fec068dc8ee498fa9a0658f", + "tag": "v1.4.0" + }, + "current_v2": { + "derived_from": null, + "expected": "current_v2.expected.json", + "h5": "current_v2.h5", + "layout": "V2Products", + "lite": false, + "role": "current", + "sha": "4726e068f17e433e2fec068dc8ee498fa9a0658f", + "tag": "v1.4.0" + }, + "current_v2_fixed_only": { + "derived_from": "current_v2", + "expected": "current_v2_fixed_only.expected.json", + "h5": "current_v2_fixed_only.h5", + "layout": "V2Products", + "lite": false, + "role": "current", + "sha": "4726e068f17e433e2fec068dc8ee498fa9a0658f", + "tag": "v1.4.0" + }, + "current_v2_lite": { + "derived_from": null, + "expected": "current_v2_lite.expected.json", + "h5": "current_v2_lite.h5", + "layout": "V2Products", + "lite": true, + "role": "current", + "sha": "4726e068f17e433e2fec068dc8ee498fa9a0658f", + "tag": "v1.4.0" + }, + "v0.11.1_v1": { + "derived_from": null, + "expected": "v0.11.1_v1.expected.json", + "h5": "v0.11.1_v1.h5", + "layout": "V1", + "lite": false, + "role": "frozen", + "sha": "b78a6d64f60ce0be5a24611541df19033620b95a", + "tag": "v0.11.1" + }, + "v2root": { + "derived_from": null, + "expected": "v2root.expected.json", + "h5": "v2root.h5", + "layout": "V2Root", + "lite": false, + "role": "frozen", + "sha": "a9ad6c63ce5532429c26b3374ec73c7ecc2f1cea", + "tag": "v0.16.3" + } + }, + "partial": false, + "spine_sha": "4726e068f17e433e2fec068dc8ee498fa9a0658f", + "spine_tag": "v1.4.0", + "spine_version": "1.4.0", + "substitutions": [], + "warnings": [] +} diff --git a/tests/test_golden.cpp b/tests/test_golden.cpp index 438b589..5337234 100644 --- a/tests/test_golden.cpp +++ b/tests/test_golden.cpp @@ -1,12 +1,18 @@ // Golden tests: every fixture in tests/data/fixtures was written by SPINE's own writer and // decoded independently with h5py (tools/fixtures/decode_raw.py). The core must reproduce the // decoded file, product by product, column by column, row by row. +// +// The cases are the *.expected.json files of the directory: the current set (current_*, written +// by the model's SPINE tag) and the frozen cases (written by older releases, never rewritten). +// tools/fixtures/make_fixtures.py writes them. #include #include +#include #include #include +#include #include #include #include @@ -29,11 +35,6 @@ using nlohmann::json; using spine::ScalarType; using spine::detail::ColumnBuffer; -const std::vector kCases = { - "v1.4.0_v1", "v1.4.0_v2", "v1.4.0_v1_lite", "v1.4.0_v2_lite", - "v1.4.0_v2_fixed_only", "v2root", "v0.11.1_v1", -}; - std::string fixture(const std::string& name, const std::string& extension) { return std::string(SPINE_READER_TEST_DATA) + "/fixtures/" + name + extension; } @@ -44,6 +45,69 @@ json expected(const std::string& name) { return json::parse(in); } +/// One fixture: its name, its role (from the name) and the writer flags its .expected.json holds. +struct Case { + std::string name; + bool current = false; ///< "current_*": written by the model's SPINE tag; otherwise frozen + bool complete = true; ///< info@complete is not false (a missing attribute counts as complete) +}; + +/// Every case of tests/data/fixtures, sorted by name. +const std::vector& all_cases() { + static const std::vector cases = [] { + const std::string suffix = ".expected.json"; + std::vector found; + for (const auto& entry : + std::filesystem::directory_iterator(std::string(SPINE_READER_TEST_DATA) + "/fixtures")) { + const std::string file = entry.path().filename().string(); + if (file.size() <= suffix.size() || + file.compare(file.size() - suffix.size(), suffix.size(), suffix) != 0) { + continue; + } + Case c; + c.name = file.substr(0, file.size() - suffix.size()); + c.current = c.name.rfind("current_", 0) == 0; + const json want = expected(c.name); + const json& complete = want["writer"]["complete"]; + c.complete = !(complete.is_boolean() && !complete.get()); + found.push_back(c); + } + std::sort(found.begin(), found.end(), + [](const Case& a, const Case& b) { return a.name < b.name; }); + return found; + }(); + return cases; +} + +/// The cases a reader opens without allow_incomplete: those whose writer.complete is not false +/// (a null, as in files of SPINE v0.11.1, counts as complete). +std::vector complete_cases() { + std::vector names; + for (const Case& c : all_cases()) { + if (c.complete) names.push_back(c.name); + } + return names; +} + +} // namespace + +TEST_CASE("the fixture directory holds the current set and the frozen cases") { + std::set current, frozen; + for (const Case& c : all_cases()) (c.current ? current : frozen).insert(c.name); + // the current set: the names the other tests and tools/fixtures/make_fixtures.py use + CHECK(current == std::set{"current_incomplete", "current_v1", "current_v1_lite", + "current_v2", "current_v2_fixed_only", + "current_v2_lite"}); + CHECK_FALSE(frozen.empty()); + for (const Case& c : all_cases()) { + INFO("fixture " << c.name); + CHECK(c.complete == (c.name != "current_incomplete")); + CHECK(expected(c.name)["case"].get() == c.name); + } +} + +namespace { + bool same_number(const json& want, double got) { if (want.is_string()) { const std::string s = want.get(); @@ -152,7 +216,7 @@ void check_values(spine::core::ProductSource& product, const json& want, std::ui TEST_CASE("the core reproduces every golden fixture") { std::lock_guard lock(spine::core::hdf5_mutex()); - for (const std::string& name : kCases) { + for (const std::string& name : complete_cases()) { INFO("fixture " << name); const json want = expected(name); auto file = spine::core::FileSource::open(fixture(name, ".h5")); @@ -191,7 +255,7 @@ TEST_CASE("the core reproduces every golden fixture") { TEST_CASE("incomplete files are refused unless allowed") { std::lock_guard lock(spine::core::hdf5_mutex()); - const std::string path = fixture("incomplete", ".h5"); + const std::string path = fixture("current_incomplete", ".h5"); CHECK_THROWS_AS(spine::core::FileSource::open(path), spine::Error); spine::core::OpenOptions options; options.allow_incomplete = true; @@ -304,7 +368,7 @@ const json* by_id(const json& rows, std::int64_t id) { TEST_CASE("the reader fills every member of every model object as the fixtures say") { std::size_t objects_checked = 0; - for (const std::string& name : kCases) { + for (const std::string& name : complete_cases()) { INFO("fixture " << name); const json want = expected(name); spine::Reader reader(fixture(name, ".h5")); @@ -383,7 +447,7 @@ std::vector list_of(const json& row, const char* column) { TEST_CASE("event relations agree with the ids the fixtures hold") { std::size_t resolved = 0, matched = 0, members = 0; - for (const std::string& name : kCases) { + for (const std::string& name : complete_cases()) { INFO("fixture " << name); const json want = expected(name); spine::Reader reader(fixture(name, ".h5")); diff --git a/tests/test_reader.cpp b/tests/test_reader.cpp index 5126b84..3be1874 100644 --- a/tests/test_reader.cpp +++ b/tests/test_reader.cpp @@ -586,12 +586,12 @@ TEST_CASE("strict mode refuses a product that lacks model fields, and only then" ReaderOptions strict; strict.strict = true; - // a complete SPINE v1.4.0 file reads - CHECK_NOTHROW(Reader(fixture_file("v1.4.0_v1.h5"), strict)); - CHECK_NOTHROW(Reader(fixture_file("v1.4.0_v2.h5"), strict)); + // a complete file of the current set reads + CHECK_NOTHROW(Reader(fixture_file("current_v1.h5"), strict)); + CHECK_NOTHROW(Reader(fixture_file("current_v2.h5"), strict)); // the same file written in lite mode lacks index and orig_index; the oldest release lacks more - for (const char* name : {"v1.4.0_v1_lite.h5", "v0.11.1_v1.h5", "v1.4.0_v2_fixed_only.h5"}) { + for (const char* name : {"current_v1_lite.h5", "v0.11.1_v1.h5", "current_v2_fixed_only.h5"}) { CAPTURE(name); CHECK_NOTHROW(Reader(fixture_file(name))); // not strict: fine try { @@ -611,7 +611,7 @@ TEST_CASE("strict mode refuses a product that lacks model fields, and only then" } TEST_CASE("incomplete files are refused unless ReaderOptions::allow_incomplete") { - const std::string path = fixture_file("incomplete.h5"); + const std::string path = fixture_file("current_incomplete.h5"); try { Reader reader(path); FAIL("the file is incomplete"); diff --git a/tests/test_reader_locking.cpp b/tests/test_reader_locking.cpp index 13efb1c..9ac698a 100644 --- a/tests/test_reader_locking.cpp +++ b/tests/test_reader_locking.cpp @@ -65,11 +65,11 @@ std::vector finished_while_locked(const std::vector> } // namespace TEST_CASE("the public operations of a Reader that touch HDF5 wait for the HDF5 lock") { - const std::string path = fixture("v1.4.0_v1.h5"); + const std::string path = fixture("current_v1.h5"); Reader reader(path); auto doomed = std::make_unique(path); // destroyed by an action Reader target(path); // overwritten by an action - Reader source(fixture("v1.4.0_v2.h5")); // moved from by an action + Reader source(fixture("current_v2.h5")); // moved from by an action std::unique_ptr constructed; // built by an action (and so destroyed after) const std::vector finished = finished_while_locked({ diff --git a/tools/EGM.md b/tools/EGM.md index e892f44..7e206ba 100644 --- a/tools/EGM.md +++ b/tools/EGM.md @@ -22,8 +22,11 @@ in the LCG_110 view (`source [./gen/README.md](./gen/README.md). - fixtures: [./fixtures/EGM.md](./fixtures/EGM.md) - `make_fixtures.py` writes `tests/data/fixtures/.h5` with SPINE's real writers (one `spine_worker.py` - subprocess per write, litify or read call, each importing a single SPINE tag; - `PYTHONHASHSEED=0`), decodes them with the SPINE-free `decode_raw.py` into - `.expected.json`, and cross-checks them. `make_member_list.py` writes or - checks `tests/golden_members.inc` from the schema. Legacy notes: + subprocess per write, litify or read call, each importing a single SPINE commit; + `PYTHONHASHSEED=0`): the current set (`current_*`) with any tag, by default the + model's, and the frozen cases with their own releases. It decodes them with the + SPINE-free `decode_raw.py` into `.expected.json`, cross-checks them, and + records each case's SPINE tag and commit in `manifest.json`. `spine_checkout.py` + resolves tags and clones commits. `make_member_list.py` writes or checks + `tests/golden_members.inc` from the schema. Legacy notes: [./fixtures/README.md](./fixtures/README.md). diff --git a/tools/fixtures/EGM.md b/tools/fixtures/EGM.md index bbfbaf6..a935967 100644 --- a/tools/fixtures/EGM.md +++ b/tools/fixtures/EGM.md @@ -4,18 +4,22 @@ Generates and checks the golden HDF5 test data in `tests/data/fixtures/`. Small files are written by SPINE's own `HDF5Writer` (one `spine_worker.py` subprocess per -write, litify or read call; each subprocess imports a single SPINE tag). +write, litify or read call; each subprocess imports a single SPINE commit). They are then decoded by an independent h5py-only decoder into `.expected.json` and cross-checked against each other and against SPINE's -`HDF5Reader`. The directory also generates `tests/golden_members.inc`, the -per-class member list from `schema/`. The hand-written -[README.md](./README.md) is the detailed reference for the cases, the expected -JSON format and the findings about the real writers. This file is the contract -summary; it does not replace that README. +`HDF5Reader`. The cases have two roles: the current set (`current_*`), written by +one tag, by default the model's SPINE tag; and the frozen cases, written by old +releases and never rewritten by a new tag. `manifest.json` records each case's tag +and commit. The directory also generates `tests/golden_members.inc`, the per-class +member list from `schema/`. The hand-written [README.md](./README.md) is the +detailed reference for the cases, the manifest, the expected JSON format and the +findings about the real writers. This file is the contract summary; it does not +replace that README. | file | role | | --- | --- | -| `make_fixtures.py` | driver: clones SPINE tags, runs the workers, decodes, cross-checks, prints sizes | +| `make_fixtures.py` | driver: resolves and clones SPINE commits, runs the workers, decodes, cross-checks, writes the manifest, prints sizes | +| `spine_checkout.py` | `resolve_tag()` and `ensure_clone()`; imports neither h5py nor numpy | | `spine_worker.py` | the only module that imports SPINE; subcommands `write`, `litify`, `read` | | `decode_raw.py` | independent V1 / V2 decoder (h5py + numpy, no SPINE); library and CLI | | `content.py` | the synthetic events as plain Python dicts (no SPINE, no I/O) | @@ -25,42 +29,77 @@ summary; it does not replace that README. ### `make_fixtures.py` (CLI) -- `CASES`: 10 cases (`v1.4.0_v1`, `v1.4.0_v2`, `v1.4.0_v1_lite`, `v1.4.0_v2_lite`, - `v2root`, `v0.11.1_v1`, `v1.4.0_v2_fixed_only`, `incomplete`, and the unshipped - `v1.4.0_exotic_v{1,2}` with `ship=False`). Optional keys: tag, fv, lite, - products, n_events, derive, incomplete, exotic, and ship (defaults to True). -- `LITE_SKIPPED = {index, orig_index, index_adapt, index_g4}`, `OBJECT_LISTS`, - `ENUM_COLUMNS`, `DEFAULT_OUT` (`/tests/data/fixtures`), `DEFAULT_SPINE_REPO` - (`https://github.com/DeepLearnPhysics/spine`). -- Helpers: `ensure_clone`, `run_worker`, `diff_docs`, `layout_free`, `restrict`, - `check_content`, `reader_check`, `main`. +- `CURRENT_CASES`: `current_v1`, `current_v2`, `current_v1_lite`, `current_v2_lite`, + `current_v2_fixed_only` (`derive="current_v2"`), `current_incomplete`, and the + unshipped `current_exotic_v{1,2}` with `ship=False`. They take their tag from `--tag`. +- `FROZEN_CASES`: `v2root` (v0.16.3, `a9ad6c63ce5532429c26b3374ec73c7ecc2f1cea`) and + `v0.11.1_v1` (v0.11.1, `b78a6d64f60ce0be5a24611541df19033620b95a`, + `reader_fallback=True`), with their tag and commit. `CASES` is both. +- Optional case keys: fv, lite, products, n_events, derive, incomplete, exotic, ship + (defaults to True), and for frozen cases tag, sha, reader_fallback. +- `PAIRS` (the cross-checks by role name), `OBJECT_LISTS`, `ENUM_COLUMNS`, + `DEFAULT_OUT` (`/tests/data/fixtures`), `DEFAULT_SPINE_REPO` + (`https://github.com/DeepLearnPhysics/spine`), `MANIFEST` (`manifest.json`), and the + exit codes `EXIT_OK` 0, `EXIT_CHECK` 1, `EXIT_USAGE` 2, `EXIT_TOOLING` 3, + `EXIT_ERROR` 4. +- Helpers: `role`, `committed_schema`, `load_schema`, `lite_dropped`, `run_worker`, + `substitutions_of`, `diff_docs`, `layout_free`, `restrict`, `lite_problems`, + `check_content`, `check_enums`, `reader_check`, `manifest_entry`, `read_manifest`, + `write_manifest`, `merge_manifest`, `parse_args`, `Run`, `manifest_only`, + `write_cases`, `main`. Errors: `UsageError`, `ToolError`, `WorkerError` (carries + the worker's exit `code`). - Flags: - `--out` (default `DEFAULT_OUT`) + - `--tag vX.Y.Z` (default "v" + the `spine_version` of the schema in use) + - `--sha SHA` (40 hex characters; given, the tag is not resolved again) + - `--schema PATH` (default the single committed `schema/spine-*.json`; read for + `lite_dropped`) + - `--frozen` (also write the frozen cases) - `--spine-repo` (default `$SPINE_REPO`, else GitHub) - `--workdir` (default `$SPINE_FIXTURE_WORKDIR`, else `/spine-reader-fixtures`; - holds the `spine_` clones) + holds the `spine_` clones) - `--libver` (default `v110`) - - `--only CASE...` + - `--only CASE...` and `--no-reader-check` (partial runs) - `--with-exotic` - - `--no-reader-check` - `--report FILE` + - `--artifacts-dir DIR` + - `--manifest PATH` (default `/manifest.json`) + - `--manifest-base PATH` (default the committed `tests/data/fixtures/manifest.json`, + read only) + - `--manifest-only` +- Standard output: one JSON line `{spine_tag, spine_sha, cases, partial, failures, + warnings, substitutions, manifest}` (`--manifest-only`: `{manifest, cases, + partial}`). + +### `spine_checkout.py` (library) + +- `resolve_tag(repo, tag) -> sha`: `git ls-remote --tags refs/tags/ + refs/tags/^{}`; the peeled `^{}` commit when listed, else the plain one. Raises + `LsRemoteError` when `ls-remote` fails and `TagNotFound` when the tag is absent. +- `ensure_clone(tag, sha, repo, workdir) -> /src`: clones into + `/spine_` and checks that `HEAD` is `sha`; raises `CheckoutError`. +- `SHA_RE`, and the errors `CheckoutError`, `LsRemoteError`, `TagNotFound` (both + subclasses of `CheckoutError`). ### `spine_worker.py` (CLI, run only by `make_fixtures.run_worker`) - `write`: `--out` (required), `--format-version` (default 1), `--lite`, `--incomplete` (skip `finalize()`), `--n-events`, `--exotic`, - `--products {all,core,min}`, `--libver`. Exits with `SPINE cannot write - format_version ` when the writer predates V2. `--exotic` ignores `--products` and - `--n-events`. + `--products {all,core,min}`, `--libver`, `--configure-enums`. `--exotic` ignores + `--products` and `--n-events`. With `--configure-enums` it passes the tag's checked + enums to `content.configure()` and prints each substitution on stderr as + `SUBSTITUTION `. - `litify`: `--src`, `--out` (required), `--libver`. Runs `litify_hdf5(mode="fixed_only", keys=DEFAULT_LITE_KEYS)`. - `read`: `--file`, `--out` (required). Dumps `spine_version`, `strict_open`, `num_entries`, `reader_version`, `raw`, and `classes` or `classes_error` as JSON. +- Exit codes: 0; 3 (`SpineApiError`: an import fails, the writer lacks a parameter it + needs, or it cannot write the asked `format_version`); 1 any other error. - `CORE_KEYS`, `KEYS` (core + `trigger`, `flashes`, `crthits`), `MIN_KEYS` - (`run_info`, `reco_particles`), `DROPPED`. -- Helpers: `spine_api` (falls back to `spine.io.core.*` for v0.11.x), `make`, - `update`, `build_event`, `exotic_batch`, `apply_libver` (a no-op for `earliest` or an - empty value), `jsonify`. + (`run_info`, `reco_particles`), `ENUMS`, `DROPPED`. +- Helpers: `spine_api` (falls back to `spine.io.core.*` for v0.11.x), `tag_enums`, + `make`, `update`, `build_event`, `exotic_batch`, `apply_libver` (a no-op for + `earliest` or an empty value), `jsonify`. ### `decode_raw.py` (library + CLI) @@ -78,10 +117,15 @@ summary; it does not replace that README. - `build_events()`: the 5 synthetic events, seeded with `random.Random(20261021)`. - `exotic_events()`: 3 events of non-object products, seeded with `random.Random(7)`. +- `configure(enums=None)`: takes `{enum name: {label: value}}` for `ParticleShape`, + `ParticlePID`, `NuCurrentType` and `NuInteractionScheme` (`LABELS`), rebinds the + label constants and rebuilds `PDG` and `MASS`; returns and stores the substitutions + in `SUBSTITUTIONS`. `configure(None)` restores the built-in values. - `NUM_EVENTS = 5`, `BATCHES = ((0, 2), (2, 5))`. -- Enum value constants (shape, pid, current, scheme), `PDG`, `MASS`, `NAN`, `INF`. -- Builders: `reco_particle`, `truth_particle`, `reco_interaction`, - `truth_interaction`. +- Enum value constants (shape, pid, current, scheme; SPINE v1.4.0's by default), + `LABELS`, `PDG`, `MASS`, `NAN`, `INF`, `SUBSTITUTIONS`. +- Builders: `reco_particle` (its `chi2_pid` defaults to `UNKNOWN` at call time), + `truth_particle`, `reco_interaction`, `truth_interaction`. ### `make_member_list.py` (CLI) @@ -93,29 +137,44 @@ summary; it does not replace that README. ## Invariants - SPINE is imported only in `spine_worker.py`. There is one `spine_worker.py` - subprocess per write, litify or read call. Each subprocess imports a single SPINE tag - (`PYTHONPATH=/src`), so SPINE versions are never mixed in one process + subprocess per write, litify or read call. Each subprocess imports a single SPINE + commit (`PYTHONPATH=/src`), so SPINE versions are never mixed in one process [make_fixtures.py:run_worker]. +- A clone is named after its commit (`spine_`), and `ensure_clone` refuses a + checkout whose `HEAD` is another commit, so a moved tag never reuses old code + [spine_checkout.py:ensure_clone]. Frozen cases are checked out at their recorded + commits. +- The role of a case is its name: `current_*` or frozen [make_fixtures.py:role]. The + current set is written by `--tag`; frozen cases only with `--frozen` + [make_fixtures.py:write_cases]. - Workers run with `PYTHONHASHSEED=0` (V1 `events` member order follows `set()` - order) and `PYTHONDONTWRITEBYTECODE=1`. A nonzero worker exit raises - `RuntimeError` [make_fixtures.py:run_worker]. -- Cases with `derive` set (`v1.4.0_v2_fixed_only` is litified from `v1.4.0_v2`) + order) and `PYTHONDONTWRITEBYTECODE=1`. A nonzero worker exit raises `WorkerError` + with the worker's exit code [make_fixtures.py:run_worker]. +- Exit codes of `make_fixtures.py` [make_fixtures.py:main]: 0 success, warnings + allowed; 1 a check failed; 2 a usage error (`UsageError`, or argparse); 3 a `write` + or `litify` worker exited 3; 4 any other error (another worker exit, a + `CheckoutError`, a missing h5py, an unexpected exception). A failed `read` worker + is a check failure (1), not a tooling error. +- Cases with `derive` set (`current_v2_fixed_only` is litified from `current_v2`) are sorted after every other case, so in a full run their source is built first. - With `--only`, the source `/.h5` must already exist [make_fixtures.py:main]. + With `--only`, the source `/.h5` must already exist [make_fixtures.py:write_cases]. - Cross-checks are not Python `assert`s. Every failure is collected, and the run - ends with `SystemExit(" cross-check(s) failed")` [make_fixtures.py:main]. - The checks: + ends with exit code 1 [make_fixtures.py:write_cases]. The checks: + - Each written file has the asked `format_version`, and `info@complete` is false + exactly for the case written incomplete. - `decode(v1)` equals `decode(v2)`, full and lite, after `layout_free` removes `case`, `layout` and `writer.format_version`. - - lite equals full minus `LITE_SKIPPED`, and lite drops nothing else. + - lite equals full minus, per object product, exactly its class's `lite_dropped` + fields in the schema [lite_problems]. - `fixed_only` equals full minus the pooled columns. - - The decoded JSON equals what SPINE's `HDF5Reader` returns. If the writer tag's - reader fails, the check notes it and retries with the v1.4.0 reader. It fails - (`: no SPINE reader could read it`) only if both fail. `--no-reader-check` + - The decoded JSON equals what SPINE's `HDF5Reader` of the writing tag returns. Only + a case with `reader_fallback` (`v0.11.1_v1`) retries with the current tag's reader; + for every other case a failing reader is a check failure. `--no-reader-check` skips this check, and with it the `--report` file. - A file with `writer.complete == False` must fail the strict open with `RuntimeError`; every other file must open. - - A check runs only when both of its cases were built (`--only` may skip some). + - A check whose pair of cases is missing is a failure, unless the run is partial + (`--only` or `--no-reader-check`), where it is skipped. - Content conventions (every case except the exotic cases and `products=min`) [make_fixtures.py:check_content]: - ids equal list positions. @@ -123,9 +182,21 @@ summary; it does not replace that README. references point back to each other. - `match_overlaps` has the same length as `match_ids` and is sorted descending. - Coverage is checked only for the full cases (products `all`, 5 events, not - derived): every `ENUM_COLUMNS` label, both bools, `nan`/`inf`/`-inf`, a - non-ASCII string, empty and non-empty variable arrays, an empty event, and a - reco interaction without particles [make_fixtures.py:check_content]. + derived): both bools, `nan`/`inf`/`-inf`, a non-ASCII string, empty and non-empty + variable arrays, an empty event, and a reco interaction without particles + [make_fixtures.py:check_content]. +- Enums, current set only [make_fixtures.py:check_enums]: a stored value of an + `ENUM_COLUMNS` column that the file's enum does not name is a failure; in the full + cases a label no row stores (`unused_label`) and an enum column stored as a plain + integer (`not_an_enum`) are warnings. The current set is written with the writing + tag's enum values; a label the tag lacks becomes `UNKNOWN` and is a substitution + [spine_worker.py:cmd_write, content.py:configure]. Frozen cases use the built-in + values and are not enum-checked. +- Manifest [make_fixtures.py:merge_manifest]: the frozen entries of `--manifest-base` + are carried over, and in a partial run its current entries too; the entries of the + cases the run wrote replace theirs. Only shipped cases have entries. The top-level + keys describe the run when it wrote a current case. `h5` and `expected` are file + names. Warnings and substitutions are `{kind, name, detail}`. - The decoder validates the layout and raises `DecodeError` on any violation, including [decode_raw.py]: - V1 regions are a selection with no points, or one whole-row hyperslab block, @@ -145,6 +216,8 @@ summary; it does not replace that README. - Products and row keys are sorted. - `content.py` floats are mostly on dyadic grids (exact in float32 and float64). `build_events` asserts `len(events) == NUM_EVENTS` [content.py:_r, build_events]. + Without `configure()`, or after `configure(None)`, the content is that of SPINE + v1.4.0's enum values. - `make_member_list.py` (repository found from its own location) requires exactly one `schema/spine-*.json` and otherwise exits with `make_member_list: expected one schema/spine-*.json, found N` [make_member_list.py:render]. `tools/gen/generate.py` @@ -152,24 +225,32 @@ summary; it does not replace that README. ## Side Effects -- Writes `/.h5` and `/.expected.json` for the shipped cases. - Unshipped exotic cases go to a `tempfile.mkdtemp(prefix="spine-fixtures-")` - scratch directory, which is removed at the end, unless `--with-exotic` is given. - Writes `--report` if given [make_fixtures.py:main]. -- Runs `git clone -q --branch [--depth 1] /spine_` for a - missing clone. This needs the network when `--spine-repo` is a URL - [make_fixtures.py:ensure_clone]. +- Writes `/.h5` and `/.expected.json` for the shipped cases, + and the merged manifest to `--manifest`. Unshipped exotic cases go to a + `tempfile.mkdtemp(prefix="spine-fixtures-")` scratch directory, which is removed at + the end, unless `--with-exotic` (then `--out`) or `--artifacts-dir` (then + `/unshipped/`, kept) is given. `--artifacts-dir` also keeps the reader output + (`/reader/`) and the worker logs (`/logs/`). Writes `--report` if given + [make_fixtures.py:write_cases]. +- `--manifest-only` writes only the manifest [make_fixtures.py:manifest_only]. +- Runs `git ls-remote --tags` to resolve a tag without `--sha` + [spine_checkout.py:resolve_tag], and `git clone -q --branch [--depth 1] + /spine_` (through a `.partial-` folder, then renamed) for a + missing clone. These need the network when `--spine-repo` is a URL + [spine_checkout.py:ensure_clone]. - Runs `nice -n 10 python spine_worker.py ...` subprocesses with a copied environment [make_fixtures.py:run_worker]. -- All progress, ok/FAIL lines and the size summary go to stderr - [make_fixtures.py:log]. +- Progress, ok/FAIL/skip/note lines, warnings and the size summary go to stderr; + the one-line JSON summary goes to stdout [make_fixtures.py:log, write_cases]. - `spine_worker.py write` deletes and rewrites `--out`. Inside the worker, `apply_libver` monkeypatches `h5py.File.__init__` to default `libver`. Fields - that an old SPINE lacks are dropped and reported on stderr. + that an old SPINE lacks are dropped and reported on stderr; so are the enum + substitutions. - `decode_raw.py` opens files read-only and writes JSON to `-o`, or to stdout. - `make_member_list.py` writes `tests/golden_members.inc`, or with `--check` prints `stale: ...` and exits 1. -- `content.py` has no I/O and does not touch the global RNG. +- `content.py` has no I/O and does not touch the global RNG; `configure()` rebinds + its module-level constants. ## Verification @@ -181,5 +262,7 @@ summary; it does not replace that README. python3 tools/fixtures/make_member_list.py --check ``` -- Regenerate and re-check the fixtures (clones SPINE tags): - `python3 tools/fixtures/make_fixtures.py`. +- Regenerate and re-check the fixtures (clones SPINE commits): + `python3 tools/fixtures/make_fixtures.py [--frozen]`. With the model's tag, every + `.expected.json` and the manifest come out byte-identical (the `.h5` bytes carry + timestamps). diff --git a/tools/fixtures/README.md b/tools/fixtures/README.md index abd5870..4c4c3bf 100644 --- a/tools/fixtures/README.md +++ b/tools/fixtures/README.md @@ -2,37 +2,115 @@ `tests/data/fixtures/.h5` are small HDF5 files written by **SPINE's own `HDF5Writer`**; `tests/data/fixtures/.expected.json` is what the files contain, decoded by -`decode_raw.py`, which uses plain h5py and nothing from SPINE. +`decode_raw.py`, which uses plain h5py and nothing from SPINE. `tests/data/fixtures/manifest.json` +records, for people, the SPINE tag and commit that wrote each case; no test reads it. | file | role | | --- | --- | -| `make_fixtures.py` | writes the cases, decodes them, runs every cross-check (failures are collected; the run exits non-zero at the end), prints sizes | +| `make_fixtures.py` | writes the cases, decodes them, runs every cross-check (failures are collected; the run exits non-zero at the end), writes the manifest, prints sizes | +| `spine_checkout.py` | resolves a SPINE tag to its commit (`git ls-remote`) and clones that commit; no h5py or numpy | | `decode_raw.py` | independent decoder: V1 (region references), V2 (offsets and pools); also a CLI | | `spine_worker.py` | the only code that imports SPINE: builds objects, runs the writer / `litify_hdf5` / `HDF5Reader` of one SPINE version | | `content.py` | the synthetic events as plain Python (no SPINE import) | | `make_member_list.py` | writes `tests/golden_members.inc`, the member names of every model class, from `schema/`; `--check` is a ctest | +## Two roles: the current set and the frozen cases + +* The **current set**, the cases named `current_*`, is written by **one SPINE tag**: by default the + model's SPINE tag ("v" + the `spine_version` of the committed `schema/spine-*.json`), or the tag + given with `--tag`. When the model moves to a new SPINE tag, the whole current set is rewritten + by that tag. +* The **frozen cases** keep the old release that wrote them and are never rewritten by a new tag: + `v2root` (SPINE v0.16.3, commit `a9ad6c63`) and `v0.11.1_v1` (SPINE v0.11.1, commit `b78a6d64`, + the oldest supported writer). `--frozen` writes them again, to reproduce how they were made; it + checks out the recorded commits. +* The golden test (`tests/test_golden.cpp`) lists `tests/data/fixtures/*.expected.json`, reads each + case's flags from the file itself, and takes its role from its name. The tests that open every + case skip a case whose `writer.complete` is `false` (`current_incomplete`); `null`, as in + `v0.11.1_v1`, counts as complete. + +**What counts as a change in content.** The decoded document includes the writer's SPINE version, +which counts as content; only the case name is ignored. After a run with a tag whose content +changed, the whole current set is committed, rewritten by that tag. A tag that changes nothing +commits nothing. Frozen cases are never rewritten. + ## Regenerating ```bash source /cvmfs/sft.cern.ch/lcg/views/LCG_110/x86_64-el9-gcc13-opt/setup.sh # python3 + numpy + h5py + yaml + scipy -python3 tools/fixtures/make_fixtures.py # clones the SPINE tags it needs -# offline / with a local SPINE repository and checkouts: +python3 tools/fixtures/make_fixtures.py # the current set, with the model's tag +python3 tools/fixtures/make_fixtures.py --frozen # ... and the frozen cases, with their own releases +python3 tools/fixtures/make_fixtures.py --tag v1.2.3 --out /tmp/f # another tag, outside the repository +python3 tools/fixtures/make_fixtures.py --tag v1.4.1 --sha --schema +# offline / with a local SPINE repository and clones: python3 tools/fixtures/make_fixtures.py --spine-repo /path/to/spine --workdir /path/to/clones -python3 tools/fixtures/make_fixtures.py --only v1.4.0_v1 v1.4.0_v2 # a subset (some cross-checks skip) -python3 tools/fixtures/decode_raw.py FILE.h5 --case NAME [--lite] -o OUT.json # decode any SPINE file +python3 tools/fixtures/make_fixtures.py --only current_v1 current_v2 # a partial run (see below) +python3 tools/fixtures/make_fixtures.py --manifest-only --tag v1.4.0 --sha # record existing files +python3 tools/fixtures/decode_raw.py FILE.h5 --case NAME [--lite] -o OUT.json # decode any SPINE file ``` +* **Tags and commits.** `--tag vX.Y.Z` names the tag that writes the current set; `--sha` gives its + commit, which is then not resolved again. Without `--sha`, `spine_checkout.resolve_tag()` runs + `git ls-remote --tags` and takes the peeled `^{}` commit of an annotated tag, or the commit of a + lightweight one. Each commit is cloned once into `--workdir`, in a folder `spine_` named + after the commit (`git clone --branch `, then a check that `HEAD` is that commit), so a tag + moved to another commit never reuses old code. Missing clones come from `--spine-repo` (default + GitHub). +* **`--schema PATH`** (default the committed `schema/spine-*.json`) is read for `lite_dropped`, the + fields lite mode drops, class by class. When the schema's `spine_version` is not the current + set's tag, the run warns (`schema_tag_mismatch`). * SPINE needs **no torch / larcv / MinkowskiEngine** for any of this. It is imported as - `PYTHONPATH=/src`, one subprocess per SPINE version, so never mix versions in one process. -* SPINE versions needed: **v1.4.0** (a, b, c, f, g), **v0.16.3** (d), **v0.11.1** (e). - `--workdir` holds the clones `spine_` (`git clone --branch `); missing ones are cloned from - `--spine-repo` (default GitHub). The v1.4.0 reader also reads the v0.11.1 file (see Surprises). + `PYTHONPATH=/src`, one subprocess per SPINE commit, so never mix versions in one process. * The files are made with the HDF5 file-format bound `--libver v110` (see Sizes). `--libver earliest` gives h5py's default format. * `PYTHONHASHSEED=0` is set for the workers: V1 `events` member order follows Python `set` order. `.h5` bytes are still not reproducible (object-header timestamps), the `.expected.json` are. -* `--with-exotic` also writes `v1.4.0_exotic_v{1,2}` (not shipped, see below). +* `--with-exotic` also writes `current_exotic_v{1,2}` to `--out` (not shipped, see below). +* `--artifacts-dir DIR` keeps what a run otherwise deletes: the unshipped cases' `.h5` and decoded + JSON (`DIR/unshipped/`), SPINE's reader output (`DIR/reader/..json`) and the worker + logs (`DIR/logs/..log`). +* **Partial runs.** `--only CASE...` and `--no-reader-check` are for development. A run that uses + either is marked `partial: true` in the manifest and on standard output. In a partial run a + cross-check whose pair of cases was not written is skipped; in a full run it is a failure. + `--only` names frozen cases only together with `--frozen`. +* **Standard output** is one JSON line: `spine_tag`, `spine_sha`, `cases`, `partial`, `failures`, + `warnings`, `substitutions` and `manifest`. Everything else (progress, `ok`/`FAIL` lines, notes, + sizes) goes to standard error. +* **Exit codes:** 0 success, warnings allowed; 1 a check failed (a cross-check, the content checks, + or the comparison with SPINE's own reader); 2 a usage error; 3 a worker could not drive SPINE's + API (an import, a writer parameter, or a `format_version` the writer cannot write); 4 any other + error (a clone, a worker that failed otherwise while writing or litifying, an exception). + +## The manifest + +`manifest.json` holds, at the top, the run that wrote the current set (`spine_tag`, `spine_sha`, +`spine_version`, `partial`, `warnings`, `substitutions`), and under `cases` one entry per shipped +case: `role` (`current` or `frozen`), `tag`, `sha`, `layout`, `lite`, `derived_from`, and the file +names `h5` and `expected`. Warnings and substitutions have the shape `{kind, name, detail}`. + +* It was created once by `--manifest-only`, which records the files already in `--out` without + writing any: each case's flags come from its `.expected.json`, its tag and commit from `--tag` and + `--sha` (current set) or from the frozen table. +* Every later run merges. `--manifest-base` (default the committed + `tests/data/fixtures/manifest.json`, read only) gives the entries carried over: those of the + frozen cases, and in a partial run also those of the current cases the run did not write. The + entries of the cases the run wrote replace theirs. The result goes to `--manifest` (default + `/manifest.json`), so a run into another `--out` keeps the frozen entries too. + +## Enum values come from the writing tag (current set only) + +The worker writes the current set with `--configure-enums`: it gives `content.configure()` the +members of `ParticleShape`, `ParticlePID`, `NuCurrentType` and `NuInteractionScheme` at the writing +tag, from `spine.constants.enums`. A label the content writes that the tag lacks is written as +`UNKNOWN` (-1 in all four, from v1.2.1 on) and listed as a substitution: SPINE v1.2.1 and v1.2.2, for +instance, have no `GIBUU`, `NUWRO` or `NEUT` scheme. After substitution: + +* a value a current case stores that the file's enum does not name is a **failure**; +* a label the file defines that no row stores is a **warning** (`unused_label`), as is an enum + column stored as a plain integer (`not_an_enum`). + +The frozen cases keep the built-in values (SPINE v1.4.0's), so `--frozen` reproduces `v2root`, which +stores `interaction_scheme` values 2 and 3 under a three-label enum. Their enums are not checked. ## Cases @@ -43,20 +121,22 @@ product (`RunInfo`, `ImageMeta3D`, `Trigger`, `Flash`, `CRTHit`, `Reco/TruthPart `index` is a scalar `array` product. Products with `source_file_*` provenance are not written (no `file_index` in the batch): nothing in the CAFMakers reads them. -| case | writer | layout | products | events | bytes (.h5 / .json) | exercises | +The sizes are those of the current set written by v1.4.0. + +| case | writer | layout | products | events | bytes at v1.4.0 (.h5 / .json) | exercises | | --- | --- | --- | --- | --- | --- | --- | -| `v1.4.0_v1` | v1.4.0, `format_version=1` | V1 | all 10 | 5 | 171,590 / 97,803 | region references + VLEN compound members, SEL_NONE regions, enums, bools, strings, `info@complete` | -| `v1.4.0_v2` | v1.4.0, `format_version=2` | V2Products | all 10 | 5 | 209,665 / 97,811 | `/products`, `event_offsets`, `fixed` + `variables/pool_i` + `_var_offsets_i`; **same objects as `v1.4.0_v1`** | -| `v1.4.0_v1_lite` | v1.4.0, `lite=True` | V1 | all 10 | 5 | 162,000 / 91,861 | columns missing (`index`, `orig_index`, `index_adapt`, `index_g4`) while derived `size`... is still stored | -| `v1.4.0_v2_lite` | v1.4.0, `lite=True` | V2Products | all 10 | 5 | 207,105 / 91,869 | same, and V2 pool **renumbering** (a pool changes its dtype between full and lite) | -| `v2root` | **v0.16.3**, `format_version=2` | V2Root | core 6 | 3 | 167,792 / 66,551 | V2 with the product groups at the file root (no `/products`), `info@format_version=2` | -| `v0.11.1_v1` | **v0.11.1** | V1 | core 6 | 3 | 135,487 / 62,916 | oldest supported release: no `format_version`/`complete`/`spine_version`/`format` attributes, fewer fields (see Surprises), different enum labels and one different dtype | -| `v1.4.0_v2_fixed_only` | `litify_hdf5(v1.4.0_v2, mode="fixed_only")` | V2Products | default 6 + `index` | 5 | 104,266 / 71,615 | no pools at all: empty `variables` groups, no `_var_offsets_*`, no strings (`units`, `topology`), no `match_ids`...; `info@litified` | -| `incomplete` | v1.4.0, V1, never finalized | V1 | `run_info`, `reco_particles` | 2 | 52,294 / 15,982 | `info@complete = False` (the state SPINE leaves while writing) | +| `current_v1` | current tag, `format_version=1` | V1 | all 10 | 5 | 171,590 / 97,804 | region references + VLEN compound members, SEL_NONE regions, enums, bools, strings, `info@complete` | +| `current_v2` | current tag, `format_version=2` | V2Products | all 10 | 5 | 209,665 / 97,812 | `/products`, `event_offsets`, `fixed` + `variables/pool_i` + `_var_offsets_i`; **same objects as `current_v1`** | +| `current_v1_lite` | current tag, `lite=True` | V1 | all 10 | 5 | 162,000 / 91,862 | columns missing (`index`, `orig_index`, `index_adapt`, `index_g4`) while derived `size`... is still stored | +| `current_v2_lite` | current tag, `lite=True` | V2Products | all 10 | 5 | 207,105 / 91,870 | same, and V2 pool **renumbering** (a pool changes its dtype between full and lite) | +| `current_v2_fixed_only` | `litify_hdf5(current_v2, mode="fixed_only")` | V2Products | default 6 + `index` | 5 | 104,266 / 71,616 | no pools at all: empty `variables` groups, no `_var_offsets_*`, no strings (`units`, `topology`), no `match_ids`...; `info@litified` | +| `current_incomplete` | current tag, V1, never finalized | V1 | `run_info`, `reco_particles` | 2 | 52,294 / 15,990 | `info@complete = False` (the state SPINE leaves while writing) | +| `v2root` (frozen) | **v0.16.3**, `format_version=2` | V2Root | core 6 | 3 | 167,792 / 66,551 | V2 with the product groups at the file root (no `/products`), `info@format_version=2` | +| `v0.11.1_v1` (frozen) | **v0.11.1** | V1 | core 6 | 3 | 135,487 / 62,916 | oldest supported release: no `format_version`/`complete`/`spine_version`/`format` attributes, fewer fields (see Surprises), different enum labels and one different dtype | (10 products = `index`, `run_info`, `meta`, `trigger`, `reco_particles`, `reco_interactions`, `truth_particles`, `truth_interactions`, `flashes`, `crthits`. "core 6" = the six `run_info`, `meta`, `reco_*`, `truth_*` products -plus `index`.) Total shipped: 1,806,607 bytes. +plus `index`.) Total shipped at v1.4.0: 1,806,620 bytes, plus the manifest. `v2root` and `v0.11.1_v1` contain the first 3 events of the 5 below (same content, fields the old SPINE does not have are dropped by `spine_worker.py`: it reports them on stderr). @@ -71,24 +151,24 @@ Events (writer called twice: events 0-1, then 2-4, so appends and absolute offse | 3 | 2 : 1 + 2 | 1 : 3 | 3 | 0 | DELTA/LOWE/GHOST shapes, KAON, `length=+inf`, `start_dedx=-inf`, scheme NEUT | | 4 | 2 : 1 + **0** | 1 : 1 | 0 | 1 | an interaction without particles (all its arrays empty, `vertex` NaN), scheme UNKNOWN | -`make_fixtures.py` checks that the full cases contain every label of `shape`, `pid`, `chi2_pid`, `current_type`, -`interaction_scheme`, both bool values, `nan`/`inf`/`-inf`, non-ASCII strings, empty and non-empty variable arrays, -an event with no object at all, an interaction without particles, and that ids, `interaction_id`, `particle_ids` and -`match_ids` obey the conventions above. Floats are mostly on a dyadic grid (exact in float32 and float64, short in -JSON); `match_overlaps`, `pid_scores`, some energies and every *derived* float (`ke`, `p`, `momentum`, sums...) -are deliberately not. +`make_fixtures.py` checks that the full cases contain both bool values, `nan`/`inf`/`-inf`, non-ASCII strings, +empty and non-empty variable arrays, an event with no object at all, an interaction without particles, and that +ids, `interaction_id`, `particle_ids` and `match_ids` obey the conventions above; the enum labels follow the rule +of the previous section. Floats are mostly on a dyadic grid (exact in float32 and float64, short in JSON); +`match_overlaps`, `pid_scores`, some energies and every *derived* float (`ke`, `p`, `momentum`, sums...) are +deliberately not. -### Not shipped, but generated and checked: `v1.4.0_exotic_v{1,2}` +### Not shipped, but generated and checked: `current_exotic_v{1,2}` Non-object products (a `string`, a 2-D `array`, a `list`, a `multi_list` product, with empty events) written by -v1.4.0 in both layouts. They exist so that `decode_raw.py` is tested on every product kind (V1 `list`/`multi_list` -groups of region references, V2 `string`/`list`/`multi_list`); the CAF products never use them, and they cost -190 kB, so they are written to a temporary directory unless `--with-exotic` is given. +the current tag in both layouts. They exist so that `decode_raw.py` is tested on every product kind (V1 +`list`/`multi_list` groups of region references, V2 `string`/`list`/`multi_list`); the CAF products never use +them, and they cost 190 kB at v1.4.0, so they are written to a scratch directory unless `--with-exotic` is given. ## Expected JSON format ```json -{ "case": "v1.4.0_v1", +{ "case": "current_v1", "writer": {"spine_version": "1.4.0", "format_version": 1, "lite": false, "complete": true}, "layout": "V1", // "V1" | "V2Root" | "V2Products" "num_events": 5, @@ -129,29 +209,40 @@ groups of region references, V2 `string`/`list`/`multi_list`); the CAF products column of each `fixed` row, and the pool spans are checked to be contiguous and in range. Products live in `/products` if it exists, else at the root (v0.16.x). -## Cross-checks (run by `make_fixtures.py`; failures make it exit non-zero) - -1. `decode(v1.4.0_v1) == decode(v1.4.0_v2)` and `decode(v1.4.0_v1_lite) == decode(v1.4.0_v2_lite)`: **no - difference at all** except `case`, `layout` and `writer.format_version`. (No legitimate difference was found: - the column types, enums, empty-event rows and every value are identical.) -2. `lite` = `full` minus exactly `index`, `orig_index` (reco and truth) and `index_adapt`, `index_g4` (truth). -3. `fixed_only` = the full V2 file restricted to its 7 products and with every pooled column removed. -4. `expected == SPINE's own HDF5Reader` of the writing version (`build_classes=False`, i.e. the stored values) for - every product, column and event of every case: 2745 / 2745 / 2637 / 2637 / 1619 / 1537 / 1999 / 393 values - (the v0.11.1 file with the v1.4.0 reader, see below). The reader adds only its administrative entries - (`index`, `file_index`, `file_entry_index`, `source_file_*`). +## Cross-checks (run by `make_fixtures.py`; failures make it exit 1) + +They key on role names, so they run for every tag. + +1. `decode(current_v1) == decode(current_v2)` and `decode(current_v1_lite) == decode(current_v2_lite)`: **no + difference at all** except `case`, `layout` and `writer.format_version`. (At v1.4.0 no legitimate difference + was found: the column types, enums, empty-event rows and every value are identical.) +2. `current_v1_lite` = `current_v1` minus, in each object product, exactly the fields its class's + `lite_dropped` lists in the schema (at v1.4.0: `index`, `orig_index` of the reco and truth classes, plus + `index_adapt`, `index_g4` of the truth classes). A lite file that drops more, or less, fails. +3. `current_v2_fixed_only` = `current_v2` restricted to its 7 products and with every pooled column removed. +4. `expected == SPINE's own HDF5Reader` of the writing tag (`build_classes=False`, i.e. the stored values) for + every product, column and event of every case: at v1.4.0, 2745 / 2745 / 2637 / 2637 / 1999 / 393 values for + the current set and 1619 / 1537 for `v2root` / `v0.11.1_v1`. The reader adds only its administrative entries + (`index`, `file_index`, `file_entry_index`, `source_file_*`). Only `v0.11.1_v1` falls back to another reader, + the current tag's (see below); for every other case a reader that fails, or disagrees, is a check failure. 5. The default `HDF5Reader` accepts every complete file and raises `RuntimeError("... is marked incomplete")` on - `incomplete.h5`. -6. Content conventions and coverage (see Cases). + `current_incomplete.h5`. +6. Each written file has the `format_version` asked for, and `info@complete` is false only for + `current_incomplete`. +7. Content conventions and coverage (see Cases), and the enum rule of the current set. + +A cross-check whose pair of cases is missing is a failure, unless the run is partial. ## Surprises and findings about the real writers +These were found with the current set written by v1.4.0 and the two frozen releases. + * **Sizes.** SPINE's layouts are one chunk per dataset (h5py auto-chunking: 6-21 KB for each `fixed`, 4 KB for each int32/float32 pool, 8 KB for `index/values` and `events`), so the size is set by the number of datasets, not by the events: a 5-event V2 file with 10 products is 306 kB with h5py's default file format, 210 kB with `libver=v110` (V1: 195 -> 172 kB). V2 is *larger* than V1 for small files. `--libver v110` changes only the HDF5 file-format bound (compact groups, extensible-array chunk indexes; needs libhdf5 >= 1.10); no SPINE logic, dtype, attribute or chunk shape differs. - Even so `v1.4.0_v2` (209,665 B) and `v1.4.0_v2_lite` (207,105 B) stay just above 200 kB. + Even so, at v1.4.0, `current_v2` (209,665 B) and `current_v2_lite` (207,105 B) stay just above 200 kB. * **Empty events.** V1: the region reference has `H5S_SEL_NONE` (0 points), not a zero-length hyperslab. V2: equal consecutive offsets (`event_offsets = [0 5 7 7 10 11]`). Scalar products (`run_info`, `meta`, `index`) are never empty. Empty lists need `ObjectList(objs, default=Cls())` in SPINE or the writer cannot infer the class. @@ -179,7 +270,8 @@ groups of region references, V2 `string`/`list`/`multi_list`); the CAF products * **v0.11.1**: `info` carries only `cfg` and `version` (a group; `format_version`, `complete`, `spine_version` and `format` appear later). The v0.11.1 `HDF5Reader` **cannot read its own file** (`AttributeError: module 'spine.data' has no attribute 'ImageMeta3D'`: the writer stores `class_name="ImageMeta3D"`, the reader resolves - it in `spine.data`, which only exports `Meta`); the v1.4.0 reader reads it (used for the cross-check). + it in `spine.data`, which only exports `Meta`); the current tag's reader (v1.4.0 so far) reads it, and is used + for the cross-check of that case only. * **Incomplete files**: `with HDF5Writer(...)` calls `finalize()`; skipping it leaves `complete=False`. SPINE's reader refuses such a file unless `ignore_incomplete=True`. A genuinely interrupted V2 file (products longer than `events`) cannot be made reproducibly with the real writer and was not hand-crafted. diff --git a/tools/fixtures/content.py b/tools/fixtures/content.py index 8da2eb8..25c86fb 100644 --- a/tools/fixtures/content.py +++ b/tools/fixtures/content.py @@ -19,6 +19,10 @@ ``interaction_scheme`` occurs in the data, both bool values occur, there are NaN / +inf / -inf values, empty variable arrays, a particle-less event, an interaction without particles and non-ASCII strings (2- and 3-byte UTF-8). + +The enum values are SPINE v1.4.0's unless ``configure()`` is given another tag's +enums: the current cases are written with the enums of their writing tag, the +frozen cases with the built-in values. """ import math @@ -27,15 +31,70 @@ NAN = float("nan") INF = float("inf") -# Enum values (spine.constants.enums) +# Enum values (spine.constants.enums). These are the values of SPINE v1.4.0; configure() replaces +# them with those of another tag. UNKNOWN is -1 in every checked enum. UNKNOWN = -1 SHOWER, TRACK, MICHEL, DELTA, LOWE, GHOST, LARCV_UNKNOWN = 0, 1, 2, 3, 4, 5, 6 PHOTON, ELECTRON, MUON, PION, PROTON, KAON = 0, 1, 2, 3, 4, 5 CC, NC = 0, 1 LARSOFT, GENIE, GIBUU, NUWRO, NEUT = 0, 1, 2, 3, 4 -PDG = {UNKNOWN: -1, PHOTON: 22, ELECTRON: 11, MUON: 13, PION: 211, PROTON: 2212, KAON: 321} -MASS = {PHOTON: 0.0, ELECTRON: 0.511, MUON: 105.66, PION: 139.57, PROTON: 938.27, KAON: 493.68} +# The labels this content writes, per SPINE enum (UNKNOWN excepted). +LABELS = { + "ParticleShape": ("SHOWER", "TRACK", "MICHEL", "DELTA", "LOWE", "GHOST", "LARCV_UNKNOWN"), + "ParticlePID": ("PHOTON", "ELECTRON", "MUON", "PION", "PROTON", "KAON"), + "NuCurrentType": ("CC", "NC"), + "NuInteractionScheme": ("LARSOFT", "GENIE", "GIBUU", "NUWRO", "NEUT"), +} +_PDG_OF = {"PHOTON": 22, "ELECTRON": 11, "MUON": 13, "PION": 211, "PROTON": 2212, "KAON": 321} +_MASS_OF = {"PHOTON": 0.0, "ELECTRON": 0.511, "MUON": 105.66, "PION": 139.57, "PROTON": 938.27, + "KAON": 493.68} +_DEFAULTS = {label: globals()[label] for labels in LABELS.values() for label in labels} + +# Keyed by PID value; rebuilt by configure(). +PDG = {UNKNOWN: -1, **{globals()[k]: v for k, v in _PDG_OF.items()}} +MASS = {globals()[k]: v for k, v in _MASS_OF.items()} + +# The labels configure() could not find in the tag and wrote as UNKNOWN, as dicts +# {kind: "substitution", name: ".