From 5f1ea3cd04b9c56f096d82730cc739dc765271f8 Mon Sep 17 00:00:00 2001 From: Pierre Granger Date: Fri, 6 Nov 2026 12:06:35 +0100 Subject: [PATCH] The schema dumper records the SPINE commit, the writer format and warnings dump_schema.py writes three new keys, which generate.py ignores (the generated C++ does not change): - spine_sha: the commit given with the new --spine-sha (40 hexadecimal characters; null without it). - writer_format: HDF5Writer's format_name, legacy_format_version, current_format_version and supported_format_versions, read with getattr so that a missing attribute is null, and default_format_version, the default of the format_version parameter of HDF5Writer.__init__ (null when the parameter does not exist). A change of the writer's default layout shows here. - warnings: {kind, name, detail} for what the dumper sees but does not map. unmapped_attribute is a FieldMetadata attribute outside MAPPED_ATTRIBUTES (at 1.4.0, only cat); unlisted_class is a class of spine.data with attr_metadata in neither DEFAULT_CLASSES nor the new EXCLUDED_CLASSES (RecoFragment, TruthFragment, Neutrino, ImageMeta2D, larcv's Particle, ParticleLabel, and the bases DataBase and PosDataBase). An alias of a listed class (Meta) is listed. Any exception while importing or reading SPINE now prints one line on stderr, "dump_schema: : ", and exits 1; an unknown --classes name is a LookupError. schema/spine-1.4.0.json is re-dumped with --spine-sha 4726e068...: only the three keys and three notes are new. --- schema/spine-1.4.0.json | 25 ++++++- tools/gen/EGM.md | 53 +++++++++++--- tools/gen/README.md | 26 ++++++- tools/gen/dump_schema.py | 153 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 240 insertions(+), 17 deletions(-) diff --git a/schema/spine-1.4.0.json b/schema/spine-1.4.0.json index fc4bdcd..f544273 100644 --- a/schema/spine-1.4.0.json +++ b/schema/spine-1.4.0.json @@ -1,13 +1,17 @@ { "schema_format": 1, "spine_version": "1.4.0", + "spine_sha": "4726e068f17e433e2fec068dc8ee498fa9a0658f", "generator": "tools/gen/dump_schema.py", "notes": [ "Field order is the order of as_dict(lite=False): the HDF5 compound member order.", "Types are the declared ones (annotation or FieldMetadata.dtype), not read from files.", "interaction_mode and interaction_type are plain int64 with no enum: SPINE interprets them under interaction_scheme (see scheme_dependent_enum on those fields and enums[*].scheme_dependent).", "derived fields (SPINE stored_property) take as default the value on a default-constructed object (default_source = default_instance).", - "float32/float64 defaults 'nan', 'inf' and '-inf' are strings in this file." + "float32/float64 defaults 'nan', 'inf' and '-inf' are strings in this file.", + "spine_sha is the commit given with --spine-sha (null without it).", + "writer_format: HDF5Writer's format attributes and the default of its format_version parameter; null when SPINE has no such attribute or parameter.", + "warnings: what the dumper sees but does not map (unmapped_attribute, unlisted_class)." ], "classes": { "RecoParticle": { @@ -8782,5 +8786,22 @@ "type_check": { "method": "declared type vs numpy dtype of HDF5Writer.get_object_dtype(cls())", "mismatches": [] - } + }, + "writer_format": { + "format_name": "spine_hdf5", + "legacy_format_version": 1, + "current_format_version": 2, + "supported_format_versions": [ + 1, + 2 + ], + "default_format_version": 1 + }, + "warnings": [ + { + "kind": "unmapped_attribute", + "name": "cat", + "detail": "FieldMetadata.cat is not read; set on RecoParticle.index, RecoParticle.orig_index, RecoParticle.fragment_ids, TruthParticle.index_adapt, TruthParticle.index_g4, TruthParticle.index, TruthParticle.orig_index, TruthParticle.fragment_ids, RecoInteraction.index, RecoInteraction.orig_index, TruthInteraction.index_adapt, TruthInteraction.index_g4, TruthInteraction.index, TruthInteraction.orig_index" + } + ] } diff --git a/tools/gen/EGM.md b/tools/gen/EGM.md index 468a192..9678b51 100644 --- a/tools/gen/EGM.md +++ b/tools/gen/EGM.md @@ -24,14 +24,26 @@ SPINE clone --dump_schema.py--> schema/spine-.json --generate.py--> C++ - `--out FILE` (overrides `--out-dir`) - `--classes NAME...` (default `DEFAULT_CLASSES`) - `--strict` + - `--spine-sha SHA` (40 hexadecimal characters, recorded as `spine_sha`) - `DEFAULT_CLASSES`: RecoParticle, TruthParticle, RecoInteraction, TruthInteraction, RunInfo, Trigger, ImageMeta3D, Flash, CRTHit, FlashHypothesis. +- `EXCLUDED_CLASSES`: the classes of `spine.data` with `attr_metadata` that are + not dumped on purpose: RecoFragment, TruthFragment, Neutrino, ImageMeta2D, + Particle (larcv), ParticleLabel, and the bases DataBase and PosDataBase. - `CLASS_ROLE`, `FLAG_NAMES` (derived, position, vector, index, categorical, pdg, - pdg_minus_one_valid, pointwise, lite_skip), `SCALAR_ELEMENT`. + pdg_minus_one_valid, pointwise, lite_skip), `MAPPED_ATTRIBUTES` (the flags but + `derived`, plus return_type, length, dtype, units, enum, reference, + reference_space, skip), `WRITER_FORMAT_ATTRIBUTES`, `SCALAR_ELEMENT`. - Helpers: `declared_type`, `dump_class`, `collect_enums`, `scheme_tables`, - `field_doc`, `compare_writer`. -- Output keys: `schema_format` (1), `spine_version`, `generator`, `notes`, - `classes`, `enums`, `scheme_tables`, `type_check {method, mismatches}`. + `field_doc`, `compare_writer`, `writer_format`, `unmapped_attributes`, + `unlisted_classes`. +- Output keys: `schema_format` (1), `spine_version`, `spine_sha` (`null` without + `--spine-sha`), `generator`, `notes`, `classes`, `enums`, `scheme_tables`, + `type_check {method, mismatches}`, `writer_format {format_name, + legacy_format_version, current_format_version, supported_format_versions, + default_format_version}` and `warnings [{kind, name, detail}]`. +- Exit codes: 0 written; 1 SPINE could not be imported or read, or `--strict` and + a type mismatch; 2 usage error. ### `generate.py` (CLI; standard library only) @@ -80,6 +92,9 @@ SPINE clone --dump_schema.py--> schema/spine-.json --generate.py--> C++ `/schema/spine-*.json`, otherwise it exits with `generate: pass --schema (found N files in schema/)` [generate.py:main]. The model describes one SPINE release. +- The model version (`SPINE_READER_MODEL_SPINE_VERSION`, the descriptors' + `schema_version`) names the schema the model was generated from, not the newest + SPINE it reads [generate.py:generate]. - Output is deterministic: generated files and globs are processed in sorted order, so a diff after regeneration is a real change [generate.py:main]. - Stale-file rule: any `.hpp`/`.cpp` directly under `include/spine/model/` or @@ -111,13 +126,33 @@ SPINE clone --dump_schema.py--> schema/spine-.json --generate.py--> C++ dataclass defaults are checked against instance values. Mismatches are recorded under `type_check.mismatches`. They fail the run (exit 1) only with `--strict` [dump_schema.py:declared_type, dump_class, main]. -- `dump_schema.py` stops on: - - an unknown `--classes` name (`dump_schema: spine.data has no class `); +- `dump_schema.py` stops, writing nothing, on any exception while it imports or + reads SPINE. It prints one line to stderr, `dump_schema: : + `, and exits 1. The cases it raises itself: + - an unknown `--classes` name (`LookupError: spine.data has no class `); - a field type it cannot map (`TypeError`: an enum not declared `int`, an array without a declared dtype, an unsupported type); - a stored name that is neither a field nor a stored property (`RuntimeError`); - - a scheme whose interaction enum cannot be identified uniquely (`RuntimeError`) + - a scheme whose interaction enum cannot be identified uniquely (`RuntimeError`). + + A SPINE that lacks an attribute the dumper reads gives `AttributeError` (for + example a `FieldMetadata` without `pdg`, or a SPINE older than `attr_metadata`) [dump_schema.py:declared_type, dump_class, scheme_tables, main]. +- `writer_format` reads each attribute with `getattr(HDF5Writer, name, None)`, and + `default_format_version` from the signature of `HDF5Writer.__init__`, so a + missing attribute or parameter is `null`, not an error. A renamed + `format_version` parameter gives `null` [dump_schema.py:writer_format]. +- `warnings` lists, first the unmapped attributes in the order `FieldMetadata` + declares them, then the unlisted classes sorted by name: + - `unmapped_attribute`: each `FieldMetadata` attribute outside + `MAPPED_ATTRIBUTES`; `detail` names the dumped fields that set it (at SPINE + 1.4.0, only `cat`); + - `unlisted_class`: each class of `vars(spine.data)` with `attr_metadata` that is + in neither `DEFAULT_CLASSES`, `--classes` nor `EXCLUDED_CLASSES`; an alias of a + listed class (`Meta` is `ImageMeta3D`) is listed. + + A comparison of two dumps compares warnings by `(kind, name)` + [dump_schema.py:unmapped_attributes, unlisted_classes]. - Fields that `as_dict` does not store go to `skipped_fields`; fields dropped by `as_dict(lite=True)` go to `lite_dropped` [dump_schema.py:dump_class]. - `check_names.py` exits 1 if any model identifier, after comments and strings are @@ -128,8 +163,8 @@ SPINE clone --dump_schema.py--> schema/spine-.json --generate.py--> C++ - `dump_schema.py` imports `spine.data`, `spine.version`, `HDF5Writer` and `spine.constants.enums`. It reads SPINE source files (for the docstrings) and - writes the JSON with `mkdir(parents=True)`. It prints field counts and the - mismatch list to stdout. + writes the JSON with `mkdir(parents=True)`. It prints field counts, the + mismatch list, the writer format and the warnings to stdout. - `generate.py` reads the schema. Unless `--check` is given, it deletes stale files and writes only the files whose content changed. It prints `SPINE : N files, M written (C classes, E enums)`, where M counts the diff --git a/tools/gen/README.md b/tools/gen/README.md index 4d9c844..cb8e136 100644 --- a/tools/gen/README.md +++ b/tools/gen/README.md @@ -14,7 +14,8 @@ SPINE clone at a tag --dump_schema.py--> schema/spine-.json --generate. source /cvmfs/sft.cern.ch/lcg/views/LCG_110/x86_64-el9-gcc13-opt/setup.sh git clone -q --branch vX.Y.Z /some/dir/spine-X.Y.Z -PYTHONPATH=/some/dir/spine-X.Y.Z/src python3 tools/gen/dump_schema.py # -> schema/spine-X.Y.Z.json +PYTHONPATH=/some/dir/spine-X.Y.Z/src python3 tools/gen/dump_schema.py \ + --spine-sha $(git -C /some/dir/spine-X.Y.Z rev-parse HEAD) # -> schema/spine-X.Y.Z.json python3 tools/gen/generate.py --schema schema/spine-X.Y.Z.json # -> include/spine/model*, src/model ``` @@ -26,6 +27,11 @@ one): the model describes one release. `generate.py --check` writes nothing and exits 1 if a committed generated file is stale. Output is deterministic, so a diff after regeneration is a real change. +`dump_schema.py` exits 1 when SPINE cannot be imported or read, after one line on stderr: +`dump_schema: : ` (for example `dump_schema: AttributeError: +'FieldMetadata' object has no attribute 'pdg'`). It writes nothing then. With `--strict`, a type +mismatch also exits 1. + ## What to check afterwards 1. **The type check printed by `dump_schema.py`.** Each field's declared type (annotation, or @@ -34,7 +40,9 @@ deterministic, so a diff after regeneration is a real change. under `type_check` in the JSON. The same section also lists any field whose dataclass default differs from the value on an instance. `--strict` makes mismatches an error. 2. **`git diff schema/`.** Fields added, removed or retyped, enum members, new `FieldMetadata` - flags. A class SPINE started storing is added to `DEFAULT_CLASSES` in `dump_schema.py`; the + flags. A change of `writer_format` (for example a new default layout) or a new entry under + `warnings` (an attribute the dumper does not map, a class it does not list) has no effect on the + generated C++, and needs a look all the same. A class SPINE started storing is added to `DEFAULT_CLASSES` in `dump_schema.py`; the header, the source and the `SPINE_READER_FOR_EACH_CLASS` entry then appear by themselves. A field or class SPINE removed disappears and `generate.py` deletes the stale files. 3. **`python3 tools/gen/check_names.py`.** No identifier of the model may be a macro of a common C, @@ -59,6 +67,20 @@ deterministic, so a diff after regeneration is a real change. ## What is in the JSON +Top level: + +| key | meaning | +| --- | --- | +| `schema_format`, `generator`, `notes` | the format of this file (1), the tool, and notes on reading it | +| `spine_version`, `spine_sha` | `spine.__version__`, and the commit given with `--spine-sha` (`null` without it) | +| `classes`, `enums`, `scheme_tables` | the model: what `generate.py` reads | +| `type_check` | the declared types against `HDF5Writer.get_object_dtype`, and dataclass defaults against instance values | +| `writer_format` | `format_name`, `legacy_format_version`, `current_format_version` and `supported_format_versions` of `HDF5Writer`, and `default_format_version`, the default of its `format_version` parameter; each is `null` when SPINE has no such attribute or parameter | +| `warnings` | `{kind, name, detail}` for what the dumper sees but does not map: `unmapped_attribute` (a `FieldMetadata` attribute outside `MAPPED_ATTRIBUTES`; at 1.4.0, `cat`) and `unlisted_class` (a class of `spine.data` with `attr_metadata` in neither `DEFAULT_CLASSES` nor `EXCLUDED_CLASSES`) | + +`generate.py` ignores `spine_sha`, `type_check`, `writer_format` and `warnings`: they describe the +dump, for comparing two dumps, and do not change the generated C++. + `classes..fields[]`, in the order of `as_dict(lite=False)` (the order of the HDF5 compound members), minus fields with `skip=True`: diff --git a/tools/gen/dump_schema.py b/tools/gen/dump_schema.py index cf7bb93..147f165 100644 --- a/tools/gen/dump_schema.py +++ b/tools/gen/dump_schema.py @@ -4,6 +4,7 @@ Usage (SPINE and its dependencies must be importable; the LCG view provides numpy and h5py):: PYTHONPATH=/src python3 tools/gen/dump_schema.py [--out-dir schema] + [--spine-sha SHA] [--strict] The output, ``schema/spine-.json``, is the only input of ``generate.py``. Everything in it is read from SPINE itself: @@ -18,6 +19,22 @@ Each field's declared type is then compared with what ``HDF5Writer.get_object_dtype`` produces for a default-constructed object, and every disagreement is listed under ``type_check``. + +It also records what a comparison of two dumps needs beyond the model: + +* ``spine_sha``: the commit given with ``--spine-sha`` (``null`` without it); +* ``writer_format``: the writer's format attributes (``format_name``, ``legacy_format_version``, + ``current_format_version``, ``supported_format_versions``) and ``default_format_version``, the + default of the ``format_version`` parameter of ``HDF5Writer.__init__``. Each is ``null`` when + SPINE has no such attribute or parameter, so a change of the default layout shows; +* ``warnings``: ``{kind, name, detail}`` for what the dumper sees but does not map: + ``unmapped_attribute``, a ``FieldMetadata`` attribute outside ``MAPPED_ATTRIBUTES``; and + ``unlisted_class``, a class of ``spine.data`` with ``attr_metadata`` that is in neither + ``DEFAULT_CLASSES`` nor ``EXCLUDED_CLASSES``. + +Exit codes: 0 written; 1 SPINE could not be imported or read (one line on stderr, +``dump_schema: : ``), or ``--strict`` and a type mismatch; 2 usage +error. """ import argparse @@ -39,9 +56,7 @@ # the class name "Meta"). # Flash, CRTHit: measured optical flashes and CRT hits that the reco objects point at. # FlashHypothesis: predicted light per interaction (RecoInteraction.flash_hypothesis_ids). -# Not included: RecoFragment/TruthFragment (stored only in full, non-lite output; no CAFMaker reads -# them), Neutrino (merged into TruthInteraction, never stored on its own), ImageMeta2D (the -# detectors served by the CAFMakers are 3D). Add a name here to include another class. +# Not included: the classes of EXCLUDED_CLASSES below. Add a name here to include another class. DEFAULT_CLASSES = [ "RecoParticle", "TruthParticle", @@ -55,6 +70,19 @@ "FlashHypothesis", ] +# Classes of spine.data with attr_metadata that are not dumped, on purpose. A class in neither list +# is reported as an unlisted_class warning. +EXCLUDED_CLASSES = [ + "RecoFragment", # stored only in full, non-lite output; no CAFMaker reads them + "TruthFragment", + "Neutrino", # merged into TruthInteraction, never stored on its own + "ImageMeta2D", # the detectors served by the CAFMakers are 3D + "Particle", # larcv's truth particle; TruthParticle carries it in the output + "ParticleLabel", # the compact particle record of cluster labels, not an output object + "DataBase", # base classes + "PosDataBase", +] + CLASS_ROLE = { "RecoParticle": "reconstructed particle (main reco product)", "TruthParticle": "truth particle matched to the reco output (main truth product)", @@ -81,6 +109,28 @@ "lite_skip", ] +# FieldMetadata attributes this dumper reads: the flags (except "derived", which it computes) and +# these. +# Any other attribute is reported as an unmapped_attribute warning. +MAPPED_ATTRIBUTES = [f for f in FLAG_NAMES if f != "derived"] + [ + "return_type", + "length", + "dtype", + "units", + "enum", + "reference", + "reference_space", + "skip", +] + +# HDF5Writer class attributes recorded under writer_format. +WRITER_FORMAT_ATTRIBUTES = [ + "format_name", + "legacy_format_version", + "current_format_version", + "supported_format_versions", +] + SCALAR_ELEMENT = {int: "int64", float: "float64", bool: "bool", str: "string"} PYTYPE_NAME = {int: "int", float: "float", bool: "bool", str: "str"} @@ -473,6 +523,76 @@ def raw_attribute_class_summary(cls): return doc.split("\n\n")[0] +# -------------------------------------------------------------------------------------- +# What a comparison of two dumps needs beyond the model +# -------------------------------------------------------------------------------------- + + +def plain(value): + """JSON form of a writer attribute: tuples become lists, anything unusual its repr.""" + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, (list, tuple)): + return [plain(v) for v in value] + return repr(value) + + +def writer_format(writer_class): + """The writer's format attributes and the default of its format_version parameter; null for + each one SPINE does not have.""" + out = {name: plain(getattr(writer_class, name, None)) for name in WRITER_FORMAT_ATTRIBUTES} + parameter = inspect.signature(writer_class.__init__).parameters.get("format_version") + if parameter is None or parameter.default is inspect.Parameter.empty: + out["default_format_version"] = None + else: + out["default_format_version"] = plain(parameter.default) + return out + + +def unmapped_attributes(classes): + """unmapped_attribute warnings: FieldMetadata attributes outside MAPPED_ATTRIBUTES.""" + import spine.data as sd + + from spine.data.field import FieldMetadata + + default = FieldMetadata() + warnings = [] + for f in dataclasses.fields(FieldMetadata): + if f.name in MAPPED_ATTRIBUTES: + continue + users = [ + f"{name}.{field['name']}" + for name in classes + for field in classes[name]["fields"] + if getattr(getattr(sd, name).attr_metadata(field["name"]), f.name) + != getattr(default, f.name) + ] + detail = f"FieldMetadata.{f.name} is not read; " + detail += f"set on {', '.join(users)}" if users else "set on no dumped field" + warnings.append({"kind": "unmapped_attribute", "name": f.name, "detail": detail}) + return warnings + + +def unlisted_classes(dumped): + """unlisted_class warnings: classes of spine.data with attr_metadata in no list. An alias of a + listed class (SPINE's Meta is ImageMeta3D) is listed.""" + import spine.data as sd + + listed = {id(getattr(sd, n)) for n in DEFAULT_CLASSES + list(dumped) + EXCLUDED_CLASSES + if hasattr(sd, n)} + warnings = [] + for name, obj in sorted(vars(sd).items()): + if isinstance(obj, type) and callable(getattr(obj, "attr_metadata", None)) and \ + id(obj) not in listed: + warnings.append({ + "kind": "unlisted_class", + "name": name, + "detail": f"{obj.__module__}.{obj.__name__} has attr_metadata, and is in neither " + "DEFAULT_CLASSES nor EXCLUDED_CLASSES of dump_schema.py", + }) + return warnings + + def main(): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--spine-src", help="SPINE `src` directory to import (else PYTHONPATH)") @@ -480,7 +600,19 @@ def main(): parser.add_argument("--out", help="output file (overrides --out-dir)") parser.add_argument("--classes", nargs="*", default=DEFAULT_CLASSES) parser.add_argument("--strict", action="store_true", help="exit 1 on any type mismatch") + parser.add_argument("--spine-sha", help="commit of the SPINE checkout, recorded as spine_sha") args = parser.parse_args() + if args.spine_sha is not None and not re.fullmatch(r"[0-9a-f]{40}", args.spine_sha): + parser.error(f"--spine-sha: not a full commit SHA: {args.spine_sha!r}") + try: + return dump(args) + except Exception as e: # one line, for a caller that maps it to an outcome + message = " ".join(str(e).split()) + print(f"dump_schema: {type(e).__name__}: {message}", file=sys.stderr) + return 1 + + +def dump(args): if args.spine_src: sys.path.insert(0, args.spine_src) @@ -500,7 +632,7 @@ def main(): for name in args.classes: cls = getattr(sd, name, None) if cls is None: - sys.exit(f"dump_schema: spine.data has no class {name}") + raise LookupError(f"spine.data has no class {name}") classes[name] = dump_class(cls, writer, enums, mismatches) tables = scheme_tables(enums) @@ -520,9 +652,12 @@ def main(): "enum_by_scheme": tables[entry["name"]], } + warnings = unmapped_attributes(classes) + unlisted_classes(classes) + result = { "schema_format": 1, "spine_version": __version__, + "spine_sha": args.spine_sha, "generator": "tools/gen/dump_schema.py", "notes": [ "Field order is the order of as_dict(lite=False): the HDF5 compound member order.", @@ -533,6 +668,10 @@ def main(): "derived fields (SPINE stored_property) take as default the value on a " "default-constructed object (default_source = default_instance).", "float32/float64 defaults 'nan', 'inf' and '-inf' are strings in this file.", + "spine_sha is the commit given with --spine-sha (null without it).", + "writer_format: HDF5Writer's format attributes and the default of its format_version " + "parameter; null when SPINE has no such attribute or parameter.", + "warnings: what the dumper sees but does not map (unmapped_attribute, unlisted_class).", ], "classes": classes, "enums": enums, @@ -541,6 +680,8 @@ def main(): "method": "declared type vs numpy dtype of HDF5Writer.get_object_dtype(cls())", "mismatches": mismatches, }, + "writer_format": writer_format(type(writer)), + "warnings": warnings, } out = Path(args.out) if args.out else Path(args.out_dir) / f"spine-{__version__}.json" @@ -555,6 +696,10 @@ def main(): print(f"type check: {len(mismatches)} mismatch(es)") for m in mismatches: print(f" {m['class']}.{m['field']} [{m['what']}]: {m['detail']}") + print("writer format: " + ", ".join(f"{k} {v}" for k, v in result["writer_format"].items())) + print(f"warnings: {len(warnings)}") + for w in warnings: + print(f" {w['kind']} {w['name']}: {w['detail']}") return 1 if (mismatches and args.strict) else 0