sacc_io: SACC read/write layer for the standard data-product layout

.github/workflows/deploy-image.yml

@@ -44,6 +44,12 @@ jobs:
       - name: Import smoke test
         run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sp_validation"
 
+      # The fast suite doesn't import the blinding stack, so a broken
+      # firecrown/smokescreen install would otherwise ship green. Prove the
+      # image can actually load it (sacc + patched firecrown + smokescreen).
+      - name: Blinding-stack import smoke test
+        run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sacc; import firecrown.likelihood; import smokescreen"
+
       # Run the fast test suite against the freshly-built image *before*
       # pushing, so a failing suite blocks publication. The image carries the
       # full stack and the test files (COPY . + editable install), so this

Dockerfile

@@ -28,8 +28,25 @@ RUN uv pip install --no-cache-dir --upgrade 'cs_util>=0.2.1'
 WORKDIR /sp_validation
 COPY . /sp_validation
 
-# Install with the test + glass extras so the image can run the unit suite in CI
-# *and* the GLASS map-level mock test. `glass` (Generator for Large Scale
-# Structure) ships `glass.ext.camb`; `cosmology` provides the `Cosmology` wrapper
-# (`Cosmology.from_camb`) GLASS consumes. Both come in via the `[glass]` extra.
-RUN uv pip install --no-cache-dir -e '.[test,glass]'
+# Install with the test + glass + blinding extras so the image can run the unit
+# suite in CI, the GLASS map-level mock test, *and* the SACC/Smokescreen blinding
+# stack. `glass` (Generator for Large Scale Structure) ships `glass.ext.camb`;
+# `cosmology` provides the `Cosmology` wrapper (`Cosmology.from_camb`) GLASS
+# consumes. The `[blinding]` extra (firecrown + smokescreen) needs the override
+# file: firecrown declares conda-forge-only / unused sampler connectors as hard
+# deps — see uv-overrides.txt for the full story.
+RUN uv pip install --no-cache-dir --overrides uv-overrides.txt -e '.[test,glass,blinding]'
+
+# Same uv gotcha as the cs_util upgrade above (astral-sh/uv #8410): if the base
+# image already carries a numpy that violates the [blinding] extra's new
+# `numpy<2.5` cap (firecrown 1.15.1 breaks on numpy 2.5 at import), the
+# editable install won't move it. Request the bound explicitly so the image is
+# deterministic either way; numpy 2.4.x is ABI-compatible with the compiled
+# stack (verified: pyccl/camb/treecorr/healpy/pymaster + fast suite).
+RUN uv pip install --no-cache-dir 'numpy>=2.2,<2.5'
+
+# firecrown is distributed for conda-forge (where NumCosmo always exists) and
+# hits NumCosmo at import time in a pip env, on paths unrelated to our use.
+# This patches the installed tree (surgical, pinned-version-checked, loud on
+# mismatch) and verifies `import firecrown.likelihood; import smokescreen`.
+RUN python scripts/patch_firecrown.py

README.md

@@ -87,6 +87,26 @@ docker run --rm -it ghcr.io/cosmostat/sp_validation:develop python -c "import sp
 
 We do not currently build images for Apple Silicon/arm64; however the amd64 images should work on these systems, albeit with reduced performance.
 
+## Local Installation
+
+Requires Python ≥ 3.12 (the floor is set by the blinding stack; the container
+already runs 3.12). With [uv](https://docs.astral.sh/uv/):
+
+```bash
+uv venv --python 3.12
+uv pip install -e '.[test]'
+```
+
+To also install the data-vector blinding stack (Smokescreen + firecrown, PRD
+[#241](https://github.com/CosmoStat/sp_validation/issues/241)), pass the
+dependency-override file — firecrown is not pip-resolvable without it (see
+`uv-overrides.txt` for why):
+
+```bash
+uv pip install --overrides uv-overrides.txt -e '.[test,blinding]'
+python scripts/patch_firecrown.py  # make pip-installed firecrown importable without NumCosmo
+```
+
 
 
 ## Flow chart

pyproject.toml

@@ -9,7 +9,10 @@ authors = [
 ]
 license = {text = "MIT"}
 readme = "README.md"
-requires-python = ">=3.11"
+# 3.12 floor set by Smokescreen 1.5.6 (and firecrown v1.15); the container base
+# (shapepipe:develop) is already python:3.12-slim-bookworm, so this aligns
+# pyproject with the actual runtime.
+requires-python = ">=3.12"
 classifiers = [
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3",
@@ -52,6 +55,10 @@ dependencies = [
     "pymaster",
     "regions",
     "reproject",
+    # SACC (LSST DESC's data-vector container) is the standard format for all
+    # data products from the tomographic round on (PRD #241); sp_validation.sacc_io
+    # is core library code, so sacc is a core dependency.
+    "sacc>=2.4,<3",
     # scipy 1.18 ported FITPACK from Fortran to C, changing the return shape of
     # RectBivariateSpline(scalar, scalar, grid=False) from 0-d `array(x)` to
     # shape-(1,) `array([x])`. camb's BBN Y_He predictor (bbn.py) wraps the
@@ -106,6 +113,31 @@ glass = [
     "glass.ext.camb==2023.6",
     "cosmology==2022.10.9",
 ]
+# Data-vector blinding (PRD #241 §3-§5): Smokescreen applies the Muir et al.
+# shift d → d + t(hidden) − t(fid), with firecrown + CCL as the theory engine
+# (only compute_theory_vector is used; sampling stays with CosmoSIS). The blind
+# must be exactly recomputable from the seed at unblinding time, so the whole
+# theory stack is pinned exactly, as a set. Smokescreen 1.5.6 + firecrown v1.15
+# both set the python floor (>=3.12).
+#
+# firecrown is not on PyPI and declares conda-forge-only / unused sampler
+# connectors as hard deps, so installing this extra requires the dependency
+# override file: `uv pip install --overrides uv-overrides.txt -e '.[blinding]'`
+# (see uv-overrides.txt; the Dockerfile does this for the container).
+# After installing this extra, run `python scripts/patch_firecrown.py` — it
+# makes pip-installed firecrown importable without NumCosmo (conda-forge-only);
+# see that script's docstring for the full story.
+blinding = [
+    "firecrown @ git+https://github.com/LSSTDESC/firecrown.git@v1.15.1",
+    "smokescreen==1.5.6",
+    "pyccl==3.3.4",
+    # firecrown 1.15.1 subclasses npt.NDArray (DataVector); numpy 2.5 turned
+    # npt.NDArray into a non-subclassable typing alias, breaking firecrown at
+    # import. firecrown's own env caps numpy<2.4; 2.4.3 is verified against
+    # the full compiled stack (pyccl/camb/treecorr/healpy/pymaster) + the
+    # sp_validation fast suite.
+    "numpy>=2.2,<2.5",
+]
 develop = ["sp_validation[test,docs]"]
 
 [tool.pytest.ini_options]
@@ -124,7 +156,7 @@ markers = [
 
 [tool.ruff]
 line-length = 88
-target-version = "py311"
+target-version = "py312"
 
 # Snakemake injects a `snakemake` object into rule scripts at runtime, so ruff
 # can't see where it's defined. Declaring it a builtin silences the false

scripts/patch_firecrown.py

@@ -0,0 +1,225 @@
+"""Make pip-installed firecrown importable without NumCosmo.
+
+Run *inside* the target environment, after installing the ``[blinding]`` extra:
+
+    python scripts/patch_firecrown.py
+
+Why this exists (PRD #241, PR 1): firecrown is the theory engine for
+Smokescreen blinding — only ``compute_theory_vector`` on the SACC-read
+cosmic-shear path is used. Upstream distributes firecrown via conda-forge,
+where NumCosmo (a GObject-introspection C library, absent from PyPI) is always
+present; in a pip/uv environment, firecrown 1.15.1 hits NumCosmo at *import
+time* through two paths that have nothing to do with cosmic shear:
+
+1. ``firecrown/generators/__init__.py`` eagerly re-exports the LSST Y1/Y10
+   predefined n(z) bin constants, defeating the lazy ``__getattr__`` that
+   ``_inferred_galaxy_zdist`` already provides — and computing those constants
+   imports NumCosmo.
+2. ``firecrown/likelihood/__init__.py`` eagerly imports the cluster
+   likelihoods, which import ``crow`` (lsstdesc-crow), which subclasses a
+   NumCosmo C class at module load (``class CountsIntegralND(Ncm.IntegralND)``).
+
+This script (a) restores laziness in ``generators``, (b) makes the cluster
+imports optional, and (c) installs a *loud* ``numcosmo_py`` shim so that any
+genuine NumCosmo use raises immediately instead of being silently faked.
+Everything is exact-string surgery against the pinned firecrown v1.15.1: if a
+target string is missing (e.g. after a version bump), the script fails loudly
+so the pin and the patch get reviewed together. Idempotent — safe to re-run.
+
+The right long-term fix is upstream (guarded/lazy imports in firecrown); until
+then this file is the entire cost of staying pip-installable.
+"""
+
+import importlib.metadata
+import importlib.util
+import subprocess
+import sys
+from pathlib import Path
+
+EXPECTED_FIRECROWN = "1.15.1"
+
+GENERATORS_OLD = """\
+    # Lazy-loaded bins (via __getattr__)
+    Y1_LENS_BINS,
+    Y1_SOURCE_BINS,
+    Y10_LENS_BINS,
+    Y10_SOURCE_BINS,
+    LSST_Y1_LENS_HARMONIC_BIN_COLLECTION,
+    LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION,
+    LSST_Y10_LENS_HARMONIC_BIN_COLLECTION,
+    LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION,
+)
+"""
+
+GENERATORS_NEW = """\
+)
+
+# NOTE (sp_validation patch, scripts/patch_firecrown.py): the LSST Y1/Y10
+# predefined bin constants are computed lazily in _inferred_galaxy_zdist via a
+# module-level __getattr__ that imports NumCosmo. Importing them EAGERLY here
+# forced NumCosmo at `import firecrown.generators` (hence at
+# `import firecrown.likelihood`), which pip cannot satisfy. Re-expose them
+# lazily instead; the SACC-read cosmic-shear path never touches them.
+_LAZY_BIN_NAMES = frozenset(
+    {
+        "Y1_LENS_BINS",
+        "Y1_SOURCE_BINS",
+        "Y10_LENS_BINS",
+        "Y10_SOURCE_BINS",
+        "LSST_Y1_LENS_HARMONIC_BIN_COLLECTION",
+        "LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION",
+        "LSST_Y10_LENS_HARMONIC_BIN_COLLECTION",
+        "LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION",
+    }
+)
+
+
+def __getattr__(name):
+    if name in _LAZY_BIN_NAMES:
+        from . import _inferred_galaxy_zdist as _z
+
+        return getattr(_z, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+"""
+
+LIKELIHOOD_OLD = """\
+# Cluster statistics
+from firecrown.likelihood._binned_cluster import BinnedCluster
+from firecrown.likelihood._binned_cluster_number_counts import (
+    BinnedClusterNumberCounts,
+)
+from firecrown.likelihood._binned_cluster_number_counts_shear import (
+    BinnedClusterShearProfile,
+)
+"""
+
+LIKELIHOOD_NEW = """\
+# Cluster statistics.
+# NOTE (sp_validation patch, scripts/patch_firecrown.py): the cluster
+# likelihoods import `crow` (lsstdesc-crow), which subclasses NumCosmo C
+# classes at module load. NumCosmo is conda-forge-only, so in a pip/uv env
+# these imports fail. They are NOT on the cosmic-shear (TwoPoint/WeakLensing)
+# path, so they become optional: without NumCosmo the cluster classes are
+# unavailable but everything else loads.
+try:
+    from firecrown.likelihood._binned_cluster import BinnedCluster
+    from firecrown.likelihood._binned_cluster_number_counts import (
+        BinnedClusterNumberCounts,
+    )
+    from firecrown.likelihood._binned_cluster_number_counts_shear import (
+        BinnedClusterShearProfile,
+    )
+except (ImportError, RuntimeError, TypeError):  # pragma: no cover
+    BinnedCluster = None  # type: ignore[assignment,misc]
+    BinnedClusterNumberCounts = None  # type: ignore[assignment,misc]
+    BinnedClusterShearProfile = None  # type: ignore[assignment,misc]
+"""
+
+SHIM = '''\
+"""Minimal loud shim for numcosmo_py (installed by sp_validation).
+
+NumCosmo is a GObject-introspection C library available only via conda-forge.
+With the companion patches to firecrown (scripts/patch_firecrown.py), the
+SACC-read cosmic-shear likelihood path never imports it; this shim provides
+the import-time names so the patched package loads, and any genuine numerical
+use of NumCosmo raises loudly rather than being silently faked.
+"""
+
+
+class _Missing:
+    def __init__(self, path="numcosmo_py"):
+        self._p = path
+
+    def __getattr__(self, name):
+        return _Missing(f"{self._p}.{name}")
+
+    def __call__(self, *a, **k):
+        raise RuntimeError(
+            f"{self._p} was called, but NumCosmo is not installed (conda-forge "
+            "only, not on PyPI). It is not needed for the SACC-read "
+            "cosmic-shear likelihood path."
+        )
+
+    def __getitem__(self, item):
+        return _Missing(f"{self._p}[...]")
+
+
+Ncm = _Missing("numcosmo_py.Ncm")
+Nc = _Missing("numcosmo_py.Nc")
+GObject = _Missing("numcosmo_py.GObject")
+
+
+def dict_to_var_dict(*a, **k):
+    raise RuntimeError("numcosmo_py.dict_to_var_dict unavailable (no NumCosmo)")
+
+
+def var_dict_to_dict(*a, **k):
+    raise RuntimeError("numcosmo_py.var_dict_to_dict unavailable (no NumCosmo)")
+'''
+
+
+def patch_file(path: Path, old: str, new: str) -> str:
+    text = path.read_text()
+    if new in text:
+        return "already patched"
+    if old not in text:
+        sys.exit(
+            f"FATAL: expected text not found in {path}.\n"
+            "firecrown has probably been bumped past the pinned version this "
+            "patch targets — review scripts/patch_firecrown.py together with "
+            "the [blinding] pin in pyproject.toml."
+        )
+    path.write_text(text.replace(old, new, 1))
+    return "patched"
+
+
+def main() -> None:
+    spec = importlib.util.find_spec("firecrown")
+    if spec is None or spec.origin is None:
+        sys.exit("FATAL: firecrown is not installed in this environment.")
+    pkg = Path(spec.origin).parent
+
+    # Metadata, not `import firecrown` — pre-patch, importing is what's broken.
+    version = importlib.metadata.version("firecrown")
+    if version != EXPECTED_FIRECROWN:
+        sys.exit(
+            f"FATAL: firecrown {version} != expected {EXPECTED_FIRECROWN}; "
+            "review this patch against the new version before bumping "
+            "EXPECTED_FIRECROWN."
+        )
+
+    print(
+        "generators/__init__.py:",
+        patch_file(pkg / "generators" / "__init__.py", GENERATORS_OLD, GENERATORS_NEW),
+    )
+    print(
+        "likelihood/__init__.py:",
+        patch_file(pkg / "likelihood" / "__init__.py", LIKELIHOOD_OLD, LIKELIHOOD_NEW),
+    )
+
+    # Loud numcosmo_py shim — only when no real NumCosmo is present.
+    if importlib.util.find_spec("numcosmo_py") is None:
+        shim_dir = pkg.parent / "numcosmo_py"
+        shim_dir.mkdir(exist_ok=True)
+        (shim_dir / "__init__.py").write_text(SHIM)
+        print("numcosmo_py shim: installed")
+    else:
+        print("numcosmo_py shim: skipped (numcosmo_py importable)")
+
+    check = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "import firecrown.likelihood; import smokescreen",
+        ],
+        capture_output=True,
+        text=True,
+    )
+    if check.returncode != 0:
+        sys.exit(f"FATAL: post-patch import check failed:\n{check.stderr}")
+    print("post-patch import check: firecrown.likelihood + smokescreen OK")
+
+
+if __name__ == "__main__":
+    main()