PR 7: adopt CosmoSIS native SACC likelihood (sacc_like), validated against the converter path

cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_sacc.ini

@@ -0,0 +1,110 @@
+#parameters used elsewhere in this file
+[DEFAULT]
+COSMOSIS_DIR = /n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa
+
+
+[pipeline]
+modules = consistency sample_S8 camb load_nz_sacc photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias sacc_like
+likelihoods = 2pt_like
+extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m
+timing = T
+debug = T
+
+[runtime]
+sampler = polychord
+verbosity = debug
+
+[polychord]
+live_points = 192
+feedback = 3
+resume = T
+base_dir = %(SCRATCH)s/polychord
+
+[test]
+
+[output]
+format = text
+lock = F
+
+[consistency]
+file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py
+verbose = F
+
+[sample_S8]
+file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py
+
+[camb]
+file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py
+mode=power
+lmax=2508
+feedback=0
+do_reionization=F
+kmin=1e-5
+kmax=20.0
+nk=200
+zmax=5.0
+zmax_background=5.0
+nz_background=500
+halofit_version=mead2020_feedback
+nonlinear=pk
+neutrino_hierarchy=normal
+kmax_extrapolate = 500.0
+
+[load_nz_sacc]
+file = %(COSMOSIS_DIR)s/number_density/load_nz_sacc/load_nz_sacc.py
+nz_file = %(SACC_FILE)s
+data_sets = source
+
+[photoz_bias]
+file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py
+mode = additive
+sample = nz_source
+bias_section = nofz_shifts
+interpolation = cubic
+output_deltaz_section_name = delta_z_out
+
+[linear_alignment]
+file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py
+method = bk_corrected
+
+[projection]
+file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py
+ell_min_logspaced = 1.0
+ell_max_logspaced = 25000.0
+n_ell_logspaced = 400
+shear-shear = source-source
+shear-intrinsic = source-source
+intrinsic-intrinsic = source-source
+get_kernel_peaks = F
+verbose = F
+
+[add_intrinsic]
+file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py
+shear-shear=T
+position-shear=F
+perbin=F
+
+[2pt_shear]
+file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so
+corr_type = 0  ; shear_cl -> shear_xi
+
+[shear_m_bias]
+file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py
+m_per_bin = True
+; Despite the parameter name, this can operate on xi as well as C_ell.
+cl_section = shear_xi_plus shear_xi_minus
+verbose = F
+
+; Native SACC likelihood via the sp_validation shim (arcmin->rad + ordering
+; guard over CosmoSIS's SaccClLikelihood). data_sets/angle ranges use the SACC
+; grammar (full data-type names + tracer pairs). like_name=2pt_like keeps the
+; block keys identical to the 2pt_like path so chain post-processing is unchanged.
+[sacc_like]
+file = %(SP_VALIDATION_MODULES)s/sacc_like_unions.py
+csl_dir = %(COSMOSIS_DIR)s
+data_file = %(SACC_FILE)s
+data_sets = galaxy_shear_xi_plus galaxy_shear_xi_minus
+like_name = 2pt_like
+
+angle_range_galaxy_shear_xi_plus_source_0_source_0 = 10.0 200.0
+angle_range_galaxy_shear_xi_minus_source_0_source_0 = 20.0 200.0

papers/cosmo_val/config/config.yaml

@@ -116,11 +116,14 @@ harmonic:
     binning: powspace
     nbins: 32
 
-# Cosmological inference data-product locations (dormant subsystem).
+# Cosmological inference data-product locations + tooling.
 inference:
   chains_dir: "/n09data/guerrini/output_chains"
   glass_mock_data_dir: "/n09data/guerrini/glass_mock_v1.4.6/results"
   glass_mock_chains_dir: "/n09data/guerrini/glass_mock_chains"
+  # CosmoSIS Standard Library checkout — fills COSMOSIS_DIR in the generated
+  # pipeline inis (the module `file =` paths and the sacc_like shim's csl_dir).
+  csl_dir: "/n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa"
 
 cosebis:
   theta_min: 1.0

pyproject.toml

@@ -138,6 +138,15 @@ blinding = [
     # sp_validation fast suite.
     "numpy>=2.2,<2.5",
 ]
+# Native SACC inference likelihood (PR 7, sp_validation.sacc_like_unions). The
+# shim subclasses CosmoSIS's SaccClLikelihood; this pins the engine. Both engine
+# module files are pure-python (no CAMB / compiled CSL), so the equality tests
+# (tests/test_sacc_like.py) run with just this extra + a CosmoSIS Standard
+# Library checkout, located via the CSL_DIR env var:
+#   git clone --depth 1 https://github.com/joezuntz/cosmosis-standard-library CSL
+#   CSL_DIR=/path/to/CSL  pytest ... test_sacc_like.py
+# Inert in CI (its Dockerfile installs only [test,glass,blinding]).
+cosmosis = ["cosmosis>=3.25"]
 develop = ["sp_validation[test,docs]"]
 
 [tool.pytest.ini_options]

src/sp_validation/one_covariance_io.py

@@ -0,0 +1,256 @@
+"""ONE_COVARIANCE_IO.
+
+:Name: one_covariance_io.py
+
+:Description: File-format glue between the SACC data-product layout
+              (:mod:`sp_validation.sacc_io`) and OneCovariance
+              (https://github.com/rreischke/OneCovariance). Two directions:
+
+              - **n(z) SACC -> OneCovariance input** (:func:`write_nz`): the
+                ``source_i`` NZ tracers of an analysis SACC are written as the
+                combined whitespace-delimited redshift file OneCovariance reads
+                (column 0 = z grid, then one ``n(z)`` column per tomographic
+                bin, no bin edges), and a matching ``[redshift]`` config stanza
+                is returned via :func:`nz_config_stanza`.
+
+              - **OneCovariance output -> SACC covariance blocks**
+                (:func:`covariance_blocks`): the flat ``covariance_list_*.dat``
+                table OneCovariance emits (one row per element pair) is reshaped
+                into dense square block(s) — reusing
+                :func:`sp_validation.statistics.cov_from_one_covariance` for the
+                per-block reshape — and paired with SACC selectors so a caller
+                can feed them straight to
+                :func:`sp_validation.sacc_io.assemble_covariance`.
+
+              OneCovariance itself is *not* a dependency: this module only
+              touches its file formats, verified against the upstream
+              ``config.ini`` (``rreischke/OneCovariance`` @ main).
+
+              n(z) file format (upstream ``config.ini`` comment, verbatim):
+
+                  ``redshift   n_1(z) ... n_{N_source}(z)``
+
+              i.e. a plain whitespace-delimited text file, column 0 the shared
+              redshift grid and one column per tomographic bin — no ``z_low``/
+              ``z_high`` edges (this is the OneCovariance convention, distinct
+              from the CosmoSIS NZDATA table which *does* carry edges). All
+              source bins must therefore share one z grid.
+
+              ``[redshift]`` config keys (upstream canonical names): a single
+              combined file goes in ``zlens_directory`` + ``zlens_file``;
+              ``value_loc_in_lensbin`` (``mid``/``left``/``right``) says where
+              in each histogram bin the tabulated ``n(z)`` value sits — ``mid``
+              for the bin-centred grids the SACC stores. NOTE: the UNIONS
+              OneCovariance template driven by
+              ``cosmo_val/pseudo_cl.py._modify_onecov_config`` writes the older
+              key names ``z_directory``/``zlens_file`` instead; pass
+              ``dir_key="z_directory"`` to match that template.
+"""
+
+import os
+
+import numpy as np
+
+from . import sacc_io
+from .statistics import cov_from_one_covariance
+
+
+def nz_table(s, n_bins):
+    """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table.
+
+    Parameters
+    ----------
+    s : sacc.Sacc
+        SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers.
+    n_bins : int
+        Number of tomographic source bins to write.
+
+    Returns
+    -------
+    numpy.ndarray
+        Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift
+        grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the
+        OneCovariance combined-file layout (``redshift  n_1(z) … n_N(z)``).
+
+    Raises
+    ------
+    ValueError
+        If any source bin is missing, or if the bins do not share one z grid
+        (OneCovariance's combined file has a single redshift column, so the
+        grids must agree bin-for-bin).
+    """
+    z0, nz0 = sacc_io.get_nz(s, 0)
+    z0 = np.asarray(z0, dtype=float)
+    columns = [z0]
+    for i in range(n_bins):
+        if sacc_io.source_name(i) not in s.tracers:
+            raise ValueError(
+                f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write "
+                f"a {n_bins}-bin OneCovariance n(z) file"
+            )
+        z_i, nz_i = sacc_io.get_nz(s, i)
+        if not np.array_equal(np.asarray(z_i, dtype=float), z0):
+            raise ValueError(
+                f"source bin {i} n(z) grid differs from source bin 0; the "
+                "OneCovariance combined n(z) file has one shared redshift column"
+            )
+        columns.append(np.asarray(nz_i, dtype=float))
+    return np.column_stack(columns)
+
+
+def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True):
+    """Write the OneCovariance combined n(z) input file from a SACC.
+
+    OneCovariance reads the source redshift distribution as a plain
+    whitespace-delimited text file whose column 0 is the shared redshift grid
+    and whose remaining columns are the per-bin ``n(z)`` (``redshift  n_1(z)
+    … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the
+    SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza
+    that points OneCovariance at it.
+
+    Parameters
+    ----------
+    s : sacc.Sacc
+        Analysis SACC with the ``source_i`` NZ tracers.
+    path : str or pathlib.Path
+        Output text-file path (overwritten). Its directory + basename become
+        the ``[redshift]`` directory/file config values.
+    n_bins : int
+        Number of tomographic source bins to write.
+    dir_key : str, optional
+        Config key for the redshift directory. Default ``"zlens_directory"``
+        (upstream canonical). Pass ``"z_directory"`` for the UNIONS template
+        driven by ``pseudo_cl.py._modify_onecov_config``.
+    header : bool, optional
+        If ``True`` (default) prepend a ``# redshift  n_1(z) …`` comment header
+        naming the columns; OneCovariance's ``genfromtxt``-style reader ignores
+        it. Set ``False`` for a bare numeric file.
+
+    Returns
+    -------
+    dict
+        The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming
+        the file just written.
+    """
+    table = nz_table(s, n_bins)
+    head = ""
+    if header:
+        cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins))
+        head = f"redshift {cols}"
+    np.savetxt(str(path), table, header=head)
+    return nz_config_stanza(
+        os.path.dirname(os.path.abspath(str(path))),
+        os.path.basename(str(path)),
+        dir_key=dir_key,
+    )
+
+
+def nz_config_stanza(
+    directory, filename, *, dir_key="zlens_directory", value_loc="mid"
+):
+    """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file.
+
+    Parameters
+    ----------
+    directory : str
+        Directory holding the n(z) file (OneCovariance ``*_directory`` value).
+    filename : str
+        n(z) file basename (OneCovariance ``zlens_file`` value).
+    dir_key : str, optional
+        Directory config key — ``"zlens_directory"`` (upstream) or
+        ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``.
+    value_loc : str, optional
+        ``value_loc_in_lensbin`` — where in each histogram bin the tabulated
+        ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``,
+        matching the bin-centred grids the SACC stores.
+
+    Returns
+    -------
+    dict
+        The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file":
+        filename, "value_loc_in_lensbin": value_loc}``. Assign these under
+        ``config["redshift"]`` of a OneCovariance ``configparser`` config.
+    """
+    if value_loc not in ("mid", "left", "right"):
+        raise ValueError(
+            f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}"
+        )
+    return {
+        dir_key: directory,
+        "zlens_file": filename,
+        "value_loc_in_lensbin": value_loc,
+    }
+
+
+def read_nz(path):
+    """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``.
+
+    Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is
+    not stored in the file). Comment/header lines are skipped.
+
+    Parameters
+    ----------
+    path : str or pathlib.Path
+        n(z) text file (column 0 = z, columns 1… = per-bin n(z)).
+
+    Returns
+    -------
+    tuple
+        ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``)
+        and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``).
+    """
+    table = np.atleast_2d(np.genfromtxt(str(path)))
+    return table[:, 0], table[:, 1:]
+
+
+def covariance_blocks(cov_list, selectors, *, gaussian=True):
+    """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks.
+
+    OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per
+    ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value
+    lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This
+    reshapes the flat table into dense square block(s) — reusing
+    :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block
+    reshape — and pairs each with its SACC selector, ready for
+    :func:`sp_validation.sacc_io.assemble_covariance`.
+
+    Single-statistic case: pass the whole table and one selector; you get one
+    ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a
+    sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a
+    contiguous slice of the flat output for one statistic / bin-pair — and each
+    is reshaped and re-paired with its selector in order. The API is thus shaped
+    to extend to multi-probe blocking without over-fitting the single-bin case.
+
+    Parameters
+    ----------
+    cov_list : numpy.ndarray or sequence
+        Either the flat OneCovariance table (2-D array, one row per pair) for a
+        single block, or — for the multi-block form — a sequence of
+        ``(selector, sub_table)`` pairs. In the multi-block form ``selectors``
+        must be ``None`` (the selectors travel with the sub-tables).
+    selectors : selector or None
+        For the single-block form, the SACC selector for the whole table (a
+        ``(data_type, tracers[, tags])`` tuple or an index array, as
+        :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the
+        multi-block form.
+    gaussian : bool, optional
+        Select the Gaussian-only column (``True``, default) or the
+        Gaussian+non-Gaussian column (``False``); passed straight through to
+        ``cov_from_one_covariance``.
+
+    Returns
+    -------
+    list
+        Ordered ``(selector, dense_cov)`` pairs, directly consumable by
+        ``sacc_io.assemble_covariance(s, blocks)``.
+    """
+    if selectors is None:
+        # Multi-block form: cov_list is a sequence of (selector, sub_table).
+        return [
+            (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian))
+            for selector, sub in cov_list
+        ]
+    # Single-block form: one flat table, one selector.
+    return [
+        (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian))
+    ]