Merge develop into the tomography branch

.github/workflows/deploy-docs.yml

@@ -1,11 +1,11 @@
 name: Build and deploy API documentation
 
 # Build the Sphinx docs inside the published image — which already carries the
-# full scientific stack autodoc must import — installing the *checked-out*
-# package on top so the docs reflect the code under review, not the code baked
-# into the image.
+# full scientific stack autodoc must import — syncing the *checked-out* uv.lock
+# (plus its `docs` extra) and package on top, so the docs reflect the code under
+# review and never re-resolve against PyPI.
 #
-#   pull_request → build only, as a check (no deploy)
+#   pull_request → build only, as a check (no deploy), when docs inputs change
 #   push: develop → build + deploy to GitHub Pages
 on:
   push:
@@ -14,6 +14,12 @@ on:
   pull_request:
     branches:
       - develop
+    paths:
+      - "docs/**"
+      - "src/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/deploy-docs.yml"
   workflow_dispatch:
 
 jobs:
@@ -23,6 +29,9 @@ jobs:
     permissions:
       contents: write
       packages: read
+    concurrency:
+      group: gh-pages-${{ github.ref }}
+      cancel-in-progress: true
 
     container:
       image: ghcr.io/cosmostat/sp_validation:develop
@@ -32,10 +41,17 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
 
+      # Exactly what uv.lock pins: --frozen never re-resolves, and --inexact
+      # keeps the image's packages outside this closure (the ShapePipe stack,
+      # the glass/workflow extras) instead of pruning them. Both land in the
+      # image's venv (UV_PROJECT_ENVIRONMENT). The project isn't uv-packaged, so
+      # the checkout goes in by hand, replacing the copy baked into the image.
       - name: Install documentation dependencies
-        run: uv pip install --no-cache-dir '.[docs]'
+        run: |
+          uv sync --frozen --inexact --no-cache --no-install-project --extra docs
+          uv pip install --no-cache --no-deps -e .
 
       # Builds on every event; a failing build fails the PR check. Deploy is
       # gated to develop pushes below.
@@ -47,14 +63,14 @@ jobs:
       # Upload the rendered HTML so it can be downloaded from the run summary —
       # the only way to preview the docs on a PR, where the deploy is skipped.
       - name: Upload built docs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: docs-html
           path: docs/_build
 
       - name: Deploy to GitHub Pages
         if: github.event_name == 'push' && github.ref == 'refs/heads/develop'
-        uses: peaceiris/actions-gh-pages@v4
+        uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: docs/_build

.github/workflows/deploy-image.yml

@@ -9,6 +9,26 @@ env:
   BRANCH: ${{ github.ref }}
 
 jobs:
+  # The workflow's DAG properties, checked through the host launcher: Snakemake
+  # on the image's Python with sp_validation absent, as on a user's machine.
+  workflow-dag:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+
+      - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+
+      - name: DAG tests
+        run: >-
+          uv run --isolated --no-project --python 3.12
+          --with snakemake --with snakemake-executor-plugin-slurm
+          --with pytest
+          pytest workflow/tests -m "not candide"
+
   build-and-push-image:
     runs-on: 
       - ubuntu-latest
@@ -19,23 +39,23 @@ jobs:
 
     steps:
       - name: Log in to the Container registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Build and export to Docker
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
         with:
           load: true
           tags: ${{ steps.meta.outputs.tags }}
@@ -52,7 +72,7 @@ jobs:
         run: docker run --rm ${{ steps.meta.outputs.tags }} python -m pytest src/sp_validation/tests -m "not slow"
 
       - name: Push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
         with:
           push: true
           tags: ${{ steps.meta.outputs.tags }}

.github/workflows/lint.yml

@@ -80,13 +80,13 @@ jobs:
 
     steps:
       - name: Checkout (PR head on pull_request_target, else the pushed ref)
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
         with:
           ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
 
       # Is this a PR whose head branch lives in THIS repo (not a fork)? Only then
       # can we push an autofix commit back to it with the workflow token.
@@ -224,7 +224,7 @@ jobs:
       - name: Tell the author (PR comment) or record it (develop-push issue)
         if: steps.ruff.outputs.tool_error == 'false'
         continue-on-error: true
-        uses: actions/github-script@v7
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
         env:
           PASSED: ${{ steps.gate.outputs.passed }}
           AUTOFIXED: ${{ steps.gate.outputs.autofixed }}

.gitignore

@@ -198,4 +198,18 @@ papers/cosmo_val/logs/
 # Ignore scratch notebooks
 scratch/*/*.ipynb
 scratch/guerrini/work_notebooks
-scratch/guerrini/launch_scripts
+scratch/guerrini/launch_scripts
+
+# Snakemake run state
+.snakemake/
+
+# CI lint-gate scratch output (lint.yml writes these in the run tree; never commit)
+check.txt
+format.txt
+report.md
+check2.txt
+format2.txt
+residual.md
+
+# Generated by docs/source/_ext/catalogue_columns.py from config/columns/
+docs/source/catalogue_columns.md

CLAUDE.md

@@ -18,7 +18,7 @@ Tests live in `src/sp_validation/tests/` and import the full scientific stack,
 so run them inside the container.
 - Run all tests: `pytest` (collects from `src/sp_validation/tests`; coverage on by default)
 - Skip the slow tests: `pytest -m "not slow"`
-- Run a single test: `pytest src/sp_validation/tests/test_cosmology.py::test_function_name`
+- Run a single test: `pytest src/sp_validation/tests/test_cosmo_val.py::test_function_name`
 
 CI runs this same suite inside the freshly-built image before publishing it
 (see `.github/workflows/deploy-image.yml`).
@@ -43,7 +43,8 @@ is the container (full scientific stack pre-built). For a local dev environment:
 - `cosmo_val.py`: Cosmology validation routines
 - `cosmology.py`: Cosmological calculations and theory
 - `galaxy.py`: Galaxy-specific processing
-- `io.py`: Input/output utilities
+- `grammar.py`: Column grammars: ShapePipe v1 -> v2 adapter and `column_map` renames (`adapt`)
+- `io.py`: Input/output; the catalogue reader (`read_catalogue`, `Catalogue`), which detects FITS/HDF5 layouts
 - `plots.py`: Plotting functions
 - `rho_tau.py`: Rho and tau statistics calculations
 - `statistics.py`: Cosmology-independent statistics (jackknife resampling, χ²/PTE, covariance↔correlation, OneCovariance reshaping)
@@ -57,18 +58,20 @@ is the container (full scientific stack pre-built). For a local dev environment:
 - **Healpy/HealSparse**: Sky map handling
 
 ### Cosmology Inference Pipeline (`cosmo_inference/`)
-Run via `./pipeline.sh` with flags:
-- `--pcf`: Calculate 2-point correlation functions
-- `--covmat`: Calculate covariance matrix with CosmoCov
-- `--inference`: Run CosmoSIS inference
-- `--mcmc_process`: Analyze MCMC chains
+Orchestrated through Snakemake, not a standalone driver; see
+`cosmo_inference/README.md`. From the repository root:
+
+```bash
+snakemake --profile workflow/profiles/candide -s workflow/Snakefile \
+    inference_fiducial --configfile <run config>
+```
 
 ### Configuration
 Main configuration in `scripts/calibration/params.py` with parameters:
-- `name`: Field/patch identifier
+- `campaign`: Campaign name (the ShapePipe tile list); names the input products
 - `data_dir`: Input data directory
 - `galaxy_cat_path`: Galaxy catalogue path (.fits/.hdf5)
-- `star_cat_path`: Star catalogue path (.fits)
+- `star_cat_path`: Star catalogue path (.hdf5, or a v1 .fits)
 
 ### Key Dependencies
 - astropy, numpy, scipy for core calculations
@@ -78,10 +81,26 @@ Main configuration in `scripts/calibration/params.py` with parameters:
 - pyccl for cosmological calculations
 
 ## Container Usage
-Recommended installation via Apptainer/Docker:
+Nothing is hand-built. CI publishes `ghcr.io/cosmostat/sp_validation:<branch>` on
+every push, and each person keeps their own copy at
+`~/.cache/sp_validation/sp_validation.sif`, managed by the `spv-container` CLI:
+
 ```bash
-apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop
+spv-container pull            # fetch :develop there (do it from a compute node)
+spv-container status          # which layer is live, and how current it is
+spv-container exec <command>  # one-off run inside it
 ```
 
+Need a package the image lacks mid-analysis? `spv-container sandbox`, then
+`spv-container exec --writable pip install <pkg>`; the sandbox then takes
+precedence over the SIF everywhere, workflow jobs included.
+
+Every rule runs inside that image, wrapped by Snakemake itself (`--profile
+workflow/profiles/candide` on the cluster, `workflow/profiles/default -j N`
+elsewhere). The `sp_validation` a rule imports comes from the *launched
+checkout*, not the image.
+
+`workflow/README.md` is the full story — profiles, image resolution, refresh.
+
 ## Notebook Configuration
 - The CosmologyValidation class must be initialized in cosmo_val

CONTRIBUTING.md

@@ -13,15 +13,49 @@ inside the project container, which ships the full stack pre-built.
 
 ### Container (recommended)
 
+CI builds and pushes an image on **every** push, tagged by the sanitized branch
+name (see
+[`.github/workflows/deploy-image.yml`](.github/workflows/deploy-image.yml)), so
+`:develop` tracks the integration branch and your branch has an image of its
+own. Nothing is built by hand.
+
+You keep your own copy of the image. `spv-container` — stdlib-only, so it runs
+straight from a checkout: symlink it onto your PATH (`ln -s
+"$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container`, the README's
+install step) or call it as `python3 src/sp_validation/container.py` — pulls it to
+`~/.cache/sp_validation/sp_validation.sif` and runs things inside it:
+
 ```bash
-# build a writeable sandbox from the published image
-apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop
-apptainer shell --writable sp_validation
+spv-container pull       # fetch :develop; ~1.5 GB, so do it from a compute node
+spv-container status     # which layer is live, and how current it is
+spv-container exec bash  # an interactive shell inside it
 ```
 
-The image is rebuilt and pushed on every push to `develop` (see
-[`.github/workflows/deploy-image.yml`](.github/workflows/deploy-image.yml)), so
-`:develop` always tracks the latest integration branch.
+That image is read-only. When you need a package it does not carry yet, unpack a
+writable sandbox once with `spv-container sandbox` and install into it with
+`spv-container exec --writable pip install <pkg>`; the sandbox then takes
+precedence everywhere, workflow jobs included. Treat it as an exploration tool —
+the real fix is adding the dependency to `pyproject.toml` — and reset it with
+`spv-container pull && spv-container sandbox --force`.
+
+Analysis runs through Snakemake, which wraps every job in `apptainer exec`
+against that same image for you — see
+[`workflow/README.md`](workflow/README.md) for the profiles and the details.
+
+Two things worth knowing while developing:
+
+- **Your checkout's code is what runs.** The workflow prepends the launched
+  checkout's `src/` to the container's `PYTHONPATH`, so the image supplies the
+  dependency stack and your working tree supplies `sp_validation`. No rebuild
+  needed to test a change. (Caveat: `rerun-triggers: code` does not watch
+  `src/`, so force reruns after editing a module.)
+- **To test a branch's own image** — when the *stack* changed, not just `src/` —
+  point the workflow at its CI tag:
+
+  ```bash
+  snakemake --profile workflow/profiles/candide \
+      --config container=docker://ghcr.io/cosmostat/sp_validation:my-branch <target>
+  ```
 
 ### Local install with `uv`
 
@@ -40,7 +74,7 @@ toolchain (`autoconf`, `automake`, `libtool`, `pkg-config`) available.
 ```bash
 pytest                       # full suite
 pytest -m "not slow"         # skip the slow tests
-pytest src/sp_validation/tests/test_cosmology.py::test_name   # a single test
+pytest src/sp_validation/tests/test_cosmo_val.py::test_name   # a single test
 ```
 
 Tests live in `src/sp_validation/tests/`. The default options (configured in