diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 689f4bcb..a1878051 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -1,11 +1,11 @@ name: Build and deploy API documentation # Build the Sphinx docs inside the published image — which already carries the -# full scientific stack autodoc must import — installing the *checked-out* -# package on top so the docs reflect the code under review, not the code baked -# into the image. +# full scientific stack autodoc must import — syncing the *checked-out* uv.lock +# (plus its `docs` extra) and package on top, so the docs reflect the code under +# review and never re-resolve against PyPI. # -# pull_request → build only, as a check (no deploy) +# pull_request → build only, as a check (no deploy), when docs inputs change # push: develop → build + deploy to GitHub Pages on: push: @@ -14,6 +14,12 @@ on: pull_request: branches: - develop + paths: + - "docs/**" + - "src/**" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/deploy-docs.yml" workflow_dispatch: jobs: @@ -23,6 +29,9 @@ jobs: permissions: contents: write packages: read + concurrency: + group: gh-pages-${{ github.ref }} + cancel-in-progress: true container: image: ghcr.io/cosmostat/sp_validation:develop @@ -32,10 +41,17 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + # Exactly what uv.lock pins: --frozen never re-resolves, and --inexact + # keeps the image's packages outside this closure (the ShapePipe stack, + # the glass/workflow extras) instead of pruning them. Both land in the + # image's venv (UV_PROJECT_ENVIRONMENT). The project isn't uv-packaged, so + # the checkout goes in by hand, replacing the copy baked into the image. - name: Install documentation dependencies - run: uv pip install --no-cache-dir '.[docs]' + run: | + uv sync --frozen --inexact --no-cache --no-install-project --extra docs + uv pip install --no-cache --no-deps -e . # Builds on every event; a failing build fails the PR check. Deploy is # gated to develop pushes below. @@ -47,14 +63,14 @@ jobs: # Upload the rendered HTML so it can be downloaded from the run summary — # the only way to preview the docs on a PR, where the deploy is skipped. - name: Upload built docs - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: docs-html path: docs/_build - name: Deploy to GitHub Pages if: github.event_name == 'push' && github.ref == 'refs/heads/develop' - uses: peaceiris/actions-gh-pages@v4 + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: docs/_build diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 192e87a6..19995403 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -9,6 +9,26 @@ env: BRANCH: ${{ github.ref }} jobs: + # The workflow's DAG properties, checked through the host launcher: Snakemake + # on the image's Python with sp_validation absent, as on a user's machine. + workflow-dag: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + + - name: DAG tests + run: >- + uv run --isolated --no-project --python 3.12 + --with snakemake --with snakemake-executor-plugin-slurm + --with pytest + pytest workflow/tests -m "not candide" + build-and-push-image: runs-on: - ubuntu-latest @@ -19,23 +39,23 @@ jobs: steps: - name: Log in to the Container registry - uses: docker/login-action@v3 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and export to Docker - uses: docker/build-push-action@v6 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: load: true tags: ${{ steps.meta.outputs.tags }} @@ -52,7 +72,7 @@ jobs: run: docker run --rm ${{ steps.meta.outputs.tags }} python -m pytest src/sp_validation/tests -m "not slow" - name: Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 with: push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a0f36765..f4e87e9d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -80,13 +80,13 @@ jobs: steps: - name: Checkout (PR head on pull_request_target, else the pushed ref) - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 # Is this a PR whose head branch lives in THIS repo (not a fork)? Only then # can we push an autofix commit back to it with the workflow token. @@ -224,7 +224,7 @@ jobs: - name: Tell the author (PR comment) or record it (develop-push issue) if: steps.ruff.outputs.tool_error == 'false' continue-on-error: true - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: PASSED: ${{ steps.gate.outputs.passed }} AUTOFIXED: ${{ steps.gate.outputs.autofixed }} diff --git a/.gitignore b/.gitignore index e9b50f37..4af80f09 100644 --- a/.gitignore +++ b/.gitignore @@ -198,4 +198,18 @@ papers/cosmo_val/logs/ # Ignore scratch notebooks scratch/*/*.ipynb scratch/guerrini/work_notebooks -scratch/guerrini/launch_scripts \ No newline at end of file +scratch/guerrini/launch_scripts + +# Snakemake run state +.snakemake/ + +# CI lint-gate scratch output (lint.yml writes these in the run tree; never commit) +check.txt +format.txt +report.md +check2.txt +format2.txt +residual.md + +# Generated by docs/source/_ext/catalogue_columns.py from config/columns/ +docs/source/catalogue_columns.md diff --git a/CLAUDE.md b/CLAUDE.md index e985d6c2..9ae25ba6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ Tests live in `src/sp_validation/tests/` and import the full scientific stack, so run them inside the container. - Run all tests: `pytest` (collects from `src/sp_validation/tests`; coverage on by default) - Skip the slow tests: `pytest -m "not slow"` -- Run a single test: `pytest src/sp_validation/tests/test_cosmology.py::test_function_name` +- Run a single test: `pytest src/sp_validation/tests/test_cosmo_val.py::test_function_name` CI runs this same suite inside the freshly-built image before publishing it (see `.github/workflows/deploy-image.yml`). @@ -43,7 +43,8 @@ is the container (full scientific stack pre-built). For a local dev environment: - `cosmo_val.py`: Cosmology validation routines - `cosmology.py`: Cosmological calculations and theory - `galaxy.py`: Galaxy-specific processing -- `io.py`: Input/output utilities +- `grammar.py`: Column grammars: ShapePipe v1 -> v2 adapter and `column_map` renames (`adapt`) +- `io.py`: Input/output; the catalogue reader (`read_catalogue`, `Catalogue`), which detects FITS/HDF5 layouts - `plots.py`: Plotting functions - `rho_tau.py`: Rho and tau statistics calculations - `statistics.py`: Cosmology-independent statistics (jackknife resampling, χ²/PTE, covariance↔correlation, OneCovariance reshaping) @@ -57,18 +58,20 @@ is the container (full scientific stack pre-built). For a local dev environment: - **Healpy/HealSparse**: Sky map handling ### Cosmology Inference Pipeline (`cosmo_inference/`) -Run via `./pipeline.sh` with flags: -- `--pcf`: Calculate 2-point correlation functions -- `--covmat`: Calculate covariance matrix with CosmoCov -- `--inference`: Run CosmoSIS inference -- `--mcmc_process`: Analyze MCMC chains +Orchestrated through Snakemake, not a standalone driver; see +`cosmo_inference/README.md`. From the repository root: + +```bash +snakemake --profile workflow/profiles/candide -s workflow/Snakefile \ + inference_fiducial --configfile +``` ### Configuration Main configuration in `scripts/calibration/params.py` with parameters: -- `name`: Field/patch identifier +- `campaign`: Campaign name (the ShapePipe tile list); names the input products - `data_dir`: Input data directory - `galaxy_cat_path`: Galaxy catalogue path (.fits/.hdf5) -- `star_cat_path`: Star catalogue path (.fits) +- `star_cat_path`: Star catalogue path (.hdf5, or a v1 .fits) ### Key Dependencies - astropy, numpy, scipy for core calculations @@ -78,10 +81,26 @@ Main configuration in `scripts/calibration/params.py` with parameters: - pyccl for cosmological calculations ## Container Usage -Recommended installation via Apptainer/Docker: +Nothing is hand-built. CI publishes `ghcr.io/cosmostat/sp_validation:` on +every push, and each person keeps their own copy at +`~/.cache/sp_validation/sp_validation.sif`, managed by the `spv-container` CLI: + ```bash -apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop +spv-container pull # fetch :develop there (do it from a compute node) +spv-container status # which layer is live, and how current it is +spv-container exec # one-off run inside it ``` +Need a package the image lacks mid-analysis? `spv-container sandbox`, then +`spv-container exec --writable pip install `; the sandbox then takes +precedence over the SIF everywhere, workflow jobs included. + +Every rule runs inside that image, wrapped by Snakemake itself (`--profile +workflow/profiles/candide` on the cluster, `workflow/profiles/default -j N` +elsewhere). The `sp_validation` a rule imports comes from the *launched +checkout*, not the image. + +`workflow/README.md` is the full story — profiles, image resolution, refresh. + ## Notebook Configuration - The CosmologyValidation class must be initialized in cosmo_val \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00a8d907..0406741b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,15 +13,49 @@ inside the project container, which ships the full stack pre-built. ### Container (recommended) +CI builds and pushes an image on **every** push, tagged by the sanitized branch +name (see +[`.github/workflows/deploy-image.yml`](.github/workflows/deploy-image.yml)), so +`:develop` tracks the integration branch and your branch has an image of its +own. Nothing is built by hand. + +You keep your own copy of the image. `spv-container` — stdlib-only, so it runs +straight from a checkout: symlink it onto your PATH (`ln -s +"$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container`, the README's +install step) or call it as `python3 src/sp_validation/container.py` — pulls it to +`~/.cache/sp_validation/sp_validation.sif` and runs things inside it: + ```bash -# build a writeable sandbox from the published image -apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop -apptainer shell --writable sp_validation +spv-container pull # fetch :develop; ~1.5 GB, so do it from a compute node +spv-container status # which layer is live, and how current it is +spv-container exec bash # an interactive shell inside it ``` -The image is rebuilt and pushed on every push to `develop` (see -[`.github/workflows/deploy-image.yml`](.github/workflows/deploy-image.yml)), so -`:develop` always tracks the latest integration branch. +That image is read-only. When you need a package it does not carry yet, unpack a +writable sandbox once with `spv-container sandbox` and install into it with +`spv-container exec --writable pip install `; the sandbox then takes +precedence everywhere, workflow jobs included. Treat it as an exploration tool — +the real fix is adding the dependency to `pyproject.toml` — and reset it with +`spv-container pull && spv-container sandbox --force`. + +Analysis runs through Snakemake, which wraps every job in `apptainer exec` +against that same image for you — see +[`workflow/README.md`](workflow/README.md) for the profiles and the details. + +Two things worth knowing while developing: + +- **Your checkout's code is what runs.** The workflow prepends the launched + checkout's `src/` to the container's `PYTHONPATH`, so the image supplies the + dependency stack and your working tree supplies `sp_validation`. No rebuild + needed to test a change. (Caveat: `rerun-triggers: code` does not watch + `src/`, so force reruns after editing a module.) +- **To test a branch's own image** — when the *stack* changed, not just `src/` — + point the workflow at its CI tag: + + ```bash + snakemake --profile workflow/profiles/candide \ + --config container=docker://ghcr.io/cosmostat/sp_validation:my-branch + ``` ### Local install with `uv` @@ -40,7 +74,7 @@ toolchain (`autoconf`, `automake`, `libtool`, `pkg-config`) available. ```bash pytest # full suite pytest -m "not slow" # skip the slow tests -pytest src/sp_validation/tests/test_cosmology.py::test_name # a single test +pytest src/sp_validation/tests/test_cosmo_val.py::test_name # a single test ``` Tests live in `src/sp_validation/tests/`. The default options (configured in diff --git a/Dockerfile b/Dockerfile index ccc930bf..f9272fd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,10 @@ # Development image with more bells and whistles -FROM ghcr.io/cosmostat/shapepipe:im_sims +FROM ghcr.io/cosmostat/shapepipe:develop +# liblapack-dev: cosmosis's MultiNest links -llapack, and the base image ships +# only the runtime liblapack.so.3 (no dev symlink). The gsl/cfitsio/fftw3 dev +# packages are what the CosmoSIS Standard Library's C sources compile against +# (they are the headers CSL's own CI installs); git is for cloning it. RUN apt-get update -y --quiet --fix-missing && \ apt-get dist-upgrade -y --quiet --fix-missing && \ apt-get install -y --quiet \ @@ -8,14 +12,56 @@ RUN apt-get update -y --quiet --fix-missing && \ automake \ libtool \ pkg-config \ + git \ htop \ npm \ - tmux + tmux \ + liblapack-dev \ + libgsl-dev \ + libcfitsio-dev \ + libfftw3-dev \ + perl \ + curl \ + ghostscript && \ + rm -rf /var/lib/apt/lists/* + +# TinyTeX pinned to a TeX Live year (frozen tlnet-final mirror); bump both once a year. +# The packages serve matplotlib's usetex figures; sfmath gives them sans-serif +# maths (`\usepackage[cm]{sfmath}`). +ENV TEXLIVE_YEAR=2025 \ + TINYTEX_VERSION=2026.02 \ + TINYTEX_DIR=/opt \ + PATH=/opt/.TinyTeX/bin/x86_64-linux:$PATH +RUN set -eux; \ + curl -fsSL https://yihui.org/tinytex/install-bin-unix.sh | sh; \ + tlmgr option sys_bin /usr/local/bin; \ + tlmgr option repository \ + "https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${TEXLIVE_YEAR}/tlnet-final/"; \ + tlmgr option docfiles 0; \ + tlmgr option srcfiles 0; \ + tlmgr install \ + type1cm \ + cm-super \ + dvipng \ + underscore \ + ulem \ + amsmath \ + amsfonts \ + geometry \ + sfmath \ + xcolor; \ + tlmgr path add; \ + latex --version >/dev/null; dvipng --version >/dev/null # The base shapepipe image provides a uv-managed venv at /app/.venv (exported as # VIRTUAL_ENV); install sp_validation's deps into that same venv rather than # spawning a second one under /sp_validation. ENV UV_PROJECT_ENVIRONMENT=/app/.venv +# $HOME is bind-mounted under apptainer, so uv would otherwise discover the +# host's managed CPythons -- including newer ones that satisfy requires-python +# -- and build a venv against an interpreter carrying none of this stack. +ENV UV_PYTHON=/app/.venv/bin/python \ + UV_PYTHON_DOWNLOADS=never WORKDIR /sp_validation @@ -26,13 +72,61 @@ WORKDIR /sp_validation # our lock — instead of pruning them. Copy the lock + manifest first so this # layer caches independently of source edits. Extras: test (CI unit suite), # glass (GLASS map-level mock — pulls glass.ext.camb + the cosmology wrapper), -# workflow (Snakemake + mpi4py runners). cs_util 0.2.2 (with cs_util.size) and a -# numba-safe numpy 2.4.6 come straight from the lock, so the old ad-hoc snakemake -# and cs_util `--upgrade` layers are gone. +# workflow (mpi4py, CosmoSIS and the other rule-script runners). cs_util and a +# numba-safe numpy come straight from the lock. COPY pyproject.toml uv.lock /sp_validation/ + +# cosmosis builds MPI-enabled polychord/multinest only when MPIFC is set: its +# setup.py exports MPIFC for conda builds only, and the sampler Makefiles gate on +# `which $(MPIFC)`. Absolute path, not a bare name: /opt/ompi/bin is not always on +# PATH, and a miss silently omits libchord_mpi.so while the install still succeeds, +# and `cosmosis --mpi` fails at load time -- which is how the pipeline runs, since +# the --smp pool is broken upstream. Must precede the sync that builds cosmosis. +ENV MPIFC=/opt/ompi/bin/mpif90 + RUN uv sync --frozen --inexact --no-install-project \ --extra test --extra glass --extra workflow +# The CosmoSIS Standard Library: the module files (camb interface, projection, +# 2pt likelihood, ...) the cosmo_inference pipelines name. The `workflow` extra +# above installs cosmosis itself; CSL is a separate tree of modules that is not +# on PyPI and has to be built against that install, so it is cloned and compiled +# here rather than left to each user (which is what the .ini templates used to +# assume, hard-coding one person's home directory). +# +# CSL_REF pins UNIONS-WL fork main: Sacha's four UNIONS commits reapplied +# on current upstream, including the scipy lpn fix. +ARG CSL_REPO=https://github.com/UNIONS-WL/cosmosis-standard-library.git +ARG CSL_REF=b7b1552a02ad9c39c9bb1e68e3f17213a8f740e1 +ENV CSL_DIR=/opt/cosmosis-standard-library + +# `python -m cosmosis.configure` emits the exports (COSMOSIS_SRC_DIR et al.) +# every CSL Makefile includes its compiler config from. Evaluated directly +# rather than through the `cosmosis-configure` wrapper: that wrapper's +# am-I-sourced probe reads unset zsh/ksh variables, which `set -u` turns into +# an error, and its `exit` then ends the sourcing shell with status 0 — make +# never runs and the layer still "succeeds". The trailing `test -f` keeps any +# such silent no-op loud. +# +# `make -C shear` rather than a bare `make`: the top-level target also descends +# into likelihood/, which builds the Planck, WMAP and ACT likelihoods -- large, +# data-dependent, and unused by any UNIONS pipeline. Everything our .ini +# templates reference is either pure Python (consistency, sample_S8, camb, +# load_nz_fits, photoz_bias, linear_alignment, add_intrinsic, shear_m_bias, +# xi_sys, 2pt_like -- no Makefile in those trees at all) or lives under shear/: +# `limber`, which project_2d.py links, and `cl_to_xi_nicaea`, whose +# nicaea_interface.so the 2pt_shear stage loads. +RUN bash -c 'set -eo pipefail; \ + export PATH=/app/.venv/bin:$PATH; \ + git clone --filter=blob:none "$CSL_REPO" "$CSL_DIR"; \ + cd "$CSL_DIR"; \ + git checkout --detach "$CSL_REF"; \ + cmds=$(python -m cosmosis.configure); \ + eval "$cmds"; \ + export GSL_INC=/usr/include GSL_LIB=/usr/lib/x86_64-linux-gnu; \ + make -C shear; \ + test -f shear/cl_to_xi_nicaea/nicaea_interface.so' + # Install sp_validation itself (editable) into the same venv; deps are already # satisfied by the sync above. COPY . /sp_validation diff --git a/README.md b/README.md index e14a2071..1d269a0c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Validation of weak-lensing catalogues (galaxy and star shapes and other paramete [![docs](https://img.shields.io/badge/docs-sphinx-blue)](https://cosmostat.github.io/sp_validation/) [![CI](https://github.com/CosmoStat/sp_validation/actions/workflows/deploy-image.yml/badge.svg)](https://github.com/CosmoStat/sp_validation/actions/workflows/deploy-image.yml) [![container](https://img.shields.io/badge/container-ghcr.io-2496ED?logo=docker&logoColor=white)](https://github.com/CosmoStat/sp_validation/pkgs/container/sp_validation) -[![python](https://img.shields.io/badge/python-3.11%2B-blue?logo=python&logoColor=white)](https://www.python.org/downloads/) +[![python](https://img.shields.io/badge/python-3.12%2B-blue?logo=python&logoColor=white)](https://www.python.org/downloads/) [![license](https://img.shields.io/badge/license-MIT-blue)](https://github.com/CosmoStat/sp_validation/blob/develop/LICENCE.txt) [![code style: ruff](https://img.shields.io/badge/code%20style-ruff-261230?logo=ruff&logoColor=white)](https://github.com/astral-sh/ruff) [![contribute](https://img.shields.io/badge/contribute-read-lightgrey)](https://github.com/CosmoStat/sp_validation/blob/develop/CONTRIBUTING.md) @@ -62,30 +62,35 @@ directive imports the shared rules under each run's own config and an output `prefix`, so runs namespace under `results//` without clobbering one another. -## Container Installation (Recommended) +## Installation -The easiest way to install sp_validation is via a container. Docker images are automatically built and pushed to the [GitHub Container Registry (GHCR)](https://github.com/CosmoStat/sp_validation/pkgs/container/sp_validation) on every push to `develop`. This image can be installed and run on most systems (including clusters) with just a few lines of code. - -We recommend running the image with **Apptainer** (formerly Singularity) which is installed on most HPC clusters. To simply run the image, use the following command: +`sp_validation` runs from a pre-built container: CI builds an image carrying +the full scientific stack on every push and publishes it to the +[GitHub Container Registry](https://github.com/CosmoStat/sp_validation/pkgs/container/sp_validation). +The bundled `spv-container` CLI installs and manages your personal copy of it: ```bash -# build writeable "sandbox" container in the current directory -# ./sp_validation will be a directory that functions like a vm -apptainer build --sandbox sp_validation docker://ghcr.io/cosmostat/sp_validation:develop - -# open a shell in the container -apptainer shell --writable sp_validation -# and confirm that the installation was successful -python -c "import sp_validation" -``` +git clone https://github.com/CosmoStat/sp_validation.git +cd sp_validation +ln -s "$PWD/src/sp_validation/container.py" ~/.local/bin/spv-container -You can also run the image with **Docker**: - -```bash -docker run --rm -it ghcr.io/cosmostat/sp_validation:develop python -c "import sp_validation" +spv-container pull # fetch the image (~1.5 GB) +spv-container exec python -c "import sp_validation" # confirm it works ``` -We do not currently build images for Apple Silicon/arm64; however the amd64 images should work on these systems, albeit with reduced performance. +That is the whole install. `pull` puts the image at its canonical per-user +path (`~/.cache/sp_validation/`), and everything else finds it there — +`spv-container exec` for one-off commands (`spv-container exec bash` for an +interactive shell) and the Snakemake workflow for cluster jobs. +`spv-container status` says what you have and how current it is; +`spv-container sandbox` gives you a writable copy for mid-analysis +`pip install`s. On a cluster, run the pull from a compute node. + +To run the analysis workflow (`workflow/`), see +[`workflow/README.md`](workflow/README.md): Snakemake runs on the host, and +the profile puts each job in the container itself. For Docker, development +installs, and more depth, see the +[installation docs](https://cosmostat.github.io/sp_validation/installation.html). diff --git a/check.txt b/check.txt deleted file mode 100644 index 1f5f344d..00000000 --- a/check.txt +++ /dev/null @@ -1 +0,0 @@ -All checks passed! diff --git a/config/calibration/mask_from_compr_v1.X.11.yaml b/config/calibration/mask_from_compr_v1.X.11.yaml index 3fa9ee3e..d76d7fb3 100644 --- a/config/calibration/mask_from_compr_v1.X.11.yaml +++ b/config/calibration/mask_from_compr_v1.X.11.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # Duplicate objects @@ -24,29 +24,29 @@ dat: kind: greater_equal value: 2 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_from_compr_v1.X.6.yaml b/config/calibration/mask_from_compr_v1.X.6.yaml index 1c9b08e5..a71f9974 100644 --- a/config/calibration/mask_from_compr_v1.X.6.yaml +++ b/config/calibration/mask_from_compr_v1.X.6.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # Duplicate objects - col_name: overlap @@ -23,29 +23,29 @@ dat: kind: greater_equal value: 2 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.10.yaml b/config/calibration/mask_v1.X.10.yaml index 703ea91b..2678a49b 100644 --- a/config/calibration/mask_v1.X.10.yaml +++ b/config/calibration/mask_v1.X.10.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.11.yaml b/config/calibration/mask_v1.X.11.yaml index 88e01643..b87aa267 100644 --- a/config/calibration/mask_v1.X.11.yaml +++ b/config/calibration/mask_v1.X.11.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.2.yaml b/config/calibration/mask_v1.X.2.yaml index 846f14ff..e7ad7758 100644 --- a/config/calibration/mask_v1.X.2.yaml +++ b/config/calibration/mask_v1.X.2.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,40 +57,40 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Faint star halos - - col_name: 1_Faint_star_halos + - col_name: MASK_1_Faint_star_halos label: "Faint star halos" kind: equal value: False # Bright star halos - - col_name: 2_Bright_star_halos + - col_name: MASK_2_Bright_star_halos label: "Bright star halos" kind: equal value: False # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.3.yaml b/config/calibration/mask_v1.X.3.yaml index bcbbc620..3c14c5fd 100644 --- a/config/calibration/mask_v1.X.3.yaml +++ b/config/calibration/mask_v1.X.3.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.4.yaml b/config/calibration/mask_v1.X.4.yaml index 903b9098..37b67325 100644 --- a/config/calibration/mask_v1.X.4.yaml +++ b/config/calibration/mask_v1.X.4.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,40 +57,40 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Faint star halos - - col_name: 1_Faint_star_halos + - col_name: MASK_1_Faint_star_halos label: "Faint star halos" kind: equal value: False # Bright star halos - - col_name: 2_Bright_star_halos + - col_name: MASK_2_Bright_star_halos label: "Bright star halos" kind: equal value: False # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.4_im_sim.yaml b/config/calibration/mask_v1.X.4_im_sim.yaml index 794566d4..8f3636c2 100644 --- a/config/calibration/mask_v1.X.4_im_sim.yaml +++ b/config/calibration/mask_v1.X.4_im_sim.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -30,17 +30,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 diff --git a/config/calibration/mask_v1.X.5.yaml b/config/calibration/mask_v1.X.5.yaml index 5f56b135..bf717380 100644 --- a/config/calibration/mask_v1.X.5.yaml +++ b/config/calibration/mask_v1.X.5.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.6.yaml b/config/calibration/mask_v1.X.6.yaml index 1b5f7da6..fc021a2c 100644 --- a/config/calibration/mask_v1.X.6.yaml +++ b/config/calibration/mask_v1.X.6.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.6_ppv1.yaml b/config/calibration/mask_v1.X.6_ppv1.yaml index a3d29cda..bcaffb15 100644 --- a/config/calibration/mask_v1.X.6_ppv1.yaml +++ b/config/calibration/mask_v1.X.6_ppv1.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -42,17 +42,17 @@ dat: value: [15, 30] # ngmix flags - - col_name: NGMIX_MOM_FAIL + - col_name: NGMIX_MCAL_TYPES_FAIL label: "ngmix moments failure" kind: equal value: 0 # invalid PSF ellipticities - - col_name: NGMIX_ELL_PSFo_NOSHEAR_0 + - col_name: NGMIX_G1_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 1" kind: not_equal value: -10 - - col_name: NGMIX_ELL_PSFo_NOSHEAR_1 + - col_name: NGMIX_G2_PSF_ORIG_NOSHEAR label: "bad PSF ellipticity comp 2" kind: not_equal value: -10 @@ -84,29 +84,29 @@ dat: col_name2: MAG_GAAP_0p7_z2 value: -99 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.7.yaml b/config/calibration/mask_v1.X.7.yaml index cbc802be..585481d9 100644 --- a/config/calibration/mask_v1.X.7.yaml +++ b/config/calibration/mask_v1.X.7.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,34 +57,34 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Bright star halos - - col_name: 2_Bright_star_halos + - col_name: MASK_2_Bright_star_halos label: "Bright star halos" kind: equal value: False # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.8.yaml b/config/calibration/mask_v1.X.8.yaml index 58234d64..5dafbb72 100644 --- a/config/calibration/mask_v1.X.8.yaml +++ b/config/calibration/mask_v1.X.8.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,40 +57,40 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Faint star halos - - col_name: 1_Faint_star_halos + - col_name: MASK_1_Faint_star_halos label: "Faint star halos" kind: equal value: False # Bright star halos - - col_name: 2_Bright_star_halos + - col_name: MASK_2_Bright_star_halos label: "Bright star halos" kind: equal value: False # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.9.yaml b/config/calibration/mask_v1.X.9.yaml index 4a448ce7..906578df 100644 --- a/config/calibration/mask_v1.X.9.yaml +++ b/config/calibration/mask_v1.X.9.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS @@ -57,29 +57,29 @@ dat: kind: not_equal value: -10 -## Using columns in 'dat_ext' group (post-processing flags) -dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.9_im_sim.overlay.yaml b/config/calibration/mask_v1.X.9_im_sim.overlay.yaml index 94c5b9ae..9c15edab 100644 --- a/config/calibration/mask_v1.X.9_im_sim.overlay.yaml +++ b/config/calibration/mask_v1.X.9_im_sim.overlay.yaml @@ -45,36 +45,36 @@ ops: with: |2 # invalid PSF ellipticities (ShapePipe-v2 grammar: scalar G1/G2 components) - # --- dat_ext (post-processing / coverage masks) ------------------------ + # --- mask bits (post-processing / coverage masks) ---------------------- - why: >- - No coverage masks on sims: the whole dat_ext group (Stars, manual mask, - r-band footprint, Maximask) is survey post-processing with no analogue - in the simulated tiles. + No coverage masks on sims: the healsparse mask-bit cuts (stars, manual + mask, r coverage, Maximask) are survey post-processing with no analogue in the + simulated tiles. drop: |2 - ## Using columns in 'dat_ext' group (post-processing flags) - dat_ext: + # Healsparse mask bits and post-processing columns (the comprehensive + # HDF5's data_ext dataset) # Stars - - col_name: 4_Stars + - col_name: MASK_4_Stars label: "Stars" kind: equal value: False # Manual mask - - col_name: 8_Manual + - col_name: MASK_8_Manual label: "manual mask" kind: equal value: False # r-band footprint - - col_name: 64_r + - col_name: MASK_64_r label: "r-band imaging" kind: equal value: False # Maximask - - col_name: 1024_Maximask + - col_name: MASK_1024_Maximask label: "maximask" kind: equal value: False diff --git a/config/calibration/mask_v1.X.9_im_sim.yaml b/config/calibration/mask_v1.X.9_im_sim.yaml index 1c6ff672..3c28cc03 100644 --- a/config/calibration/mask_v1.X.9_im_sim.yaml +++ b/config/calibration/mask_v1.X.9_im_sim.yaml @@ -9,7 +9,7 @@ params: verbose: True # Masks -## Using columns in 'dat' group (ShapePipe flags) +## Cuts on catalogue columns, in the v2 grammar (sp_validation.grammar) dat: # SExtractor flags - col_name: FLAGS diff --git a/config/calibration/mask_v2.0.yaml b/config/calibration/mask_v2.0.yaml new file mode 100644 index 00000000..930ae42f --- /dev/null +++ b/config/calibration/mask_v2.0.yaml @@ -0,0 +1,160 @@ +# Config file for masking and calibration, ShapePipe v2 catalogues. +# +# ShapePipe v2 replaces the single IMAFLAGS_ISO column with per-reason boolean +# mask columns MASK__